scint 0.6.0 → 0.7.1

data/lib/scint/credentials.rb

@@ -13,9 +13,10 @@ module Scint
   # Lookup order (first match wins):
   #   1. Inline credentials in the request URI itself (user:pass@host)
   #   2. Credentials registered at runtime (from Gemfile source: URIs)
-  #   3. Scint config:   $XDG_CONFIG_HOME/scint/credentials
-  #   4. Bundler config:  ~/.bundle/config
-  #   5. Environment variables (BUNDLE_HOST__NAME format)
+  #   3. Bundler local config: $BUNDLE_APP_CONFIG/config or ./.bundle/config
+  #   4. Scint config:   $XDG_CONFIG_HOME/scint/credentials
+  #   5. Bundler global config: ~/.bundle/config
+  #   6. Environment variables (BUNDLE_HOST__NAME / BUNDLE_HTTPS://... format)
   #
   # All config files use Bundler's key format:
   #   BUNDLE_PKGS__SHOPIFY__IO: "token:secret"
@@ -80,7 +81,7 @@ module Scint
       end
 
       # 2–5. Registered + config files + env
-      auth = lookup_host(uri.host)
+      auth = lookup_uri(uri)
       return nil unless auth
 
       user, password = auth.split(":", 2)
@@ -99,27 +100,35 @@ module Scint
 
     private
 
-    def lookup_host(host)
-      return nil unless host
+    def lookup_uri(uri)
+      return nil unless uri&.host
 
       # 2. Runtime-registered (from Gemfile inline URIs)
-      registered = @mutex.synchronize { @registered[host] }
+      registered = @mutex.synchronize { @registered[uri.host] }
       return registered if registered
 
-      # 3–4. Config files (scint, then bundler)
-      key = self.class.key_for_host(host)
-      val = @file_config[key]
-      return val if val
+      # 3–5. Config files (bundler local, scint, bundler global)
+      keys = self.class.keys_for_uri_lookup(uri)
+      keys.each do |key|
+        val = @file_config[key]
+        return val if val
+      end
+
+      # 6. Environment variable
+      keys.each do |key|
+        val = ENV[key]
+        return val if val
+      end
 
-      # 5. Environment variable
-      ENV[key]
+      nil
     end
 
     def load_config_files
       config = {}
       # Load in reverse priority (later overrides earlier)
-      load_yaml_into(config, bundler_config_path)
+      load_yaml_into(config, bundler_global_config_path)
       load_yaml_into(config, scint_credentials_path)
+      load_yaml_into(config, bundler_local_config_path)
       config
     end
 
@@ -137,7 +146,17 @@ module Scint
       File.join(xdg, "scint", "credentials")
     end
 
-    def bundler_config_path
+    def bundler_local_config_path
+      app_config = ENV["BUNDLE_APP_CONFIG"]
+      dir = if app_config && !app_config.empty?
+        app_config
+      else
+        File.join(Dir.pwd, ".bundle")
+      end
+      File.join(dir, "config")
+    end
+
+    def bundler_global_config_path
       File.join(Dir.home, ".bundle", "config")
     end
 
@@ -149,5 +168,49 @@ module Scint
       key.upcase!
       "BUNDLE_#{key}"
     end
+
+    def self.key_for_uri_string(uri_string)
+      key = uri_string.to_s.dup
+      key.gsub!(".", "__")
+      key.gsub!("-", "___")
+      key.upcase!
+      "BUNDLE_#{key}"
+    end
+
+    def self.key_for_source_uri(uri)
+      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+      keys_for_uri_lookup(uri).find { |key| key != key_for_host(uri.host) }
+    rescue StandardError
+      nil
+    end
+
+    def self.keys_for_uri_lookup(uri)
+      uri = URI.parse(uri.to_s) unless uri.is_a?(URI)
+      keys = []
+
+      normalized = "#{uri.scheme || 'https'}://#{uri.host}"
+      default_port = (uri.scheme == "http" ? 80 : 443)
+      normalized += ":#{uri.port}" if uri.port && uri.port != default_port
+
+      path = uri.path.to_s
+      path = "/" if path.empty?
+      path = "/#{path}" unless path.start_with?("/")
+      path += "/" unless path.end_with?("/")
+
+      segments = path.split("/").reject(&:empty?)
+      candidate_paths = ["/"]
+      unless segments.empty?
+        1.upto(segments.length) do |i|
+          candidate_paths << "/#{segments.first(i).join('/')}/"
+        end
+      end
+
+      candidate_paths.reverse_each do |candidate_path|
+        keys << key_for_uri_string("#{normalized}#{candidate_path}")
+      end
+
+      keys << key_for_host(uri.host) if uri.host
+      keys.compact.uniq
+    end
   end
 end

data/lib/scint/debug/io_trace.rb

@@ -110,7 +110,9 @@ module Scint
 
           next unless container.method_defined?(original_name) || container.private_method_defined?(original_name)
 
-          container.send(:alias_method, method_name, original_name)
+          with_silenced_warnings do
+            container.send(:alias_method, method_name, original_name)
+          end
           container.send(:remove_method, original_name)
 
           visibility = entry[:visibility]
@@ -146,12 +148,21 @@ module Scint
         visibility = method_visibility(container, method_name)
 
         container.send(:alias_method, original_name, method_name)
-        container.send(:define_method, method_name) do |*args, **kwargs, &block|
-          Scint::Debug::IOTrace.log(op_name, args: args, kwargs: kwargs)
-          if kwargs.empty?
-            send(original_name, *args, &block)
-          else
-            send(original_name, *args, **kwargs, &block)
+        if container.instance_methods(false).include?(method_name) ||
+           container.private_instance_methods(false).include?(method_name) ||
+           container.protected_instance_methods(false).include?(method_name)
+          container.send(:remove_method, method_name)
+        elsif container.method_defined?(method_name) || container.private_method_defined?(method_name) || container.protected_method_defined?(method_name)
+          container.send(:undef_method, method_name)
+        end
+        with_silenced_warnings do
+          container.send(:define_method, method_name) do |*args, **kwargs, &block|
+            Scint::Debug::IOTrace.log(op_name, args: args, kwargs: kwargs)
+            if kwargs.empty?
+              send(original_name, *args, &block)
+            else
+              send(original_name, *args, **kwargs, &block)
+            end
           end
         end
 
@@ -173,6 +184,14 @@ module Scint
         nil
       end
 
+      def with_silenced_warnings
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        yield
+      ensure
+        $VERBOSE = verbose
+      end
+
       def disable_unlocked
         return unless @enabled || @log_io
 

data/lib/scint/downloader/fetcher.rb

@@ -54,7 +54,13 @@ module Scint
               current_uri = URI.parse(location)
               next
             else
-              raise NetworkError, "HTTP #{response.code} for #{uri}: #{response.message}"
+              raise NetworkError.new(
+                http_error_message(current_uri, response),
+                uri: current_uri.to_s,
+                http_status: response.code.to_i,
+                response_headers: response_headers_hash(response),
+                response_body: response.body.to_s,
+              )
             end
           end
 
@@ -108,6 +114,24 @@ module Scint
           http
         end
       end
+
+      def http_error_message(uri, response)
+        message = "HTTP #{response.code} for #{uri}: #{response.message}"
+        body = response.body.to_s
+        return message if body.empty?
+
+        excerpt = body.gsub(/\s+/, " ").strip
+        return message if excerpt.empty?
+
+        excerpt = "#{excerpt[0, 277]}..." if excerpt.length > 280
+        "#{message} -- #{excerpt}"
+      end
+
+      def response_headers_hash(response)
+        headers = {}
+        response.each_header { |k, v| headers[k] = v }
+        headers
+      end
     end
   end
 end

data/lib/scint/downloader/pool.rb

@@ -4,41 +4,59 @@ require_relative "fetcher"
 require_relative "../worker_pool"
 require_relative "../platform"
 require_relative "../errors"
+require "uri"
 
 module Scint
   module Downloader
     class Pool
       MAX_RETRIES = 3
       BACKOFF_BASE = 0.5 # seconds
+      DEFAULT_PER_HOST_LIMIT = 4
 
       attr_reader :size
 
-      def initialize(size: nil, on_progress: nil, credentials: nil)
+      def initialize(size: nil, on_progress: nil, credentials: nil, per_host_limit: DEFAULT_PER_HOST_LIMIT)
         @size = size || [Platform.cpu_count * 2, 50].min
         @on_progress = on_progress
         @credentials = credentials
+        @per_host_limit = [per_host_limit.to_i, 1].max
         @fetchers = {} # thread_id => Fetcher
         @fetcher_mutex = Thread::Mutex.new
+        @host_slots = Hash.new(0)
+        @host_waiters = {}
+        @host_mutex = Thread::Mutex.new
       end
 
       # Download a single URI to dest_path with retry logic.
       # Returns { path:, size: }
       def download(uri, dest_path, checksum: nil)
-        retries = 0
-        begin
-          fetcher = thread_fetcher
-          fetcher.fetch(uri, dest_path, checksum: checksum)
-        rescue NetworkError, Errno::ECONNRESET, Errno::ECONNREFUSED,
-               Errno::ETIMEDOUT, Net::ReadTimeout, Net::OpenTimeout,
-               SocketError, IOError => e
-          retries += 1
-          if retries <= MAX_RETRIES
-            sleep(BACKOFF_BASE * (2**(retries - 1)))
-            # Reset connection on retry
-            reset_thread_fetcher
-            retry
+        with_host_slot(uri) do
+          retries = 0
+          begin
+            fetcher = thread_fetcher
+            fetcher.fetch(uri, dest_path, checksum: checksum)
+          rescue NetworkError, Errno::ECONNRESET, Errno::ECONNREFUSED,
+                 Errno::ETIMEDOUT, Net::ReadTimeout, Net::OpenTimeout,
+                 SocketError, IOError => e
+            retries += 1
+            if retries <= MAX_RETRIES
+              sleep(BACKOFF_BASE * (2**(retries - 1)))
+              # Reset connection on retry
+              reset_thread_fetcher
+              retry
+            end
+            if e.is_a?(NetworkError)
+              raise NetworkError.new(
+                "Failed to download #{uri} after #{MAX_RETRIES} retries: #{e.message}",
+                uri: (e.uri || uri.to_s),
+                http_status: e.http_status,
+                response_headers: e.response_headers,
+                response_body: e.response_body,
+              )
+            end
+
+            raise NetworkError, "Failed to download #{uri} after #{MAX_RETRIES} retries: #{e.message}"
           end
-          raise NetworkError, "Failed to download #{uri} after #{MAX_RETRIES} retries: #{e.message}"
         end
       end
 
@@ -107,6 +125,40 @@ module Scint
           old&.close
         end
       end
+
+      def with_host_slot(uri)
+        key = host_slot_key(uri)
+        return yield unless key
+
+        waiter = nil
+        @host_mutex.synchronize do
+          waiter = (@host_waiters[key] ||= Thread::ConditionVariable.new)
+          while @host_slots[key] >= @per_host_limit
+            waiter.wait(@host_mutex)
+          end
+          @host_slots[key] += 1
+        end
+
+        begin
+          yield
+        ensure
+          @host_mutex.synchronize do
+            @host_slots[key] -= 1 if @host_slots[key] > 0
+            waiter.broadcast
+          end
+        end
+      end
+
+      def host_slot_key(uri)
+        parsed = uri.is_a?(URI) ? uri : URI.parse(uri.to_s)
+        return nil unless parsed.host
+
+        scheme = parsed.scheme || "https"
+        port = parsed.port || (scheme == "https" ? 443 : 80)
+        "#{scheme}://#{parsed.host}:#{port}"
+      rescue URI::InvalidURIError
+        nil
+      end
     end
   end
 end

data/lib/scint/errors.rb

@@ -26,6 +26,16 @@ module Scint
   end
 
   class NetworkError < BundlerError
+    attr_reader :uri, :http_status, :response_headers, :response_body
+
+    def initialize(message = nil, uri: nil, http_status: nil, response_headers: nil, response_body: nil)
+      super(message)
+      @uri = uri
+      @http_status = http_status
+      @response_headers = response_headers
+      @response_body = response_body
+    end
+
     def status_code
       7
     end

data/lib/scint/fs.rb

@@ -21,28 +21,75 @@ module Scint
       @mkdir_mutex.synchronize { @mkdir_cache[path] = true }
     end
 
+    # Detect best file copy strategy once per process.
+    # Returns :reflink, :hardlink, or :copy.
+    @copy_strategy = nil
+    @copy_strategy_mutex = Thread::Mutex.new
+
+    def detect_copy_strategy(src_dir, dst_dir)
+      @copy_strategy_mutex.synchronize do
+        return @copy_strategy if @copy_strategy
+
+        @copy_strategy = _probe_copy_strategy(src_dir, dst_dir)
+      end
+    end
+
+    def _probe_copy_strategy(src_dir, dst_dir)
+      # Create a temp file in src to test against dst
+      probe_src = File.join(src_dir, ".scint_probe_#{$$}")
+      probe_dst = File.join(dst_dir, ".scint_probe_#{$$}")
+      begin
+        File.write(probe_src, "x")
+        mkdir_p(dst_dir)
+
+        if Platform.macos?
+          if system("cp", "-c", probe_src, probe_dst, [:out, :err] => File::NULL)
+            return :reflink
+          end
+          File.delete(probe_dst) if File.exist?(probe_dst)
+        elsif Platform.linux?
+          if system("cp", "--reflink=always", probe_src, probe_dst, [:out, :err] => File::NULL)
+            return :reflink
+          end
+          File.delete(probe_dst) if File.exist?(probe_dst)
+        end
+
+        begin
+          File.link(probe_src, probe_dst)
+          return :hardlink
+        rescue SystemCallError
+          File.delete(probe_dst) if File.exist?(probe_dst)
+        end
+
+        :copy
+      ensure
+        File.delete(probe_src) if File.exist?(probe_src)
+        File.delete(probe_dst) if File.exist?(probe_dst)
+      end
+    rescue StandardError
+      :copy
+    end
+
     # APFS clonefile (CoW copy). Falls back to hardlink, then regular copy.
     def clonefile(src, dst)
       src = src.to_s
       dst = dst.to_s
       mkdir_p(File.dirname(dst))
 
-      # Try APFS clonefile via cp -c (macOS)
-      if Platform.macos?
-        return if system("cp", "-c", src, dst, [:out, :err] => File::NULL)
-      end
-
-      # Try Linux reflink copy-on-write where supported (btrfs/xfs/etc).
-      if Platform.linux?
-        return if system("cp", "--reflink=always", src, dst, [:out, :err] => File::NULL)
-      end
-
-      # Fallback: hardlink
-      begin
-        File.link(src, dst)
-        return
-      rescue SystemCallError
-        # cross-device or unsupported
+      case detect_copy_strategy(File.dirname(src), File.dirname(dst))
+      when :reflink
+        if Platform.macos?
+          return if system("cp", "-c", src, dst, [:out, :err] => File::NULL)
+        elsif Platform.linux?
+          return if system("cp", "--reflink=always", src, dst, [:out, :err] => File::NULL)
+        end
+      when :hardlink
+        begin
+          File.link(src, dst)
+          return
+        rescue SystemCallError
+          # fall through to copy
+        end
       end
 
       # Final fallback: regular copy
@@ -58,26 +105,164 @@ module Scint
       raise Errno::ENOENT, src_dir unless Dir.exist?(src_dir)
       mkdir_p(dst_dir)
 
-      # Fast path on macOS/APFS: copy-on-write clone of full tree.
-      if Platform.macos?
-        src_contents = File.join(src_dir, ".")
-        return if system("cp", "-cR", src_contents, dst_dir, [:out, :err] => File::NULL)
-      end
+      strategy = detect_copy_strategy(src_dir, dst_dir)
 
-      # Fast path on Linux filesystems with reflink support.
-      if Platform.linux?
+      if strategy == :reflink
         src_contents = File.join(src_dir, ".")
-        return if system("cp", "--reflink=always", "-R", src_contents, dst_dir, [:out, :err] => File::NULL)
+        if Platform.macos?
+          return if system("cp", "-cR", src_contents, dst_dir, [:out, :err] => File::NULL)
+        elsif Platform.linux?
+          return if system("cp", "--reflink=always", "-R", src_contents, dst_dir, [:out, :err] => File::NULL)
+        end
       end
 
       hardlink_tree(src_dir, dst_dir)
     end
 
+    # Materialize a tree using a manifest to avoid directory scans.
+    # Manifest entries must be hashes with "path" and "type" keys.
+    # Uses the fastest available file copy strategy (reflink > hardlink > copy).
+    def materialize_from_manifest(src_dir, dst_dir, entries)
+      src_dir = src_dir.to_s
+      dst_dir = dst_dir.to_s
+      entries = Array(entries)
+      raise Errno::ENOENT, src_dir unless Dir.exist?(src_dir)
+      mkdir_p(dst_dir)
+
+      strategy = detect_copy_strategy(src_dir, dst_dir)
+
+      entries.each do |entry|
+        rel = entry["path"].to_s
+        next if rel.empty? || rel.start_with?("/") || rel.include?("..")
+
+        src_path = File.join(src_dir, rel)
+        dst_path = File.join(dst_dir, rel)
+
+        case entry["type"]
+        when "dir"
+          mkdir_p(dst_path)
+        when "symlink"
+          mkdir_p(File.dirname(dst_path))
+          next if File.exist?(dst_path) || File.symlink?(dst_path)
+
+          target = File.readlink(src_path)
+          begin
+            File.symlink(target, dst_path)
+          rescue Errno::EEXIST
+            next
+          end
+        else
+          mkdir_p(File.dirname(dst_path))
+          next if File.exist?(dst_path)
+
+          begin
+            _link_or_copy(src_path, dst_path, strategy)
+          rescue Errno::EEXIST
+            next
+          rescue SystemCallError
+            next if File.exist?(dst_path)
+            raise
+          end
+        end
+      end
+    end
+
+    # Fast file link/copy using a pre-detected strategy (no per-file probing).
+    def _link_or_copy(src, dst, strategy)
+      case strategy
+      when :reflink
+        if Platform.macos?
+          return if system("cp", "-c", src, dst, [:out, :err] => File::NULL)
+        elsif Platform.linux?
+          return if system("cp", "--reflink=always", src, dst, [:out, :err] => File::NULL)
+        end
+        # reflink failed for this file, try hardlink
+        begin
+          File.link(src, dst)
+          return
+        rescue SystemCallError; end
+        FileUtils.cp(src, dst)
+      when :hardlink
+        begin
+          File.link(src, dst)
+          return
+        rescue SystemCallError; end
+        FileUtils.cp(src, dst)
+      else
+        FileUtils.cp(src, dst)
+      end
+    end
+
+    LINKER_SCRIPT = File.expand_path("linker.sh", __dir__).freeze
+
+    # Bulk-link cached gem directories into dst_parent.
+    # Opens one helper process (linker.sh) and writes gem basenames to its
+    # stdin. The helper probes the fastest FS strategy once then applies it
+    # to every gem.
+    def bulk_link_gems(src_parent, dst_parent, gem_names)
+      src_parent = src_parent.to_s
+      dst_parent = dst_parent.to_s
+      gem_names = Array(gem_names)
+      return 0 if gem_names.empty?
+
+      mkdir_p(dst_parent)
+
+      IO.popen(["/bin/bash", LINKER_SCRIPT], "w") do |io|
+        io.puts src_parent
+        io.puts dst_parent
+        gem_names.each { |name| io.puts name }
+      end
+
+      gem_names.size
+    end
+
+    # Clone many source directories into one destination parent directory.
+    # This is significantly faster than one process per gem on large warm
+    # installs because it batches cp invocations while preserving CoW/reflink.
+    # Returns the number of source trees requested.
+    def clone_many_trees(src_dirs, dst_parent, chunk_size: 64)
+      dst_parent = dst_parent.to_s
+      mkdir_p(dst_parent)
+
+      sources = Array(src_dirs).map(&:to_s).uniq
+      sources.select! { |src| Dir.exist?(src) }
+      return 0 if sources.empty?
+
+      copied = 0
+      strategy = sources.first ? detect_copy_strategy(sources.first, dst_parent) : :copy
+
+      sources.each_slice([chunk_size.to_i, 1].max) do |slice|
+        pending = slice.reject do |src|
+          Dir.exist?(File.join(dst_parent, File.basename(src)))
+        end
+        next if pending.empty?
+
+        ok = false
+        if strategy == :reflink
+          if Platform.macos?
+            ok = system("cp", "-cR", *pending, dst_parent, [:out, :err] => File::NULL)
+          elsif Platform.linux?
+            ok = system("cp", "--reflink=always", "-R", *pending, dst_parent, [:out, :err] => File::NULL)
+          end
+        end
+
+        unless ok
+          pending.each do |src|
+            clone_tree(src, File.join(dst_parent, File.basename(src)))
+          end
+        end
+
+        copied += pending.length
+      end
+
+      copied
+    end
+
     # Recursively hardlink all files from src_dir into dst_dir.
     # Directory structure is recreated; files are hardlinked.
     def hardlink_tree(src_dir, dst_dir)
-      src_dir = src_dir.to_s
-      dst_dir = dst_dir.to_s
+      src_dir = File.expand_path(src_dir.to_s)
+      dst_dir = File.expand_path(dst_dir.to_s)
       raise Errno::ENOENT, src_dir unless Dir.exist?(src_dir)
       mkdir_p(dst_dir)
 
@@ -88,6 +273,10 @@ module Scint
         Dir.each_child(src_root) do |entry|
           src_path = File.join(src_root, entry)
           dst_path = File.join(dst_root, entry)
+
+          # Guard against recursive copy when destination is nested under source.
+          next if dst_dir == src_path || dst_dir.start_with?("#{src_path}/")
+
           stat = File.lstat(src_path)
 
           if stat.directory?

data/lib/scint/gem/package.rb

@@ -17,7 +17,9 @@ module Scint
           tar.each do |entry|
             if entry.full_name == "metadata.gz"
               gz = Zlib::GzipReader.new(StringIO.new(entry.read))
-              return ::Gem::Specification.from_yaml(gz.read)
+              yaml = gz.read
+              yaml.force_encoding("UTF-8") if yaml.encoding == Encoding::US_ASCII
+              return ::Gem::Specification.from_yaml(yaml)
             end
           end
         end
@@ -38,7 +40,9 @@ module Scint
             case entry.full_name
             when "metadata.gz"
               gz = Zlib::GzipReader.new(StringIO.new(entry.read))
-              gemspec = ::Gem::Specification.from_yaml(gz.read)
+              yaml = gz.read
+              yaml.force_encoding("UTF-8") if yaml.encoding == Encoding::US_ASCII
+              gemspec = ::Gem::Specification.from_yaml(yaml)
             when "data.tar.gz"
               # Write data.tar.gz to a temp file for extraction
               tmp = File.join(dest_dir, ".data.tar.gz.tmp")