scint 0.7.0 → 0.7.1

data/lib/scint/installer/preparer.rb

@@ -4,9 +4,16 @@ require_relative "../downloader/pool"
 require_relative "../gem/package"
 require_relative "../gem/extractor"
 require_relative "../cache/layout"
+require_relative "../cache/manifest"
+require_relative "../cache/validity"
 require_relative "../fs"
 require_relative "../errors"
 require_relative "../spec_utils"
+require_relative "../platform"
+require_relative "./promoter"
+require_relative "./extension_builder"
+require_relative "../source/git"
+require_relative "../source/path"
 
 module Scint
   module Installer
@@ -45,21 +52,30 @@ module Scint
         already_cached = []
 
         sorted.each do |entry|
-          inbound = @layout.inbound_path(entry.spec)
-          extracted = @layout.extracted_path(entry.spec)
+          spec = entry.spec
+          inbound = @layout.inbound_path(spec)
+          assembling = @layout.assembling_path(spec)
+          cached = @layout.cached_path(spec)
 
-          if File.directory?(extracted)
-            # Already extracted -- load gemspec from cache or re-read
-            gemspec = load_cached_spec(entry.spec) || read_gemspec_from_extracted(extracted, entry.spec)
+          if Cache::Validity.cached_valid?(spec, @layout)
+            gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
             already_cached << PreparedGem.new(
-              spec: entry.spec,
-              extracted_path: extracted,
+              spec: spec,
+              extracted_path: cached,
               gemspec: gemspec,
               from_cache: true,
             )
+          elsif File.directory?(assembling)
+            gemspec = read_gemspec_from_extracted(assembling, spec)
+            already_cached << PreparedGem.new(
+              spec: spec,
+              extracted_path: assembling,
+              gemspec: gemspec,
+              from_cache: false,
+            )
           elsif File.exist?(inbound)
             # Downloaded but not extracted
-            already_cached << extract_gem(entry.spec, inbound)
+            already_cached << extract_gem(spec, inbound)
           else
             # Need to download
             to_download << entry
@@ -105,65 +121,98 @@ module Scint
 
       # Prepare a single entry (for use with scheduler).
       def prepare_one(entry)
-        inbound = @layout.inbound_path(entry.spec)
-        extracted = @layout.extracted_path(entry.spec)
+        spec = entry.spec
+        inbound = @layout.inbound_path(spec)
+        assembling = @layout.assembling_path(spec)
+        cached = @layout.cached_path(spec)
 
-        if File.directory?(extracted)
-          gemspec = load_cached_spec(entry.spec) || read_gemspec_from_extracted(extracted, entry.spec)
+        if Cache::Validity.cached_valid?(spec, @layout)
+          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
           return PreparedGem.new(
-            spec: entry.spec,
-            extracted_path: extracted,
+            spec: spec,
+            extracted_path: cached,
             gemspec: gemspec,
             from_cache: true,
           )
         end
 
+        if File.directory?(assembling)
+          gemspec = read_gemspec_from_extracted(assembling, spec)
+          return PreparedGem.new(
+            spec: spec,
+            extracted_path: assembling,
+            gemspec: gemspec,
+            from_cache: false,
+          )
+        end
+
+
         unless File.exist?(inbound)
           uri = gem_download_uri(entry)
           @download_pool.download(uri, inbound)
         end
 
-        extract_gem(entry.spec, inbound)
+        extract_gem(spec, inbound)
       end
 
       private
 
       def extract_gem(spec, gem_path)
-        dest = @layout.extracted_path(spec)
+        cached = @layout.cached_path(spec)
+        assembling = @layout.assembling_path(spec)
+
+        if Cache::Validity.cached_valid?(spec, @layout)
+          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
+          return PreparedGem.new(spec: spec, extracted_path: cached, gemspec: gemspec, from_cache: true)
+        end
 
-        if File.directory?(dest)
-          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(dest, spec)
-          return PreparedGem.new(spec: spec, extracted_path: dest, gemspec: gemspec, from_cache: true)
+        if File.directory?(assembling)
+          gemspec = read_gemspec_from_extracted(assembling, spec)
+          return PreparedGem.new(spec: spec, extracted_path: assembling, gemspec: gemspec, from_cache: false)
         end
 
-        # Extract to temp dir, then atomic move
-        tmp_dest = "#{dest}.#{Process.pid}.tmp"
+        # Extract to temp dir in assembling, then atomic move
+        tmp_dest = "#{assembling}.#{Process.pid}.tmp"
         FileUtils.rm_rf(tmp_dest) if File.exist?(tmp_dest)
 
         result = @package.extract(gem_path, tmp_dest)
-        FS.atomic_move(tmp_dest, dest)
-
-        # Cache the gemspec as Marshal for fast future loads
-        cache_spec(spec, result[:gemspec])
+        FS.atomic_move(tmp_dest, assembling)
 
-        PreparedGem.new(
-          spec: spec,
-          extracted_path: dest,
-          gemspec: result[:gemspec],
-          from_cache: false,
-        )
+        if ExtensionBuilder.needs_build?(spec, assembling)
+          PreparedGem.new(
+            spec: spec,
+            extracted_path: assembling,
+            gemspec: result[:gemspec],
+            from_cache: false,
+          )
+        else
+          promote_assembled(spec, assembling, result[:gemspec])
+          PreparedGem.new(
+            spec: spec,
+            extracted_path: @layout.cached_path(spec),
+            gemspec: result[:gemspec],
+            from_cache: false,
+          )
+        end
       end
 
       def load_cached_spec(spec)
-        path = @layout.spec_cache_path(spec)
+        path = @layout.cached_spec_path(spec)
         return nil unless File.exist?(path)
-        Marshal.load(File.binread(path))
-      rescue ArgumentError, TypeError, EOFError
+
+        data = File.binread(path)
+        if data.start_with?("---")
+          data.force_encoding("UTF-8") if data.encoding != Encoding::UTF_8
+          return Gem::Specification.from_yaml(data)
+        end
+
+        Marshal.load(data)
+      rescue ArgumentError, TypeError, EOFError, StandardError
         nil
       end
 
       def cache_spec(spec, gemspec)
-        path = @layout.spec_cache_path(spec)
+        path = @layout.cached_spec_path(spec)
         FS.atomic_write(path, Marshal.dump(gemspec))
       rescue StandardError
         # Non-fatal: cache miss on next load
@@ -173,10 +222,91 @@ module Scint
         pattern = File.join(extracted_dir, "*.gemspec")
         candidates = Dir.glob(pattern)
         if candidates.any?
+          version = spec.respond_to?(:version) ? spec.version.to_s : nil
+          old_version = ENV["VERSION"]
           begin
-            ::Gem::Specification.load(candidates.first)
-          rescue StandardError
+            ENV["VERSION"] = version if version && !ENV["VERSION"]
+            SpecUtils.load_gemspec(candidates.first, isolate: true)
+          rescue SystemExit, StandardError
             nil
+          ensure
+            ENV["VERSION"] = old_version
+          end
+        end
+      end
+
+      def promote_assembled(spec, assembling_path, gemspec)
+        return unless assembling_path && Dir.exist?(assembling_path)
+
+        cached_dir = @layout.cached_path(spec)
+        promoter = Promoter.new(root: @layout.root)
+        lock_key = "#{Platform.abi_key}-#{@layout.full_name(spec)}"
+
+        promoter.validate_within_root!(@layout.root, assembling_path, label: "assembling")
+        promoter.validate_within_root!(@layout.root, cached_dir, label: "cached")
+
+        result = nil
+        promoter.with_staging_dir(prefix: "cached") do |staging|
+          FS.clone_tree(assembling_path, staging)
+          manifest = build_cached_manifest(spec, staging)
+          spec_payload = gemspec ? Marshal.dump(gemspec) : nil
+          result = promoter.promote_tree(
+            staging_path: staging,
+            target_path: cached_dir,
+            lock_key: lock_key,
+          )
+          if result == :promoted
+            write_cached_metadata(spec, spec_payload, manifest)
+          end
+        end
+        FileUtils.rm_rf(assembling_path) if Dir.exist?(assembling_path)
+        result
+      rescue StandardError
+        FileUtils.rm_rf(cached_dir) if Dir.exist?(cached_dir)
+        raise
+      end
+
+      def write_cached_metadata(spec, spec_payload, manifest)
+        spec_path = @layout.cached_spec_path(spec)
+        manifest_path = @layout.cached_manifest_path(spec)
+        FS.mkdir_p(File.dirname(spec_path))
+
+        FS.atomic_write(spec_path, spec_payload) if spec_payload
+        Cache::Manifest.write(manifest_path, manifest)
+      end
+
+      def build_cached_manifest(spec, cached_dir)
+        Cache::Manifest.build(
+          spec: spec,
+          gem_dir: cached_dir,
+          abi_key: Platform.abi_key,
+          source: manifest_source_for(spec),
+          extensions: ExtensionBuilder.needs_build?(spec, cached_dir),
+        )
+      end
+
+      def manifest_source_for(spec)
+        source = spec.source
+        if source.is_a?(Source::Git)
+          {
+            "type" => "git",
+            "uri" => source.uri.to_s,
+            "revision" => source.revision || source.ref || source.branch || source.tag,
+          }.compact
+        elsif source.is_a?(Source::Path)
+          {
+            "type" => "path",
+            "path" => File.expand_path(source.path.to_s),
+            "uri" => source.path.to_s,
+          }
+        else
+          source_str = source.to_s
+          if source_str.start_with?("http://", "https://")
+            { "type" => "rubygems", "uri" => source_str }
+          elsif source_str.start_with?("/", ".", "~")
+            { "type" => "path", "path" => File.expand_path(source_str), "uri" => source_str }
+          else
+            { "type" => "rubygems", "uri" => source_str }
           end
         end
       end

data/lib/scint/installer/promoter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "../fs"
+require "fileutils"
+require "securerandom"
+
+module Scint
+  module Installer
+    class Promoter
+      attr_reader :root, :lock_dir, :staging_dir
+
+      def initialize(root:, lock_dir: nil, staging_dir: nil)
+        @root = File.expand_path(root.to_s)
+        @lock_dir = File.expand_path(lock_dir.to_s) if lock_dir
+        @staging_dir = File.expand_path(staging_dir.to_s) if staging_dir
+
+        @lock_dir ||= File.join(@root, "locks", "promotion")
+        @staging_dir ||= File.join(@root, "staging")
+      end
+
+      def with_lock(lock_key)
+        lock_path = lock_path_for(lock_key)
+        validate_within_root!(@root, lock_path, label: "lock")
+        FS.mkdir_p(File.dirname(lock_path))
+
+        File.open(lock_path, "w") do |file|
+          file.flock(File::LOCK_EX)
+          yield
+        ensure
+          file.flock(File::LOCK_UN) rescue nil
+        end
+      end
+
+      def with_staging_dir(prefix:)
+        FS.mkdir_p(@staging_dir)
+        staging_path = File.join(@staging_dir, staging_suffix(prefix))
+        validate_within_root!(@root, staging_path, label: "staging")
+        FileUtils.rm_rf(staging_path) if File.exist?(staging_path)
+
+        begin
+          FS.mkdir_p(staging_path)
+          yield staging_path
+        ensure
+          FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+        end
+      end
+
+      def promote_tree(staging_path:, target_path:, lock_key:)
+        validate_within_root!(@root, staging_path, label: "staging")
+        validate_within_root!(@root, target_path, label: "target")
+        raise CacheError, "Staging path does not exist: #{staging_path}" unless Dir.exist?(staging_path)
+
+        with_lock(lock_key) do
+          if Dir.exist?(target_path)
+            FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+            return :exists
+          end
+
+          begin
+            FS.atomic_move(staging_path, target_path)
+          rescue StandardError
+            FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+            raise
+          end
+
+          :promoted
+        end
+      end
+
+      def validate_within_root!(root_path, candidate_path, label: "path")
+        root_expanded = File.expand_path(root_path.to_s)
+        candidate_expanded = File.expand_path(candidate_path.to_s)
+        within_root = candidate_expanded == root_expanded ||
+                      candidate_expanded.start_with?("#{root_expanded}/")
+        return if within_root
+
+        raise CacheError, "#{label.capitalize} escapes cache root: #{candidate_path}"
+      end
+
+      def lock_path_for(lock_key)
+        safe_key = sanitize_key(lock_key)
+        File.join(@lock_dir, "#{safe_key}.lock")
+      end
+
+      def sanitize_key(key)
+        key.to_s.gsub(/[^0-9A-Za-z._-]/, "_")
+      end
+
+      def staging_suffix(prefix)
+        safe = sanitize_key(prefix)
+        token = SecureRandom.hex(6)
+        "#{safe}.#{Process.pid}.#{Thread.current.object_id}.#{token}"
+      end
+    end
+  end
+end

data/lib/scint/linker.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+# scint-linker — bulk-hardlink cached gems into a bundle.
+#
+# Protocol (stdin):
+#   Line 1: source parent dir  (e.g. ~/.cache/scint/cached/ruby-3.4.7-x86_64-linux)
+#   Line 2: dest parent dir    (e.g. .bundle/ruby/3.4.0/gems)
+#   Remaining lines: gem directory basenames (e.g. "rack-3.2.4")
+#
+# The script probes the best copy strategy once (cpio, hardlink, reflink,
+# copy) and uses it for every gem. When cpio is available it reads
+# .scint-files from each cached gem so only listed files are materialized.
+# Otherwise falls back to cp -al / cp --reflink / cp -R.
+
+set -euo pipefail
+
+read -r SRC
+read -r DST
+mkdir -p "$DST"
+
+# ── detect strategy ──────────────────────────────────────────────
+STRATEGY=""
+
+# Probe needs a real file to test against. Find one quickly.
+probe_src=""
+for candidate in "$SRC"/*/.scint-files; do
+  [ -f "$candidate" ] && probe_src="$candidate" && break
+done
+
+if [ -z "$probe_src" ]; then
+  # No .scint-files at all — fall back to cp -al or cp -R
+  if cp -al --version >/dev/null 2>&1; then
+    STRATEGY="cp-al"
+  else
+    STRATEGY="cp"
+  fi
+else
+  probe_dst="$DST/.scint-probe-$$"
+
+  # 1. cpio -pld (file-list driven hardlinks — ideal)
+  if command -v cpio >/dev/null 2>&1; then
+    if echo "$probe_src" | cpio -pld "$DST" >/dev/null 2>&1; then
+      rm -f "$probe_dst" 2>/dev/null
+      STRATEGY="cpio"
+    fi
+  fi
+
+  # 2. hardlink via ln
+  if [ -z "$STRATEGY" ]; then
+    if ln "$probe_src" "$probe_dst" 2>/dev/null; then
+      rm -f "$probe_dst"
+      STRATEGY="cp-al"
+    fi
+  fi
+
+  # 3. reflink (btrfs/xfs/APFS)
+  if [ -z "$STRATEGY" ]; then
+    if [ "$(uname)" = "Darwin" ]; then
+      if cp -c "$probe_src" "$probe_dst" 2>/dev/null; then
+        rm -f "$probe_dst"
+        STRATEGY="reflink"
+      fi
+    else
+      if cp --reflink=always "$probe_src" "$probe_dst" 2>/dev/null; then
+        rm -f "$probe_dst"
+        STRATEGY="reflink"
+      fi
+    fi
+  fi
+
+  rm -f "$probe_dst" 2>/dev/null
+  [ -z "$STRATEGY" ] && STRATEGY="cp"
+fi
+
+# ── link gems ────────────────────────────────────────────────────
+
+link_cpio() {
+  # Read .scint-files, prefix each line with gem name, pipe to cpio -pld.
+  # One cpio call per gem (cpio needs a single source root).
+  local gem="$1"
+  local dotfiles="$SRC/$gem/.scint-files"
+  if [ -f "$dotfiles" ]; then
+    (cd "$SRC/$gem" && cpio -pld "$DST/$gem" < "$dotfiles" 2>/dev/null)
+  else
+    cp -al "$SRC/$gem" "$DST/$gem"
+  fi
+}
+
+# For cp-al: batch all gems into one call
+BATCH=()
+
+flush_batch() {
+  [ ${#BATCH[@]} -eq 0 ] && return
+  case "$STRATEGY" in
+    cp-al)
+      cp -al "${BATCH[@]}" "$DST/" 2>/dev/null || {
+        # Individual fallback on batch failure
+        for s in "${BATCH[@]}"; do
+          cp -al "$s" "$DST/" 2>/dev/null || cp -R "$s" "$DST/"
+        done
+      }
+      ;;
+    reflink)
+      if [ "$(uname)" = "Darwin" ]; then
+        cp -cR "${BATCH[@]}" "$DST/" 2>/dev/null || {
+          for s in "${BATCH[@]}"; do cp -R "$s" "$DST/"; done
+        }
+      else
+        cp --reflink=always -R "${BATCH[@]}" "$DST/" 2>/dev/null || {
+          for s in "${BATCH[@]}"; do cp -R "$s" "$DST/"; done
+        }
+      fi
+      ;;
+    cp)
+      cp -R "${BATCH[@]}" "$DST/"
+      ;;
+  esac
+  BATCH=()
+}
+
+while IFS= read -r gem; do
+  [ -z "$gem" ] && continue
+  [ -d "$SRC/$gem" ] || continue
+  [ -d "$DST/$gem" ] && continue
+
+  case "$STRATEGY" in
+    cpio)
+      link_cpio "$gem"
+      ;;
+    *)
+      BATCH+=("$SRC/$gem")
+      # Flush in chunks to stay under ARG_MAX
+      [ ${#BATCH[@]} -ge 200 ] && flush_batch
+      ;;
+  esac
+done
+
+flush_batch

data/lib/scint/spec_utils.rb

@@ -1,9 +1,28 @@
 # frozen_string_literal: true
+require "open3"
+require "rbconfig"
 
 module Scint
   module SpecUtils
     module_function
 
+    GEMSPEC_LOAD_MUTEX = Thread::Mutex.new
+
+    GEMSPEC_SUBPROCESS_LOADER = <<~'RUBY'
+      path = ARGV.fetch(0)
+      begin
+        Dir.chdir(File.dirname(path)) do
+          spec = Gem::Specification.load(path)
+          exit(2) unless spec
+          STDOUT.binmode
+          STDOUT.write(Marshal.dump(spec))
+        end
+      rescue Exception => e
+        warn("#{e.class}: #{e.message}")
+        exit(1)
+      end
+    RUBY
+
     def name(spec)
       spec.respond_to?(:name) ? spec.name : spec[:name]
     end
@@ -54,5 +73,61 @@ module Scint
     def full_key_for(name, version, platform = "ruby")
       "#{name}-#{version}-#{platform_value(platform)}"
     end
+
+    # Load a gemspec with path-sensitive fallback.
+    #
+    # Some gemspecs depend on process cwd (e.g. `require "./lib/..."`,
+    # `File.read("README")`). A direct load can fail when evaluated from a
+    # different working directory. For those failures, evaluate in a dedicated
+    # subprocess that can safely chdir without affecting install worker threads.
+    def load_gemspec(path, isolate: false)
+      absolute = File.expand_path(path.to_s)
+      return nil unless File.file?(absolute)
+      return load_gemspec_in_subprocess(absolute) if isolate
+
+      spec = load_gemspec_direct(absolute)
+      return spec if spec
+
+      load_gemspec_in_subprocess(absolute)
+    rescue StandardError, ScriptError => e
+      return load_gemspec_in_subprocess(absolute) if gemspec_path_context_error?(e)
+
+      nil
+    end
+
+    def gemspec_path_context_error?(error)
+      return true if error.is_a?(LoadError)
+      return true if error.is_a?(Errno::ENOENT)
+
+      message = error.message.to_s
+      message.include?("conflicting chdir during another chdir block") ||
+        message.include?("cannot load such file -- ./") ||
+        message.include?("cannot load such file -- ../")
+    end
+    private_class_method :gemspec_path_context_error?
+
+    def load_gemspec_direct(absolute_path)
+      GEMSPEC_LOAD_MUTEX.synchronize do
+        ::Gem::Specification.load(absolute_path)
+      end
+    end
+    private_class_method :load_gemspec_direct
+
+    def load_gemspec_in_subprocess(absolute_path)
+      out, _err, status = Open3.capture3(
+        RbConfig.ruby,
+        "-rrubygems",
+        "-e",
+        GEMSPEC_SUBPROCESS_LOADER,
+        absolute_path,
+      )
+      return nil unless status.success?
+      return nil if out.nil? || out.empty?
+
+      Marshal.load(out)
+    rescue StandardError
+      nil
+    end
+    private_class_method :load_gemspec_in_subprocess
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scint
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Tobi Lutke
@@ -27,8 +27,10 @@ files:
 - lib/bundler/setup.rb
 - lib/scint.rb
 - lib/scint/cache/layout.rb
+- lib/scint/cache/manifest.rb
 - lib/scint/cache/metadata_store.rb
 - lib/scint/cache/prewarm.rb
+- lib/scint/cache/validity.rb
 - lib/scint/cli.rb
 - lib/scint/cli/add.rb
 - lib/scint/cli/cache.rb
@@ -56,6 +58,8 @@ files:
 - lib/scint/installer/linker.rb
 - lib/scint/installer/planner.rb
 - lib/scint/installer/preparer.rb
+- lib/scint/installer/promoter.rb
+- lib/scint/linker.sh
 - lib/scint/lockfile/parser.rb
 - lib/scint/lockfile/writer.rb
 - lib/scint/platform.rb