scint 0.7.0 → 0.8.0

data/lib/scint/installer/planner.rb

@@ -2,6 +2,7 @@
 
 require_relative "extension_builder"
 require_relative "../platform"
+require_relative "../cache/validity"
 
 module Scint
   module Installer
@@ -12,16 +13,16 @@ module Scint
       # Compare resolved specs against what's already installed.
       # Returns an Array of PlanEntry with action set to one of:
       #   :skip      — already installed in bundle_path
-      #   :link      — extracted in global cache, just needs linking
+      #   :link      — cached in global cache, just needs linking
       #   :download  — needs downloading from remote
       #   :build_ext — has native extensions that need compiling
       #
       # Download entries are sorted largest-first so big gems start early,
       # keeping the pipeline saturated while small gems fill in gaps.
-      def plan(resolved_specs, bundle_path, cache_layout)
+      def plan(resolved_specs, bundle_path, cache_layout, telemetry: nil)
         ruby_dir = Platform.ruby_install_dir(bundle_path)
         entries = resolved_specs.map do |spec|
-          plan_one(spec, ruby_dir, cache_layout)
+          plan_one(spec, ruby_dir, cache_layout, telemetry: telemetry)
         end
 
         # Keep built-ins first, then downloads (big->small), then the rest.
@@ -32,7 +33,7 @@ module Scint
         builtins + downloads + rest
       end
 
-      def plan_one(spec, ruby_dir, cache_layout)
+      def plan_one(spec, ruby_dir, cache_layout, telemetry: nil)
         full = cache_layout.full_name(spec)
         gem_path = File.join(ruby_dir, "gems", full)
         spec_path = File.join(ruby_dir, "specifications", "#{full}.gemspec")
@@ -48,10 +49,12 @@ module Scint
 
         # Already installed? Require both gem files and specification.
         if Dir.exist?(gem_path) && File.exist?(spec_path)
-          if extension_link_missing?(spec, ruby_dir, cache_layout)
-            extracted = cache_layout.extracted_path(spec)
+          cache_source = Cache::Validity.source_path_for(spec, cache_layout, telemetry: telemetry)
+          if extension_link_missing?(spec, ruby_dir, cache_layout, cache_source)
             action = ExtensionBuilder.cached_build_available?(spec, cache_layout) ? :link : :build_ext
-            return PlanEntry.new(spec: spec, action: action, cached_path: extracted, gem_path: gem_path)
+            return PlanEntry.new(spec: spec, action: action, cached_path: cache_source, gem_path: gem_path) if cache_source
+
+            return PlanEntry.new(spec: spec, action: :download, cached_path: nil, gem_path: gem_path)
           end
 
           return PlanEntry.new(spec: spec, action: :skip, cached_path: nil, gem_path: gem_path)
@@ -64,39 +67,32 @@ module Scint
           return PlanEntry.new(spec: spec, action: action, cached_path: local_source, gem_path: gem_path)
         end
 
-        # Extracted in global cache?
-        extracted = cache_layout.extracted_path(spec)
-        if Dir.exist?(extracted)
-          action = needs_ext_build?(spec, cache_layout) ? :build_ext : :link
-          return PlanEntry.new(spec: spec, action: action, cached_path: extracted, gem_path: gem_path)
+        cache_source = Cache::Validity.source_path_for(spec, cache_layout, telemetry: telemetry)
+        if cache_source
+          action = needs_ext_build?(spec, cache_layout, cache_source) ? :build_ext : :link
+          return PlanEntry.new(spec: spec, action: action, cached_path: cache_source, gem_path: gem_path)
         end
 
         # Needs downloading
         PlanEntry.new(spec: spec, action: :download, cached_path: nil, gem_path: gem_path)
       end
 
-      def needs_ext_build?(spec, cache_layout)
-        extracted = cache_layout.extracted_path(spec)
-        return false unless ExtensionBuilder.needs_build?(spec, extracted)
+      def needs_ext_build?(spec, cache_layout, source_dir)
+        return false unless source_dir
+        return false unless ExtensionBuilder.needs_build?(spec, source_dir)
 
         !ExtensionBuilder.cached_build_available?(spec, cache_layout)
       end
 
-      def extension_link_missing?(spec, ruby_dir, cache_layout)
-        extracted = cache_layout.extracted_path(spec)
-        return false unless Dir.exist?(extracted)
-        return false unless ExtensionBuilder.needs_build?(spec, extracted)
+      def extension_link_missing?(spec, ruby_dir, cache_layout, source_dir)
+        return false unless source_dir
+        return false unless ExtensionBuilder.needs_build?(spec, source_dir)
 
         full = cache_layout.full_name(spec)
-        ext_install_dir = File.join(
-          ruby_dir,
-          "extensions",
-          Platform.gem_arch,
-          Platform.extension_api_version,
-          full,
-        )
-
-        !Dir.exist?(ext_install_dir)
+        gem_dir = File.join(ruby_dir, "gems", full)
+        marker = File.join(gem_dir, ExtensionBuilder::BUILD_MARKER)
+
+        !File.exist?(marker)
       end
 
       # Rough size estimate for download ordering.

data/lib/scint/installer/preparer.rb

@@ -4,9 +4,16 @@ require_relative "../downloader/pool"
 require_relative "../gem/package"
 require_relative "../gem/extractor"
 require_relative "../cache/layout"
+require_relative "../cache/manifest"
+require_relative "../cache/validity"
 require_relative "../fs"
 require_relative "../errors"
 require_relative "../spec_utils"
+require_relative "../platform"
+require_relative "./promoter"
+require_relative "./extension_builder"
+require_relative "../source/git"
+require_relative "../source/path"
 
 module Scint
   module Installer
@@ -45,21 +52,30 @@ module Scint
         already_cached = []
 
         sorted.each do |entry|
-          inbound = @layout.inbound_path(entry.spec)
-          extracted = @layout.extracted_path(entry.spec)
+          spec = entry.spec
+          inbound = @layout.inbound_path(spec)
+          assembling = @layout.assembling_path(spec)
+          cached = @layout.cached_path(spec)
 
-          if File.directory?(extracted)
-            # Already extracted -- load gemspec from cache or re-read
-            gemspec = load_cached_spec(entry.spec) || read_gemspec_from_extracted(extracted, entry.spec)
+          if Cache::Validity.cached_valid?(spec, @layout)
+            gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
             already_cached << PreparedGem.new(
-              spec: entry.spec,
-              extracted_path: extracted,
+              spec: spec,
+              extracted_path: cached,
               gemspec: gemspec,
               from_cache: true,
             )
+          elsif File.directory?(assembling)
+            gemspec = read_gemspec_from_extracted(assembling, spec)
+            already_cached << PreparedGem.new(
+              spec: spec,
+              extracted_path: assembling,
+              gemspec: gemspec,
+              from_cache: false,
+            )
           elsif File.exist?(inbound)
             # Downloaded but not extracted
-            already_cached << extract_gem(entry.spec, inbound)
+            already_cached << extract_gem(spec, inbound)
           else
             # Need to download
             to_download << entry
@@ -105,65 +121,98 @@ module Scint
 
       # Prepare a single entry (for use with scheduler).
       def prepare_one(entry)
-        inbound = @layout.inbound_path(entry.spec)
-        extracted = @layout.extracted_path(entry.spec)
+        spec = entry.spec
+        inbound = @layout.inbound_path(spec)
+        assembling = @layout.assembling_path(spec)
+        cached = @layout.cached_path(spec)
 
-        if File.directory?(extracted)
-          gemspec = load_cached_spec(entry.spec) || read_gemspec_from_extracted(extracted, entry.spec)
+        if Cache::Validity.cached_valid?(spec, @layout)
+          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
           return PreparedGem.new(
-            spec: entry.spec,
-            extracted_path: extracted,
+            spec: spec,
+            extracted_path: cached,
             gemspec: gemspec,
             from_cache: true,
           )
         end
 
+        if File.directory?(assembling)
+          gemspec = read_gemspec_from_extracted(assembling, spec)
+          return PreparedGem.new(
+            spec: spec,
+            extracted_path: assembling,
+            gemspec: gemspec,
+            from_cache: false,
+          )
+        end
+
+
         unless File.exist?(inbound)
           uri = gem_download_uri(entry)
           @download_pool.download(uri, inbound)
         end
 
-        extract_gem(entry.spec, inbound)
+        extract_gem(spec, inbound)
       end
 
       private
 
       def extract_gem(spec, gem_path)
-        dest = @layout.extracted_path(spec)
+        cached = @layout.cached_path(spec)
+        assembling = @layout.assembling_path(spec)
+
+        if Cache::Validity.cached_valid?(spec, @layout)
+          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(cached, spec)
+          return PreparedGem.new(spec: spec, extracted_path: cached, gemspec: gemspec, from_cache: true)
+        end
 
-        if File.directory?(dest)
-          gemspec = load_cached_spec(spec) || read_gemspec_from_extracted(dest, spec)
-          return PreparedGem.new(spec: spec, extracted_path: dest, gemspec: gemspec, from_cache: true)
+        if File.directory?(assembling)
+          gemspec = read_gemspec_from_extracted(assembling, spec)
+          return PreparedGem.new(spec: spec, extracted_path: assembling, gemspec: gemspec, from_cache: false)
         end
 
-        # Extract to temp dir, then atomic move
-        tmp_dest = "#{dest}.#{Process.pid}.tmp"
+        # Extract to temp dir in assembling, then atomic move
+        tmp_dest = "#{assembling}.#{Process.pid}.tmp"
         FileUtils.rm_rf(tmp_dest) if File.exist?(tmp_dest)
 
         result = @package.extract(gem_path, tmp_dest)
-        FS.atomic_move(tmp_dest, dest)
-
-        # Cache the gemspec as Marshal for fast future loads
-        cache_spec(spec, result[:gemspec])
+        FS.atomic_move(tmp_dest, assembling)
 
-        PreparedGem.new(
-          spec: spec,
-          extracted_path: dest,
-          gemspec: result[:gemspec],
-          from_cache: false,
-        )
+        if ExtensionBuilder.needs_build?(spec, assembling)
+          PreparedGem.new(
+            spec: spec,
+            extracted_path: assembling,
+            gemspec: result[:gemspec],
+            from_cache: false,
+          )
+        else
+          promote_assembled(spec, assembling, result[:gemspec])
+          PreparedGem.new(
+            spec: spec,
+            extracted_path: @layout.cached_path(spec),
+            gemspec: result[:gemspec],
+            from_cache: false,
+          )
+        end
       end
 
       def load_cached_spec(spec)
-        path = @layout.spec_cache_path(spec)
+        path = @layout.cached_spec_path(spec)
         return nil unless File.exist?(path)
-        Marshal.load(File.binread(path))
-      rescue ArgumentError, TypeError, EOFError
+
+        data = File.binread(path)
+        if data.start_with?("---")
+          data.force_encoding("UTF-8") if data.encoding != Encoding::UTF_8
+          return Gem::Specification.from_yaml(data)
+        end
+
+        Marshal.load(data)
+      rescue ArgumentError, TypeError, EOFError, StandardError
         nil
       end
 
       def cache_spec(spec, gemspec)
-        path = @layout.spec_cache_path(spec)
+        path = @layout.cached_spec_path(spec)
         FS.atomic_write(path, Marshal.dump(gemspec))
       rescue StandardError
         # Non-fatal: cache miss on next load
@@ -173,10 +222,91 @@ module Scint
         pattern = File.join(extracted_dir, "*.gemspec")
         candidates = Dir.glob(pattern)
         if candidates.any?
+          version = spec.respond_to?(:version) ? spec.version.to_s : nil
+          old_version = ENV["VERSION"]
           begin
-            ::Gem::Specification.load(candidates.first)
-          rescue StandardError
+            ENV["VERSION"] = version if version && !ENV["VERSION"]
+            SpecUtils.load_gemspec(candidates.first, isolate: true)
+          rescue SystemExit, StandardError
             nil
+          ensure
+            ENV["VERSION"] = old_version
+          end
+        end
+      end
+
+      def promote_assembled(spec, assembling_path, gemspec)
+        return unless assembling_path && Dir.exist?(assembling_path)
+
+        cached_dir = @layout.cached_path(spec)
+        promoter = Promoter.new(root: @layout.root)
+        lock_key = "#{Platform.abi_key}-#{@layout.full_name(spec)}"
+
+        promoter.validate_within_root!(@layout.root, assembling_path, label: "assembling")
+        promoter.validate_within_root!(@layout.root, cached_dir, label: "cached")
+
+        result = nil
+        promoter.with_staging_dir(prefix: "cached") do |staging|
+          FS.clone_tree(assembling_path, staging)
+          manifest = build_cached_manifest(spec, staging)
+          spec_payload = gemspec ? Marshal.dump(gemspec) : nil
+          result = promoter.promote_tree(
+            staging_path: staging,
+            target_path: cached_dir,
+            lock_key: lock_key,
+          )
+          if result == :promoted
+            write_cached_metadata(spec, spec_payload, manifest)
+          end
+        end
+        FileUtils.rm_rf(assembling_path) if Dir.exist?(assembling_path)
+        result
+      rescue StandardError
+        FileUtils.rm_rf(cached_dir) if Dir.exist?(cached_dir)
+        raise
+      end
+
+      def write_cached_metadata(spec, spec_payload, manifest)
+        spec_path = @layout.cached_spec_path(spec)
+        manifest_path = @layout.cached_manifest_path(spec)
+        FS.mkdir_p(File.dirname(spec_path))
+
+        FS.atomic_write(spec_path, spec_payload) if spec_payload
+        Cache::Manifest.write(manifest_path, manifest)
+      end
+
+      def build_cached_manifest(spec, cached_dir)
+        Cache::Manifest.build(
+          spec: spec,
+          gem_dir: cached_dir,
+          abi_key: Platform.abi_key,
+          source: manifest_source_for(spec),
+          extensions: ExtensionBuilder.needs_build?(spec, cached_dir),
+        )
+      end
+
+      def manifest_source_for(spec)
+        source = spec.source
+        if source.is_a?(Source::Git)
+          {
+            "type" => "git",
+            "uri" => source.uri.to_s,
+            "revision" => source.revision || source.ref || source.branch || source.tag,
+          }.compact
+        elsif source.is_a?(Source::Path)
+          {
+            "type" => "path",
+            "path" => File.expand_path(source.path.to_s),
+            "uri" => source.path.to_s,
+          }
+        else
+          source_str = source.to_s
+          if source_str.start_with?("http://", "https://")
+            { "type" => "rubygems", "uri" => source_str }
+          elsif source_str.start_with?("/", ".", "~")
+            { "type" => "path", "path" => File.expand_path(source_str), "uri" => source_str }
+          else
+            { "type" => "rubygems", "uri" => source_str }
           end
         end
       end

data/lib/scint/installer/promoter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "../fs"
+require "fileutils"
+require "securerandom"
+
+module Scint
+  module Installer
+    class Promoter
+      attr_reader :root, :lock_dir, :staging_dir
+
+      def initialize(root:, lock_dir: nil, staging_dir: nil)
+        @root = File.expand_path(root.to_s)
+        @lock_dir = File.expand_path(lock_dir.to_s) if lock_dir
+        @staging_dir = File.expand_path(staging_dir.to_s) if staging_dir
+
+        @lock_dir ||= File.join(@root, "locks", "promotion")
+        @staging_dir ||= File.join(@root, "staging")
+      end
+
+      def with_lock(lock_key)
+        lock_path = lock_path_for(lock_key)
+        validate_within_root!(@root, lock_path, label: "lock")
+        FS.mkdir_p(File.dirname(lock_path))
+
+        File.open(lock_path, "w") do |file|
+          file.flock(File::LOCK_EX)
+          yield
+        ensure
+          file.flock(File::LOCK_UN) rescue nil
+        end
+      end
+
+      def with_staging_dir(prefix:)
+        FS.mkdir_p(@staging_dir)
+        staging_path = File.join(@staging_dir, staging_suffix(prefix))
+        validate_within_root!(@root, staging_path, label: "staging")
+        FileUtils.rm_rf(staging_path) if File.exist?(staging_path)
+
+        begin
+          FS.mkdir_p(staging_path)
+          yield staging_path
+        ensure
+          FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+        end
+      end
+
+      def promote_tree(staging_path:, target_path:, lock_key:)
+        validate_within_root!(@root, staging_path, label: "staging")
+        validate_within_root!(@root, target_path, label: "target")
+        raise CacheError, "Staging path does not exist: #{staging_path}" unless Dir.exist?(staging_path)
+
+        with_lock(lock_key) do
+          if Dir.exist?(target_path)
+            FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+            return :exists
+          end
+
+          begin
+            FS.atomic_move(staging_path, target_path)
+          rescue StandardError
+            FileUtils.rm_rf(staging_path) if Dir.exist?(staging_path)
+            raise
+          end
+
+          :promoted
+        end
+      end
+
+      def validate_within_root!(root_path, candidate_path, label: "path")
+        root_expanded = File.expand_path(root_path.to_s)
+        candidate_expanded = File.expand_path(candidate_path.to_s)
+        within_root = candidate_expanded == root_expanded ||
+                      candidate_expanded.start_with?("#{root_expanded}/")
+        return if within_root
+
+        raise CacheError, "#{label.capitalize} escapes cache root: #{candidate_path}"
+      end
+
+      def lock_path_for(lock_key)
+        safe_key = sanitize_key(lock_key)
+        File.join(@lock_dir, "#{safe_key}.lock")
+      end
+
+      def sanitize_key(key)
+        key.to_s.gsub(/[^0-9A-Za-z._-]/, "_")
+      end
+
+      def staging_suffix(prefix)
+        safe = sanitize_key(prefix)
+        token = SecureRandom.hex(6)
+        "#{safe}.#{Process.pid}.#{Thread.current.object_id}.#{token}"
+      end
+    end
+  end
+end

data/lib/scint/linker.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+# scint-linker — bulk-hardlink cached gems into a bundle.
+#
+# Protocol (stdin):
+#   Line 1: source parent dir  (e.g. ~/.cache/scint/cached/ruby-3.4.7-x86_64-linux)
+#   Line 2: dest parent dir    (e.g. .bundle/ruby/3.4.0/gems)
+#   Remaining lines: gem directory basenames (e.g. "rack-3.2.4")
+#
+# The script probes the best copy strategy once (cpio, hardlink, reflink,
+# copy) and uses it for every gem. When cpio is available it reads
+# .scint-files from each cached gem so only listed files are materialized.
+# Otherwise falls back to cp -al / cp --reflink / cp -R.
+
+set -euo pipefail
+
+read -r SRC
+read -r DST
+mkdir -p "$DST"
+
+# ── detect strategy ──────────────────────────────────────────────
+STRATEGY=""
+
+# Probe needs a real file to test against. Find one quickly.
+probe_src=""
+for candidate in "$SRC"/*/.scint-files; do
+  [ -f "$candidate" ] && probe_src="$candidate" && break
+done
+
+if [ -z "$probe_src" ]; then
+  # No .scint-files at all — fall back to cp -al or cp -R
+  if cp -al --version >/dev/null 2>&1; then
+    STRATEGY="cp-al"
+  else
+    STRATEGY="cp"
+  fi
+else
+  probe_dst="$DST/.scint-probe-$$"
+
+  # 1. cpio -pld (file-list driven hardlinks — ideal)
+  if command -v cpio >/dev/null 2>&1; then
+    if echo "$probe_src" | cpio -pld "$DST" >/dev/null 2>&1; then
+      rm -f "$probe_dst" 2>/dev/null
+      STRATEGY="cpio"
+    fi
+  fi
+
+  # 2. hardlink via ln
+  if [ -z "$STRATEGY" ]; then
+    if ln "$probe_src" "$probe_dst" 2>/dev/null; then
+      rm -f "$probe_dst"
+      STRATEGY="cp-al"
+    fi
+  fi
+
+  # 3. reflink (btrfs/xfs/APFS)
+  if [ -z "$STRATEGY" ]; then
+    if [ "$(uname)" = "Darwin" ]; then
+      if cp -c "$probe_src" "$probe_dst" 2>/dev/null; then
+        rm -f "$probe_dst"
+        STRATEGY="reflink"
+      fi
+    else
+      if cp --reflink=always "$probe_src" "$probe_dst" 2>/dev/null; then
+        rm -f "$probe_dst"
+        STRATEGY="reflink"
+      fi
+    fi
+  fi
+
+  rm -f "$probe_dst" 2>/dev/null
+  [ -z "$STRATEGY" ] && STRATEGY="cp"
+fi
+
+# ── link gems ────────────────────────────────────────────────────
+
+link_cpio() {
+  # Read .scint-files, prefix each line with gem name, pipe to cpio -pld.
+  # One cpio call per gem (cpio needs a single source root).
+  local gem="$1"
+  local dotfiles="$SRC/$gem/.scint-files"
+  if [ -f "$dotfiles" ]; then
+    (cd "$SRC/$gem" && cpio -pld "$DST/$gem" < "$dotfiles" 2>/dev/null)
+  else
+    cp -al "$SRC/$gem" "$DST/$gem"
+  fi
+}
+
+# For cp-al: batch all gems into one call
+BATCH=()
+
+flush_batch() {
+  [ ${#BATCH[@]} -eq 0 ] && return
+  case "$STRATEGY" in
+    cp-al)
+      cp -al "${BATCH[@]}" "$DST/" 2>/dev/null || {
+        # Individual fallback on batch failure
+        for s in "${BATCH[@]}"; do
+          cp -al "$s" "$DST/" 2>/dev/null || cp -R "$s" "$DST/"
+        done
+      }
+      ;;
+    reflink)
+      if [ "$(uname)" = "Darwin" ]; then
+        cp -cR "${BATCH[@]}" "$DST/" 2>/dev/null || {
+          for s in "${BATCH[@]}"; do cp -R "$s" "$DST/"; done
+        }
+      else
+        cp --reflink=always -R "${BATCH[@]}" "$DST/" 2>/dev/null || {
+          for s in "${BATCH[@]}"; do cp -R "$s" "$DST/"; done
+        }
+      fi
+      ;;
+    cp)
+      cp -R "${BATCH[@]}" "$DST/"
+      ;;
+  esac
+  BATCH=()
+}
+
+while IFS= read -r gem; do
+  [ -z "$gem" ] && continue
+  [ -d "$SRC/$gem" ] || continue
+  [ -d "$DST/$gem" ] && continue
+
+  case "$STRATEGY" in
+    cpio)
+      link_cpio "$gem"
+      ;;
+    *)
+      BATCH+=("$SRC/$gem")
+      # Flush in chunks to stay under ARG_MAX
+      [ ${#BATCH[@]} -ge 200 ] && flush_batch
+      ;;
+  esac
+done
+
+flush_batch