scint 0.1.0

data/lib/scint/scheduler.rb

@@ -0,0 +1,392 @@
+# frozen_string_literal: true
+
+require_relative "worker_pool"
+require_relative "progress"
+require_relative "platform"
+
+module Scint
+  class Scheduler
+    # Job types in priority order (lower index = higher priority)
+    PRIORITIES = {
+      fetch_index: 0,
+      git_clone:   1,
+      resolve:     2,
+      download:    3,
+      extract:     4,
+      link:        5,
+      build_ext:   6,
+      binstub:     7,
+    }.freeze
+
+    Job = Struct.new(:id, :type, :name, :payload, :state, :result, :error,
+                     :depends_on, :enqueued_at, keyword_init: true)
+
+    attr_reader :errors, :progress
+
+    # max_workers: hard ceiling (default cpu_count * 2, capped at 50)
+    # initial_workers: how many threads to start with (default 1 — grow dynamically)
+    # per_type_limits: optional hash { job_type => max_concurrent }
+    # Example: { build_ext: 1, binstub: 1, link: 30 }
+    def initialize(max_workers: nil, initial_workers: 1, progress: nil, fail_fast: false, per_type_limits: {})
+      @max_workers = [max_workers || Platform.cpu_count * 2, 50].min
+      @initial_workers = [[initial_workers, 1].max, @max_workers].min
+      @current_workers = @initial_workers
+      @progress = progress || Progress.new
+      @fail_fast = fail_fast
+      @aborted = false
+
+      @mutex = Thread::Mutex.new
+      @cv = Thread::ConditionVariable.new
+
+      @jobs = {}           # id => Job
+      @pending = []        # sorted by priority
+      @running = {}        # id => Job
+      @completed = {}      # id => Job
+      @failed = {}         # id => Job
+      @running_by_type = Hash.new(0)
+      @per_type_limits = normalize_per_type_limits(per_type_limits)
+
+      @errors = []         # collected errors
+      @next_id = 0
+      @pool = nil
+      @started = false
+      @shutting_down = false
+      @in_flight_follow_ups = 0  # track follow-ups being executed
+
+      # Callbacks: type => [proc]
+      @on_complete_callbacks = Hash.new { |h, k| h[k] = [] }
+
+      # Waiters: type => [ConditionVariable]
+      @type_waiters = Hash.new { |h, k| h[k] = [] }
+    end
+
+    def start
+      return if @started
+      @started = true
+      @progress.start
+
+      @pool = WorkerPool.new(@max_workers, name: "scheduler")
+      @pool.start(@initial_workers) do |job|
+        execute_job(job)
+      end
+
+      # Dispatcher thread: pulls from priority queue and feeds the pool
+      @dispatcher = Thread.new do
+        Thread.current.name = "scheduler-dispatch"
+        begin
+          dispatch_loop
+        rescue Exception => e
+          $stderr.puts "\n!!! DISPATCHER THREAD CRASHED !!!"
+          $stderr.puts "Exception: #{e.class}: #{e.message}"
+          $stderr.puts e.backtrace.first(10).map { |l| "  #{l}" }.join("\n")
+          raise
+        end
+      end
+    end
+
+    # Enqueue a job. Returns the job id.
+    # depends_on: array of job ids that must complete before this runs.
+    # follow_up: proc that receives (job) and can enqueue more jobs.
+    def enqueue(type, name, payload = nil, depends_on: [], follow_up: nil)
+      raise "scheduler not started" unless @started
+      raise "unknown job type: #{type}" unless PRIORITIES.key?(type)
+
+      job = nil
+      @mutex.synchronize do
+        return nil if @aborted
+
+        id = @next_id += 1
+        job = Job.new(
+          id: id,
+          type: type,
+          name: name,
+          payload: { data: payload, follow_up: follow_up },
+          state: :pending,
+          depends_on: depends_on.dup,
+        )
+        @jobs[id] = job
+        insert_pending(job)
+        @cv.broadcast
+      end
+
+      @progress.on_enqueue(job.id, job.type, job.name)
+      job.id
+    end
+
+    # Wait for all jobs of a specific type to complete.
+    # Returns once no pending/running jobs of that type remain.
+    def wait_for(type)
+      @mutex.synchronize do
+        loop do
+          pending_of_type = @pending.any? { |j| j.type == type }
+          running_of_type = @running.values.any? { |j| j.type == type }
+          break if @aborted
+          break unless pending_of_type || running_of_type
+          @cv.wait(@mutex)
+        end
+      end
+    end
+
+    # Wait for a specific job to complete. Returns the Job.
+    def wait_for_job(job_id)
+      @mutex.synchronize do
+        loop do
+          job = @jobs[job_id]
+          return job if job.nil? || job.state == :completed || job.state == :failed
+          @cv.wait(@mutex)
+        end
+      end
+    end
+
+    # Wait for ALL jobs to finish (including any in-flight follow-ups).
+    def wait_all
+      @mutex.synchronize do
+        loop do
+          break if @running.empty? && @in_flight_follow_ups == 0 && (@pending.empty? || @aborted)
+          @cv.wait(@mutex)
+        end
+      end
+    end
+
+    # Register a callback for when jobs of a given type complete.
+    def on_complete(type, &block)
+      @mutex.synchronize do
+        @on_complete_callbacks[type] << block
+      end
+    end
+
+    # Scale to exactly n workers (clamped to [current, max_workers]).
+    # Never shrinks — if n < current workers, this is a no-op.
+    def scale_to(n)
+      return unless @pool
+      target = [[n, @current_workers].max, @max_workers].min
+      return if target <= @current_workers
+
+      @pool.grow_to(target)
+      @current_workers = target
+    end
+
+    # Auto-scale based on pending queue depth.
+    # Formula: target = clamp(queue_depth / 4, 1, max_workers)
+    def adjust_workers(queue_depth = nil)
+      depth = queue_depth || @mutex.synchronize { @pending.size }
+      target = [[1, (depth / 4.0).ceil].max, @max_workers].min
+      scale_to(target)
+    end
+
+    # Scale worker count based on workload hint (e.g. gem count, download count).
+    # Convenience wrapper: after Gemfile parse pass gem_count,
+    # after resolution pass download_count.
+    def scale_workers(hint)
+      return unless @pool
+      target = [[1, (hint / 3.0).ceil].max, @max_workers].min
+      scale_to(target)
+    end
+
+    def current_workers
+      @current_workers
+    end
+
+    def max_workers
+      @max_workers
+    end
+
+    # Gracefully shut down: wait for all work, then stop threads.
+    def shutdown
+      return unless @started
+      wait_all
+
+      @mutex.synchronize { @shutting_down = true }
+      @cv.broadcast
+      @dispatcher&.join(5)
+      @pool&.stop
+      @progress.stop
+      @started = false
+    end
+
+    def stats
+      @mutex.synchronize do
+        {
+          pending: @pending.size,
+          running: @running.size,
+          completed: @completed.size,
+          failed: @failed.size,
+          total: @jobs.size,
+          workers: @current_workers,
+          max_workers: @max_workers,
+        }
+      end
+    end
+
+    def failed?
+      @mutex.synchronize { !@failed.empty? || !@errors.empty? }
+    end
+
+    def aborted?
+      @mutex.synchronize { @aborted }
+    end
+
+    private
+
+    def dispatch_loop
+      loop do
+        job = nil
+
+        @mutex.synchronize do
+          loop do
+            break if @shutting_down
+            break if @aborted && @running.empty?
+            if @aborted
+              @cv.wait(@mutex)
+              next
+            end
+
+            # Backpressure: keep at most @current_workers in-flight so a
+            # fail-fast error can still halt most pending work.
+            if @running.size >= @current_workers
+              @cv.wait(@mutex)
+              next
+            end
+
+            job = pick_ready_job
+            break if job
+
+            # Nothing ready yet — wait for state change
+            @cv.wait(@mutex)
+          end
+
+          if job
+            job.state = :running
+            @running[job.id] = job
+            @running_by_type[job.type] += 1
+          end
+        end
+
+        break if (@shutting_down || (@aborted && job.nil?)) && job.nil?
+        next unless job
+
+        @progress.on_start(job.id, job.type, job.name)
+
+        @pool.enqueue(job) do |pool_job|
+          finished_job = pool_job[:payload]
+          handle_completion(finished_job, pool_job[:error])
+        end
+      end
+    end
+
+    # Must be called inside @mutex
+    def pick_ready_job
+      @pending.each_with_index do |job, idx|
+        next unless type_slot_available?(job.type)
+
+        # Check if dependencies are met
+        deps_met = job.depends_on.all? do |dep_id|
+          dep = @jobs[dep_id]
+          dep && (dep.state == :completed || dep.state == :failed)
+        end
+
+        if deps_met
+          @pending.delete_at(idx)
+          return job
+        end
+      end
+      nil
+    end
+
+    # Insert into pending list maintaining priority order
+    def insert_pending(job)
+      priority = PRIORITIES[job.type] || 99
+      idx = @pending.bsearch_index { |j| (PRIORITIES[j.type] || 99) > priority }
+      if idx
+        @pending.insert(idx, job)
+      else
+        @pending.push(job)
+      end
+    end
+
+    def execute_job(job)
+      data = job.payload[:data]
+      if data.respond_to?(:call)
+        data.call
+      elsif data.is_a?(Hash) && data[:proc]
+        data[:proc].call
+      else
+        data
+      end
+    end
+
+    def handle_completion(job, error)
+      follow_up = job.payload[:follow_up]
+      callbacks = nil
+      run_follow_up = false
+
+      @mutex.synchronize do
+        @running.delete(job.id)
+        @running_by_type[job.type] -= 1 if @running_by_type[job.type] > 0
+
+        if error
+          job.state = :failed
+          job.error = error
+          @failed[job.id] = job
+          @errors << { job_id: job.id, type: job.type, name: job.name, error: error }
+          @aborted = true if @fail_fast
+        else
+          job.state = :completed
+          @completed[job.id] = job
+
+          callbacks = @on_complete_callbacks[job.type].dup
+
+          if follow_up && !@aborted
+            run_follow_up = true
+            @in_flight_follow_ups += 1
+          end
+        end
+
+        @cv.broadcast
+      end
+
+      # Run progress callbacks outside mutex (they don't mutate scheduler state)
+      if error
+        @progress.on_fail(job.id, job.type, job.name, error)
+      else
+        @progress.on_complete(job.id, job.type, job.name)
+        callbacks&.each { |cb| cb.call(job) }
+      end
+
+      # Run follow-up outside mutex so it can call enqueue without deadlock
+      if run_follow_up
+        begin
+          follow_up.call(job)
+        rescue => e
+          @mutex.synchronize do
+            @errors << { job_id: job.id, type: job.type, name: job.name, error: e, phase: :follow_up }
+          end
+        ensure
+          @mutex.synchronize do
+            @in_flight_follow_ups -= 1
+            @cv.broadcast
+          end
+        end
+      end
+    end
+
+    def normalize_per_type_limits(limits)
+      out = {}
+      limits.each do |type, limit|
+        next unless PRIORITIES.key?(type)
+        next if limit.nil?
+
+        n = limit.to_i
+        out[type] = n if n > 0
+      end
+      out
+    end
+
+    def type_slot_available?(type)
+      limit = @per_type_limits[type]
+      return true unless limit
+
+      @running_by_type[type] < limit
+    end
+  end
+end

data/lib/scint/source/base.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Scint
+  module Source
+    # Abstract base class for all gem sources.
+    # Subclasses must implement: #name, #uri, #specs, #fetch_spec,
+    # #cache_slug, #to_lock, #eql?, #hash.
+    class Base
+      def name
+        raise NotImplementedError
+      end
+
+      def uri
+        raise NotImplementedError
+      end
+
+      # Return an array of available specs from this source.
+      def specs
+        raise NotImplementedError
+      end
+
+      # Fetch a specific spec by name, version, and platform.
+      def fetch_spec(name, version, platform = "ruby")
+        raise NotImplementedError
+      end
+
+      # A unique slug used for cache directory naming.
+      def cache_slug
+        raise NotImplementedError
+      end
+
+      # Lockfile representation (the header section, e.g. "GEM\n  remote: ...\n  specs:\n")
+      def to_lock
+        raise NotImplementedError
+      end
+
+      def to_s
+        "#{self.class.name.split("::").last}: #{uri}"
+      end
+
+      def ==(other)
+        eql?(other)
+      end
+    end
+  end
+end

data/lib/scint/source/git.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require "digest/sha2"
+
+module Scint
+  module Source
+    class Git < Base
+      attr_reader :uri, :revision, :ref, :branch, :tag, :submodules, :glob
+
+      DEFAULT_GLOB = "{,*,*/*}.gemspec"
+
+      def initialize(uri:, revision: nil, ref: nil, branch: nil, tag: nil,
+                     submodules: nil, glob: nil, name: nil)
+        @uri = uri.to_s
+        @revision = revision
+        @ref = ref || branch || tag
+        @branch = branch
+        @tag = tag
+        @submodules = submodules
+        @glob = glob || DEFAULT_GLOB
+        @name = name
+      end
+
+      def self.from_lock(options)
+        new(
+          uri: options.delete("remote"),
+          revision: options["revision"],
+          ref: options["ref"],
+          branch: options["branch"],
+          tag: options["tag"],
+          submodules: options["submodules"],
+          glob: options["glob"],
+          name: options["name"],
+        )
+      end
+
+      def name
+        @name || File.basename(uri, ".git")
+      end
+
+      def specs
+        [] # Loaded from checked-out gemspec at resolution time
+      end
+
+      def fetch_spec(name, version, platform = "ruby")
+        nil
+      end
+
+      def cache_slug
+        "#{name}-#{uri_hash}"
+      end
+
+      def to_lock
+        out = String.new("GIT\n")
+        out << "  remote: #{@uri}\n"
+        out << "  revision: #{@revision}\n" if @revision
+        out << "  ref: #{@ref}\n" if @ref && @ref != @branch && @ref != @tag
+        out << "  branch: #{@branch}\n" if @branch
+        out << "  tag: #{@tag}\n" if @tag
+        out << "  submodules: #{@submodules}\n" if @submodules
+        out << "  glob: #{@glob}\n" unless @glob == DEFAULT_GLOB
+        out << "  specs:\n"
+        out
+      end
+
+      def eql?(other)
+        other.is_a?(Git) &&
+          uri == other.uri &&
+          ref == other.ref &&
+          branch == other.branch &&
+          tag == other.tag &&
+          submodules == other.submodules
+      end
+
+      def hash
+        [self.class, uri, ref, branch, tag, submodules].hash
+      end
+
+      def to_s
+        at = tag || branch || ref
+        "git: #{@uri}#{" (#{at})" if at}"
+      end
+
+      private
+
+      def uri_hash
+        Digest::SHA256.hexdigest(@uri)[0, 12]
+      end
+    end
+  end
+end

data/lib/scint/source/path.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Scint
+  module Source
+    class Path < Base
+      attr_reader :path, :glob
+
+      DEFAULT_GLOB = "{,*,*/*}.gemspec"
+
+      def initialize(path:, glob: nil, name: nil, version: nil)
+        @path = path.to_s
+        @glob = glob || DEFAULT_GLOB
+        @name = name
+        @version = version
+      end
+
+      def self.from_lock(options)
+        new(
+          path: options.delete("remote"),
+          glob: options["glob"],
+          name: options["name"],
+          version: options["version"],
+        )
+      end
+
+      def name
+        @name || File.basename(@path)
+      end
+
+      def uri
+        @path
+      end
+
+      def specs
+        [] # Loaded from gemspec on disk
+      end
+
+      def fetch_spec(name, version, platform = "ruby")
+        nil
+      end
+
+      def cache_slug
+        name
+      end
+
+      def to_lock
+        out = String.new("PATH\n")
+        out << "  remote: #{@path}\n"
+        out << "  glob: #{@glob}\n" unless @glob == DEFAULT_GLOB
+        out << "  specs:\n"
+        out
+      end
+
+      def eql?(other)
+        other.is_a?(Path) &&
+          File.expand_path(path) == File.expand_path(other.path)
+      end
+
+      def hash
+        [self.class, File.expand_path(path)].hash
+      end
+
+      def to_s
+        "path: #{@path}"
+      end
+    end
+  end
+end

data/lib/scint/source/rubygems.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require "uri"
+
+module Scint
+  module Source
+    class Rubygems < Base
+      attr_reader :remotes
+
+      def initialize(remotes: [])
+        @remotes = Array(remotes).map { |r| normalize_uri(r) }
+      end
+
+      def self.from_lock(options)
+        remotes = Array(options["remote"]).reverse
+        new(remotes: remotes)
+      end
+
+      def name
+        "rubygems"
+      end
+
+      def uri
+        @remotes.first
+      end
+
+      def add_remote(remote)
+        remote = normalize_uri(remote)
+        @remotes << remote unless @remotes.include?(remote)
+      end
+
+      def specs
+        [] # Populated by compact index client at resolution time
+      end
+
+      def fetch_spec(name, version, platform = "ruby")
+        nil # Delegated to compact index / API at resolution time
+      end
+
+      def cache_slug
+        uri_obj = URI.parse(@remotes.first.to_s)
+        path = uri_obj.path.gsub("/", "-").sub(/^-/, "")
+        slug = uri_obj.host.to_s
+        slug += path unless path.empty? || path == "-"
+        slug
+      end
+
+      def to_lock
+        out = String.new("GEM\n")
+        @remotes.reverse_each do |remote|
+          out << "  remote: #{remote}\n"
+        end
+        out << "  specs:\n"
+        out
+      end
+
+      def eql?(other)
+        other.is_a?(Rubygems) && @remotes == other.remotes
+      end
+
+      def hash
+        [self.class, @remotes].hash
+      end
+
+      def to_s
+        "rubygems (#{@remotes.join(", ")})"
+      end
+
+      private
+
+      def normalize_uri(uri)
+        uri = uri.to_s
+        uri = "#{uri}/" unless uri.end_with?("/")
+        uri
+      end
+    end
+  end
+end

data/lib/scint/vendor/pub_grub/assignment.rb

@@ -0,0 +1,20 @@
+module Scint::PubGrub
+  class Assignment
+    attr_reader :term, :cause, :decision_level, :index
+    def initialize(term, cause, decision_level, index)
+      @term = term
+      @cause = cause
+      @decision_level = decision_level
+      @index = index
+    end
+
+    def self.decision(package, version, decision_level, index)
+      term = Term.new(VersionConstraint.exact(package, version), true)
+      new(term, :decision, decision_level, index)
+    end
+
+    def decision?
+      cause == :decision
+    end
+  end
+end