gel 0.2.0

data/lib/gel/httpool.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "net/http"
+
+class Gel::Httpool
+  include MonitorMixin
+
+  require "logger"
+  Logger = ::Logger.new($stderr)
+  Logger.level = $DEBUG ? ::Logger::DEBUG : ::Logger::WARN
+
+  def initialize
+    super()
+
+    @pool = {}
+    @cond = new_cond
+
+    if block_given?
+      begin
+        yield self
+      ensure
+        close
+      end
+    end
+  end
+
+  def request(uri, request = Net::HTTP::Get.new(uri))
+    with_connection(uri) do |http|
+      logger.debug { "GET #{uri}" }
+
+      if uri.user
+        request.basic_auth(uri.user, uri.password || "")
+      end
+
+      t = Time.now
+      response = http.request(request)
+      logger.debug { "HTTP #{response.code} (#{response.message}) #{uri} [#{Time.now - t}s]" }
+      response
+    end
+  end
+
+  def close
+    https = nil
+
+    synchronize do
+      https = @pool.values.flatten
+      @pool = nil
+      @cond.broadcast
+    end
+
+    https.each(&:finish)
+  end
+
+  private
+
+  def ident_for(uri)
+    "#{uri.scheme}://#{uri.host}:#{uri.port}"
+  end
+
+  def logger
+    Logger
+  end
+
+  def connect(ident, uri)
+    logger.debug { "Connect #{ident}" }
+    t = Time.now
+    http = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https")
+    logger.debug { "  Connected #{ident} [#{Time.now - t}s]" }
+
+    http
+  end
+
+  def queue_new_connection(ident, uri)
+    Thread.new do
+      begin
+        http = connect(ident, uri)
+
+        synchronize do
+          unless Thread.current[:discard]
+            Thread.current[:result] = http
+            http = nil
+            @cond.broadcast
+          end
+        end
+
+        if http
+          checkin ident, http
+        end
+      rescue StandardError => exception
+        synchronize do
+          unless Thread.current[:discard]
+            Thread.current[:error] = exception
+            http = nil
+            @cond.broadcast
+          end
+        end
+      end
+    end
+  end
+
+  def checkin(ident, http)
+    synchronize do
+      if @pool
+        @pool[ident].push http
+        http = nil
+        @cond.broadcast
+      end
+    end
+
+    if http
+      http.finish
+    end
+  end
+
+  def checkout(ident, uri)
+    synchronize do
+      @pool[ident] ||= []
+
+      return @pool[ident].pop unless @pool[ident].empty?
+
+      thread = queue_new_connection(ident, uri)
+
+      @cond.wait_while { !thread[:result] && @pool[ident].empty? }
+
+      if thread[:result]
+        thread[:result]
+      elsif thread[:error]
+        raise thread[:error]
+      else
+        thread[:discard] = true
+        @pool[ident].pop
+      end
+    end
+  end
+
+  def with_connection(uri)
+    ident = ident_for(uri)
+    http = checkout(ident, uri)
+
+    yield http
+
+  ensure
+    if http
+      checkin ident, http
+    end
+  end
+end

data/lib/gel/installer.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "net/http"
+
+require_relative "work_pool"
+require_relative "git_depot"
+require_relative "package"
+require_relative "package/installer"
+
+class Gel::Installer
+  DOWNLOAD_CONCURRENCY = 6
+  COMPILE_CONCURRENCY = 4
+
+  include MonitorMixin
+
+  attr_reader :store
+
+  def initialize(store)
+    super()
+
+    @trace = nil
+
+    @messages = Queue.new
+
+    @store = store
+    @dependencies = Hash.new { |h, k| h[k] = [] }
+    @weights = Hash.new(1)
+    @pending = Hash.new(0)
+
+    @download_pool = Gel::WorkPool.new(DOWNLOAD_CONCURRENCY, monitor: self, name: "gel-download", collect_errors: true)
+    @compile_pool = Gel::WorkPool.new(COMPILE_CONCURRENCY, monitor: self, name: "gel-compile", collect_errors: true)
+
+    @download_pool.queue_order = -> ((_, name)) { -@weights[name] }
+    @compile_pool.queue_order = -> ((_, name)) { -@weights[name] }
+
+    @git_depot = Gel::GitDepot.new(store)
+
+    @compile_waiting = []
+  end
+
+  def known_dependencies(deps)
+    deps = deps.dup
+
+    synchronize do
+      @dependencies.update(deps) { |k, l, r| deps[k] = r - l; l | r }
+      return if deps.values.all?(&:empty?)
+
+      deps.each do |dependent, dependencies|
+        dependencies.each do |dependency|
+          add_weight dependency, @weights[dependent]
+        end
+      end
+
+      # Every time we learn about a new dependency, we reorder the
+      # queues to ensure the most depended-on gems are processed first.
+      # This ensures we can start compiling extension gems as soon as
+      # possible.
+      @download_pool.reorder_queue!
+      @compile_pool.reorder_queue!
+    end
+  end
+
+  def install_gem(catalogs, name, version)
+    synchronize do
+      raise "catalogs is nil" if catalogs.nil?
+      @pending[name] += 1
+      @download_pool.queue(name) do
+        work_download([catalogs, name, version])
+      end
+    end
+  end
+
+  def load_git_gem(remote, revision, name)
+    synchronize do
+      @pending[name] += 1
+      @download_pool.queue(name) do
+        work_git(remote, revision, name)
+      end
+    end
+  end
+
+  def work_git(remote, revision, name)
+    @git_depot.checkout(remote, revision)
+
+    @messages << "Using #{name} (git)\n"
+    @pending[name] -= 1
+  end
+
+  def download_gem(catalogs, name, version)
+    catalogs.each do |catalog|
+      if fpath = catalog.cached_gem(name, version)
+        return fpath
+      end
+    end
+
+    catalogs.each do |catalog|
+      begin
+        return catalog.download_gem(name, version)
+      rescue Net::HTTPExceptions
+      end
+    end
+
+    raise "Unable to locate #{name} #{version} in: #{catalogs.join ", "}"
+  end
+
+  def work_download((catalogs, name, version))
+    fpath = download_gem(catalogs, name, version)
+
+    installer = Gel::Package::Installer.new(store)
+    g = Gel::Package.extract(fpath, installer)
+    known_dependencies g.spec.name => g.spec.runtime_dependencies.keys
+    if g.needs_compile?
+      synchronize do
+        add_weight name, 1000
+
+        @compile_pool.queue(g.spec.name) do
+          work_compile(g)
+        end
+      end
+    else
+      work_install(g)
+    end
+  end
+
+  def work_compile(g)
+    synchronize do
+      unless compile_ready?(g.spec.name)
+        @compile_waiting << g
+        return
+      end
+    end
+
+    g.compile
+    work_install(g)
+  end
+
+  def work_install(g)
+    @messages << "Installing #{g.spec.name} (#{g.spec.version})\n"
+    g.install
+    @pending[g.spec.name] -= 1
+
+    synchronize do
+      compile_recheck, @compile_waiting = @compile_waiting, []
+
+      compile_recheck.each do |g|
+        @compile_pool.queue(g.spec.name) do
+          work_compile(g)
+        end
+      end
+    end
+  end
+
+  def wait(output = nil)
+    clear = ""
+    tty = output && output.isatty
+
+    pools = { "Downloading" => @download_pool, "Compiling" => @compile_pool }
+
+    return if pools.values.all?(&:idle?)
+
+    update_status = lambda do
+      synchronize do
+        if output
+          output.write clear
+          output.write @messages.pop until @messages.empty?
+
+          if tty
+            messages = pools.map { |label, pool| pool_status(label, pool, label == "Compiling" ? @compile_waiting.size : 0) }.compact
+            if messages.empty?
+              msgline = ""
+            else
+              msgline = "[" + messages.join(";   ") + "]"
+            end
+            clear = "\r" + " " * msgline.size + "\r"
+            output.write msgline
+          end
+        else
+          @messages.pop until @messages.empty?
+        end
+        pools.values.all?(&:idle?) && @compile_waiting.empty?
+      end
+    end
+
+    pools.values.map do |pool|
+      Thread.new do
+        Thread.current.abort_on_exception = true
+
+        pool.wait(&update_status)
+        pools.values.each(&:tick!)
+        pool.stop
+      end
+    end.each(&:join)
+
+    errors = @download_pool.errors + @compile_pool.errors
+
+    if errors.empty?
+      if output
+        output.write "Installed #{@download_pool.count} gems\n"
+      end
+    else
+      if output
+        output.write "Installed #{@download_pool.count - errors.size} of #{@download_pool.count} gems\n\nErrors encountered with #{errors.size} gems:\n\n"
+        errors.each do |(_, name), exception|
+          output.write "#{name}\n  #{exception}\n\n"
+        end
+      end
+
+      if errors.first
+        raise errors.first.last
+      else
+        raise "Errors encountered while installing gems"
+      end
+    end
+  end
+
+  private
+
+  def compile_ready?(name)
+    @dependencies[name].all? do |dep|
+      if @pending[dep] == 0
+        compile_ready?(dep)
+      elsif @download_pool.errors.any? { |(_, failed_name), ex| failed_name == dep }
+        raise "Depends on #{dep.inspect}, which failed to download"
+      elsif @compile_pool.errors.any? { |(_, failed_name), ex| failed_name == dep }
+        raise "Depends on #{dep.inspect}, which failed to compile"
+      else
+        false
+      end
+    end
+  end
+
+  def pool_status(label, pool, extra_queue = 0)
+    st = pool.status
+    queue = st[:queued] + extra_queue
+
+    return if st[:active].empty? && queue.zero?
+
+    msg = +"#{label}:"
+    msg << " #{st[:active].join(" ")}" unless st[:active].empty?
+    msg << " +#{queue}" unless queue.zero?
+    msg
+  end
+
+  def add_weight(name, weight)
+    @weights[name] += weight
+    @dependencies[name].each do |dependency|
+      add_weight dependency, weight
+    end
+  end
+end

data/lib/gel/lock_loader.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+class Gel::LockLoader
+  attr_reader :filename
+  attr_reader :gemfile
+
+  def initialize(filename, gemfile = nil)
+    @filename = filename
+    @gemfile = gemfile
+  end
+
+  def lock_content
+    @lock_content ||= Gel::LockParser.new.parse(File.read(filename))
+  end
+
+  def each_gem
+    lock_content.each do |(section, body)|
+      case section
+      when "GEM", "PATH", "GIT"
+        specs = body["specs"]
+        specs.each do |gem_spec, dep_specs|
+          gem_spec =~ /\A(.+) \(([^-]+)(?:-(.+))?\)\z/
+          name, version, platform = $1, $2, $3
+
+          if dep_specs
+            deps = dep_specs.map do |spec|
+              spec =~ /\A(.+?)(?: \((.+)\))?\z/
+              [$1, $2 ? $2.split(", ") : []]
+            end
+          else
+            deps = []
+          end
+
+          sym =
+            case section
+            when "GEM"; :gem
+            when "PATH"; :path
+            when "GIT"; :git
+            end
+          yield sym, body, name, version, platform, deps
+        end
+      when "PLATFORMS", "DEPENDENCIES"
+      when "BUNDLED WITH"
+      else
+        warn "Unknown lockfile section #{section.inspect}"
+      end
+    end
+  end
+
+  def bundler_version
+    _, (version,) = lock_content.assoc("BUNDLED WITH")
+    version
+  end
+
+  def gem_names
+    names = []
+    each_gem do |section, body, name, version, platform, deps|
+      names << name
+    end
+    names
+  end
+
+  def activate(env, base_store, install: false, output: nil)
+    locked_store = Gel::LockedStore.new(base_store)
+
+    locks = {}
+
+    if install
+      require_relative "installer"
+      installer = Gel::Installer.new(base_store)
+    end
+
+    filtered_gems = Hash.new(nil)
+    top_gems = []
+    if gemfile && env
+      gemfile.gems.each do |name, *|
+        filtered_gems[name] = false
+      end
+      env.filtered_gems(gemfile.gems).each do |name, *|
+        top_gems << name
+        filtered_gems[name] = true
+      end
+    elsif pair = lock_content.assoc("DEPENDENCIES")
+      _, list = pair
+      top_gems = list.map { |name| name.split(" ", 2)[0].chomp("!") }
+      top_gems.each do |name|
+        filtered_gems[name] = true
+      end
+    end
+
+    gems = {}
+    each_gem do |section, body, name, version, platform, deps|
+      next if env && !env.platform?(platform)
+
+      gems[name] = [section, body, version, platform, deps]
+
+      installer.known_dependencies name => deps.map(&:first) if installer
+    end
+
+    walk = lambda do |name|
+      filtered_gems[name] = true
+      next unless gems[name]
+      gems[name].last.map(&:first).each do |dep_name|
+        walk[dep_name] unless filtered_gems[dep_name]
+      end
+    end
+
+    top_gems.each(&walk)
+
+    require_relative "git_depot"
+    require_relative "work_pool"
+
+    Gel::WorkPool.new(8) do |work_pool|
+      git_depot = Gel::GitDepot.new(base_store)
+
+      gems.each do |name, (section, body, version, platform, _deps)|
+        next unless filtered_gems[name]
+
+        if section == :gem
+          if installer && !base_store.gem?(name, version, platform)
+            require_relative "catalog"
+            catalogs = body["remote"].map { |r| Gel::Catalog.new(r, work_pool: work_pool) }
+            installer.install_gem(catalogs, name, platform ? "#{version}-#{platform}" : version)
+          end
+
+          locks[name] = version
+        else
+          if section == :git
+            remote = body["remote"].first
+            revision = body["revision"].first
+
+            dir = git_depot.git_path(remote, revision)
+            if installer && !Dir.exist?(dir)
+              installer.load_git_gem(remote, revision, name)
+
+              locks[name] = -> { Gel::DirectGem.new(dir, name, version) }
+              next
+            end
+          else
+            dir = File.expand_path(body["remote"].first, File.dirname(filename))
+          end
+
+          locks[name] = Gel::DirectGem.new(dir, name, version)
+        end
+      end
+
+      installer.wait(output) if installer
+
+      locks.each do |name, locked|
+        locks[name] = locked.call if locked.is_a?(Proc)
+      end
+    end
+
+    locked_store.lock(locks)
+
+    if env
+      env.open(locked_store)
+
+      env.gems_from_lock(locks)
+    end
+
+    locked_store
+  end
+end