gel 0.3.0 → 0.8.0.pre1

data/lib/gel/installer.rb

@@ -9,6 +9,9 @@ require_relative "package"
 require_relative "package/installer"
 
 class Gel::Installer
+  class SkipCatalog < Exception
+  end
+
   DOWNLOAD_CONCURRENCY = 6
   COMPILE_CONCURRENCY = 4
 
@@ -62,6 +65,8 @@ class Gel::Installer
   end
 
   def install_gem(catalogs, name, version)
+    raise "Refusing to install incompatible #{name.inspect}" if Gel::Environment::IGNORE_LIST.include?(name)
+
     synchronize do
       raise "catalogs is nil" if catalogs.nil?
       @pending[name] += 1
@@ -72,6 +77,13 @@ class Gel::Installer
   end
 
   def load_git_gem(remote, revision, name)
+    dir = @git_depot.git_path(remote, revision)
+
+    if Dir.exist?(dir)
+      # already checked out; we assume it's also compiled if necessary
+      return
+    end
+
     synchronize do
       @pending[name] += 1
       @download_pool.queue(name) do
@@ -81,22 +93,35 @@ class Gel::Installer
   end
 
   def work_git(remote, revision, name)
-    @git_depot.checkout(remote, revision)
+    dir = @git_depot.checkout(remote, revision)
 
-    @messages << "Using #{name} (git)\n"
-    @pending[name] -= 1
-  end
+    if filename = ["#{dir}/#{name}.gemspec", "#{dir}/#{name}/#{name}.gemspec"].detect { |f| File.exist?(f) }
+      spec = Gel::GemspecParser.parse(File.read(filename), filename)
+      g = Gel::Package::Installer::GitCompiler.new(spec, store, dir)
 
-  def download_gem(catalogs, name, version)
-    catalogs.each do |catalog|
-      if fpath = catalog.cached_gem(name, version)
-        return fpath
+      known_dependencies g.spec.name => g.spec.runtime_dependencies.map(&:first)
+      if g.needs_compile?
+        synchronize do
+          add_weight name, 1000
+
+          @compile_pool.queue(g.spec.name) do
+            work_compile(g)
+          end
+        end
+      else
+        work_install(g)
       end
+    else
+      clear_pending(name)
     end
+  end
 
+  def download_gem(catalogs, name, version)
     catalogs.each do |catalog|
       begin
-        return catalog.download_gem(name, version)
+        if fpath = catalog.download_gem(name, version)
+          return fpath
+        end
       rescue Net::HTTPExceptions
       end
     end
@@ -131,21 +156,34 @@ class Gel::Installer
       end
     end
 
-    g.compile
-    work_install(g)
+    begin
+      g.compile
+    rescue
+      clear_pending(g.spec.name)
+      raise
+    else
+      work_install(g)
+    end
   end
 
   def work_install(g)
-    @messages << "Installing #{g.spec.name} (#{g.spec.version})\n"
-    g.install
-    @pending[g.spec.name] -= 1
+    if g.is_a?(Gel::Package::Installer::GemInstaller)
+      @messages << "Installing #{g.spec.name} (#{g.spec.version})\n"
+      g.install
+    end
+
+    clear_pending(g.spec.name)
+  end
+
+  def clear_pending(name)
+    @pending[name] -= 1
 
     synchronize do
       compile_recheck, @compile_waiting = @compile_waiting, []
 
-      compile_recheck.each do |g|
-        @compile_pool.queue(g.spec.name) do
-          work_compile(g)
+      compile_recheck.each do |gem|
+        @compile_pool.queue(gem.spec.name) do
+          work_compile(gem)
         end
       end
     end
@@ -202,7 +240,7 @@ class Gel::Installer
       if output
         output.write "Installed #{@download_pool.count - errors.size} of #{@download_pool.count} gems\n\nErrors encountered with #{errors.size} gems:\n\n"
         errors.each do |(_, name), exception|
-          output.write "#{name}\n  #{exception}\n\n"
+          output.write "#{name}\n#{exception.to_s.gsub(/^(?=.)/, "  ")}\n\n"
         end
       end
 
@@ -218,12 +256,12 @@ class Gel::Installer
 
   def compile_ready?(name)
     @dependencies[name].all? do |dep|
-      if @pending[dep] == 0
-        compile_ready?(dep)
-      elsif @download_pool.errors.any? { |(_, failed_name), ex| failed_name == dep }
-        raise "Depends on #{dep.inspect}, which failed to download"
+      if @download_pool.errors.any? { |(_, failed_name), ex| failed_name == dep }
+        raise Gel::Error::ExtensionDependencyError.new(dependency: dep, failure: "download")
       elsif @compile_pool.errors.any? { |(_, failed_name), ex| failed_name == dep }
-        raise "Depends on #{dep.inspect}, which failed to compile"
+        raise Gel::Error::ExtensionDependencyError.new(dependency: dep, failure: "compile")
+      elsif @pending[dep] == 0
+        compile_ready?(dep)
       else
         false
       end

data/lib/gel/lock_loader.rb

@@ -1,64 +1,21 @@
 # frozen_string_literal: true
 
+require_relative "resolved_gem_set"
+require_relative "git_catalog"
+require_relative "path_catalog"
+
+require_relative "support/gem_platform"
+
 class Gel::LockLoader
-  attr_reader :filename
   attr_reader :gemfile
 
-  def initialize(filename, gemfile = nil)
-    @filename = filename
+  def initialize(gem_set, gemfile = nil)
+    @gem_set = gem_set
     @gemfile = gemfile
   end
 
-  def lock_content
-    @lock_content ||= Gel::LockParser.new.parse(File.read(filename))
-  end
-
-  def each_gem
-    lock_content.each do |(section, body)|
-      case section
-      when "GEM", "PATH", "GIT"
-        specs = body["specs"]
-        specs.each do |gem_spec, dep_specs|
-          gem_spec =~ /\A(.+) \(([^-]+)(?:-(.+))?\)\z/
-          name, version, platform = $1, $2, $3
-
-          if dep_specs
-            deps = dep_specs.map do |spec|
-              spec =~ /\A(.+?)(?: \((.+)\))?\z/
-              [$1, $2 ? $2.split(", ") : []]
-            end
-          else
-            deps = []
-          end
-
-          sym =
-            case section
-            when "GEM"; :gem
-            when "PATH"; :path
-            when "GIT"; :git
-            end
-          yield sym, body, name, version, platform, deps
-        end
-      when "PLATFORMS", "DEPENDENCIES"
-      when "RUBY VERSION"
-      when "BUNDLED WITH"
-      else
-        warn "Unknown lockfile section #{section.inspect}"
-      end
-    end
-  end
-
-  def bundler_version
-    _, (version,) = lock_content.assoc("BUNDLED WITH")
-    version
-  end
-
   def gem_names
-    names = []
-    each_gem do |section, body, name, version, platform, deps|
-      names << name
-    end
-    names
+    @gem_set.gem_names
   end
 
   def activate(env, base_store, install: false, output: nil)
@@ -71,95 +28,113 @@ class Gel::LockLoader
       installer = Gel::Installer.new(base_store)
     end
 
-    filtered_gems = Hash.new(nil)
-    top_gems = []
+    gems_to_process = []
     if gemfile && env
-      gemfile.gems.each do |name, *|
-        filtered_gems[name] = false
-      end
       env.filtered_gems(gemfile.gems).each do |name, *|
-        top_gems << name
-        filtered_gems[name] = true
-      end
-    elsif pair = lock_content.assoc("DEPENDENCIES")
-      _, list = pair
-      top_gems = list.map { |name| name.split(" ", 2)[0].chomp("!") }
-      top_gems.each do |name|
-        filtered_gems[name] = true
+        gems_to_process << name
       end
+    elsif list = @gem_set.dependency_names
+      gems_to_process = list
     end
 
-    gems = {}
-    each_gem do |section, body, name, version, platform, deps|
-      next if env && !env.platform?(platform)
-
-      gems[name] = [section, body, version, platform, deps]
+    local_platform = Gel::Support::GemPlatform.local
+    all_gems = @gem_set.gems
 
-      installer.known_dependencies name => deps.map(&:first) if installer
+    deferred_direct_gem = lambda do |dir, name, version|
+      -> { Gel::DirectGem.new(dir, name, version) }
     end
 
-    walk = lambda do |name|
-      filtered_gems[name] = true
-      next unless gems[name]
-      gems[name].last.map(&:first).each do |dep_name|
-        walk[dep_name] unless filtered_gems[dep_name]
-      end
-    end
+    processed_gems = {}
+    while name = gems_to_process.shift
+      next if processed_gems[name]
+      processed_gems[name] = true
 
-    top_gems.each(&walk)
+      next if Gel::Environment::IGNORE_LIST.include?(name)
 
-    require_relative "git_depot"
-    require_relative "work_pool"
+      next unless all_versions = all_gems[name]
 
-    Gel::WorkPool.new(8) do |work_pool|
-      git_depot = Gel::GitDepot.new(base_store)
+      resolved_gems = all_versions.select do |rg|
+        local_platform =~ rg.platform || rg.platform.nil?
+      end.sort_by { |rg| rg.platform&.size || 0 }.reverse
 
-      gems.each do |name, (section, body, version, platform, _deps)|
-        next unless filtered_gems[name]
+      next if resolved_gems.empty?
 
-        if section == :gem
-          if installer && !base_store.gem?(name, version, platform)
-            require_relative "catalog"
-            catalogs = body["remote"].map { |r| Gel::Catalog.new(r, work_pool: work_pool) }
-            installer.install_gem(catalogs, name, platform ? "#{version}-#{platform}" : version)
-          end
+      resolved_gem = resolved_gems.first
+
+
+      case resolved_gem.catalog
+      when Gel::GitCatalog
+        dir = resolved_gem.catalog.path
 
-          locks[name] = version
-        else
-          if section == :git
-            remote = body["remote"].first
-            revision = body["revision"].first
+        installer.known_dependencies name => resolved_gem.deps.map(&:first) if installer
+        installer&.load_git_gem(resolved_gem.catalog.remote, resolved_gem.catalog.revision, name)
 
-            dir = git_depot.git_path(remote, revision)
-            if installer && !Dir.exist?(dir)
-              installer.load_git_gem(remote, revision, name)
+        locks[name] = deferred_direct_gem.call(dir, name, resolved_gem.version)
+      when Gel::PathCatalog
+        path = resolved_gem.catalog.path
 
-              locks[name] = -> { Gel::DirectGem.new(dir, name, version) }
-              next
+        dir = File.expand_path(path, File.dirname(@gem_set.filename))
+
+        installer.known_dependencies name => resolved_gem.deps.map(&:first) if installer
+        locks[name] = Gel::DirectGem.new(dir, name, resolved_gem.version)
+      else
+        unless resolved_gem = resolved_gems.find { |rg| base_store.gem?(name, rg.version, rg.platform) }
+          if installer
+            require_relative "catalog"
+
+            catalogs = @gem_set.server_catalogs
+
+            if resolved_gems.size > 1
+              skipped_matches = []
+
+              catalog_infos = catalogs.map { |c| c.gem_info(name) }
+              resolved_gems.each do |rg|
+                catalog_infos.each do |info|
+                  s = rg.platform ? "#{rg.version}-#{rg.platform}" : rg.version
+                  if i = info[s]
+                    if i[:ruby] && !Gel::Support::GemRequirement.new(i[:ruby].split("&")).satisfied_by?(Gel::Support::GemVersion.new(RUBY_VERSION))
+                      skipped_matches << s
+                    else
+                      resolved_gem = rg
+                      break
+                    end
+                  end
+                end
+
+                break if resolved_gem
+              end
+
+              if resolved_gem.nil?
+                raise UnsatisfiableRubyVersionError.new(name: name, running: RUBY_VERSION, attempted_platforms: skipped_matches)
+              end
+            else
+              resolved_gem = resolved_gems.first
             end
+
+            installer.known_dependencies name => resolved_gem.deps.map(&:first)
+            installer.install_gem(catalogs, name, resolved_gem.platform ? "#{resolved_gem.version}-#{resolved_gem.platform}" : resolved_gem.version)
           else
-            dir = File.expand_path(body["remote"].first, File.dirname(filename))
+            raise Gel::Error::MissingGemError.new(name: name)
           end
-
-          locks[name] = Gel::DirectGem.new(dir, name, version)
         end
+
+        locks[name] = resolved_gem.version.to_s
       end
 
-      installer.wait(output) if installer
 
-      locks.each do |name, locked|
-        locks[name] = locked.call if locked.is_a?(Proc)
+      resolved_gem.deps.map(&:first).each do |dep_name|
+        gems_to_process.unshift dep_name
       end
     end
 
-    locked_store.lock(locks)
-
-    if env
-      env.open(locked_store)
+    installer.wait(output) if installer
 
-      env.gems_from_lock(locks)
+    locks.each do |name, locked|
+      locks[name] = locked.call if locked.is_a?(Proc)
     end
 
+    locked_store.lock(locks)
+
     locked_store
   end
 end

data/lib/gel/lock_parser.rb

@@ -1,34 +1,27 @@
 # frozen_string_literal: true
 
-require "strscan"
-
 class Gel::LockParser
-  def parse(content)
+  def parse(input)
     sections = []
 
-    scanner = StringScanner.new(content)
-    while section = scanner.scan(/^(\w.+)\n/)
-      section.chomp!
+    input = input.dup
+    while input.sub!(/\A(\w.+)\n/, "")
+      section = $1
+
       case section
       when "GIT", "PATH", "GEM"
         content = {}
-        while scanner.skip(/^  \b/)
-          label = scanner.scan(/[^:]+:/).chop
-          if scanner.skip(/ /)
-            value = scanner.scan(/.*/)
-            scanner.skip(/\n/)
-            (content[label] ||= []) << value
-          else
-            scanner.skip(/\n/)
+        while input.sub!(/\A {2}\b([^:]+):(\s)/, "")
+          label = $1
+          separator = $2
+
+          if separator == "\n"
             value = []
-            while scanner.skip(/^    \b/)
-              entry = scanner.scan(/.*/)
-              scanner.skip(/\n/)
+            while input.sub!(/\A {4}\b(.*)\n/, "")
+              entry = $1
               children = []
-              while scanner.skip(/^      \b/)
-                child = scanner.scan(/.*/)
-                children << child
-                scanner.skip(/\n/)
+              while input.sub!(/\A {6}\b(.*)\n/, "")
+                children << $1
               end
               if children.empty?
                 value << [entry]
@@ -37,24 +30,23 @@ class Gel::LockParser
               end
             end
             content[label] = value
+          else
+            input.sub!(/\A(.*)\n/, "")
+            (content[label] ||= []) << $1
           end
         end
       when "PLATFORMS", "DEPENDENCIES"
         content = []
-        while scanner.skip(/^  \b/)
-          entry = scanner.scan(/.*/)
-          content << entry
-          scanner.skip(/\n/)
+        while input.sub!(/\A {2}\b(.*)\n/, "")
+          content << $1
         end
-      when "BUNDLED WITH"
+      when "BUNDLED WITH", "RUBY VERSION"
         content = []
-        while scanner.skip(/^   \b/)
-          entry = scanner.scan(/.*/)
-          content << entry
-          scanner.skip(/\n/)
+        while input.sub!(/\A {3}\b(.*)\n/, "")
+          content << $1
         end
       end
-      scanner.skip(/\n+/)
+      input.sub!(/\A\n+/, "")
 
       sections << [section, content]
     end

data/lib/gel/locked_store.rb

@@ -2,6 +2,7 @@
 
 class Gel::LockedStore
   attr_reader :inner
+  attr_reader :locked_versions
 
   def initialize(inner)
     @inner = inner
@@ -11,6 +12,10 @@ class Gel::LockedStore
     @full_cache = false
   end
 
+  def stub_set
+    @inner.stub_set
+  end
+
   def paths
     @inner.paths
   end
@@ -20,7 +25,7 @@ class Gel::LockedStore
   end
 
   def inspect
-    content = @locked_versions.map { |name, version| "#{name}=#{version.is_a?(String) ? version : version.root}" }
+    content = (@locked_versions || []).map { |name, version| "#{name}=#{version.is_a?(String) ? version : version.root}" }
     content = ["(none)"] if content.empty?
     content.sort!
 
@@ -33,8 +38,8 @@ class Gel::LockedStore
     inner_versions = {}
     locks.each do |name, version|
       if version.is_a?(Gel::StoreGem)
-        version.libs do |file, subdir|
-          @lib_cache[file] << [version, subdir]
+        version.libs do |file, subdir, ext|
+          @lib_cache[file] << [version, subdir, ext]
         end
       else
         inner_versions[name] = version
@@ -43,8 +48,8 @@ class Gel::LockedStore
 
     g = @inner.gems(inner_versions)
     @inner.libs_for_gems(inner_versions) do |name, version, subs|
-      subs.each do |subdir, files|
-        v = [g[name], subdir]
+      subs.each do |(subdir, ext), files|
+        v = [g[name], subdir, ext]
         files.each do |file|
           @lib_cache[file] << v
         end
@@ -83,29 +88,33 @@ class Gel::LockedStore
   end
 
   def gems_for_lib(file)
-    if c = @lib_cache.fetch(file, nil)
-      c.each { |gem, subdir| yield gem, subdir }
-      return
-    end
+    search_name, search_ext = Gel::Util.split_filename_for_require(file)
 
-    hits = []
-    unless @full_cache
-      @inner.gems_for_lib(file) do |gem, subdir|
-        if locked?(gem)
-          hits << [gem, subdir]
-          yield gem, subdir
+    unless hits = @lib_cache.fetch(search_name, nil)
+      hits = []
+
+      unless @full_cache
+        @inner.gems_for_lib(search_name) do |gem, subdir, ext|
+          if locked?(gem)
+            hits << [gem, subdir, ext]
+          end
         end
       end
-    end
 
-    locked_gems.each do |gem|
-      if File.exist?(gem.path(file) + ".rb")
-        hits << [gem, nil]
-        yield gem, nil
+      locked_gems.each do |gem|
+        gem.entries_for_lib(search_name) do |subdir, ext|
+          hits << [gem, subdir, ext]
+        end
       end
+
+      @lib_cache[file] = hits
     end
 
-    @lib_cache[file] = hits
+    hits.each do |gem, subdir, ext|
+      if Gel::Util.ext_matches_requested?(ext, search_ext)
+        yield gem, subdir, ext
+      end
+    end
   end
 
   def each(gem_name = nil)

data/lib/gel/multi_store.rb

@@ -10,12 +10,21 @@ class Gel::MultiStore
   attr_reader :monitor
 
   def initialize(root, stores)
-    @root = File.realpath(File.expand_path(root)) if root
+    @root = root && File.realpath(File.expand_path(root))
     @stores = stores
 
     @monitor = Monitor.new
   end
 
+  def marshal_dump
+    [@root, @stores]
+  end
+
+  def marshal_load(v)
+    @root, @stores = v
+    @monitor = Monitor.new
+  end
+
   def stub_set
     @stub_set ||= Gel::StubSet.new(@root)
   end
@@ -47,8 +56,8 @@ class Gel::MultiStore
 
   def gems_for_lib(file)
     @stores.each do |_, store|
-      store.gems_for_lib(file) do |gem, subdir|
-        yield gem, subdir
+      store.gems_for_lib(file) do |gem, subdir, ext|
+        yield gem, subdir, ext
       end
     end
   end
@@ -73,7 +82,7 @@ class Gel::MultiStore
     result = {}
 
     @stores.each do |_, store|
-      result.update(store.gems(name_version_pairs)) do |l, r|
+      result.update(store.gems(name_version_pairs)) do |_, l, r|
         l
       end
     end

data/lib/gel/null_solver.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class Gel::NullSolver
+  NullPackage = Struct.new(:name, :platform)
+
+  def initialize(gemfile:, catalog_set:, platforms:)
+    @gemfile = gemfile
+    @catalog_set = catalog_set
+    @platforms = platforms
+
+    @platform = @platforms.first
+
+    @packages = Hash.new { |h, k| h[k] = NullPackage.new(k, @platform) }
+
+    @solution = {}
+    @constraints = Hash.new { |h, k| h[k] = [] }
+
+    @gemfile.gems.each do |name, constraints, options|
+      raise "NullSolver can't apply platform constraints" if options[:platforms]
+      @constraints[name].concat(constraints || [])
+    end
+  end
+
+  def solved?
+    gems_to_solve.empty?
+  end
+
+  def work
+    name = gems_to_solve.first
+
+    req = Gel::Support::GemRequirement.new(@constraints[name])
+
+    choice =
+      @catalog_set.entries_for(@packages[name]).map(&:gem_version).
+      sort_by { |v| [v.prerelease? ? 0 : 1, v] }.
+      reverse.find { |v| req.satisfied_by?(v) }
+
+    if choice.nil?
+      raise "Failed to resolve #{name.inspect} (#{req.inspect}) given #{@solution.inspect}"
+    end
+
+    @solution[name] = choice
+
+    @catalog_set.dependencies_for(@packages[name], choice).each do |dep_name, constraints|
+      if @solution[dep_name]
+        new_req = Gel::Support::GemRequirement.new(constraints)
+        unless new_req.satisfied_by?(@solution[dep_name])
+          raise "Already chose #{dep_name.inspect} #{@solution[dep_name]}, which is incompatible with #{name.inspect} #{choice.inspect} (wants #{new_req})"
+        end
+      else
+        @constraints[name].concat(constraints || [])
+      end
+    end
+  end
+
+  def each_resolved_package
+    @solution.each do |name, version|
+      yield @packages[name], version
+    end
+  end
+
+  private
+
+  def gems_to_solve
+    @constraints.keys - @solution.keys
+  end
+end