prebundler 0.0.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14809b2ea93fc2442016314d9f173db3be3e383f
-  data.tar.gz: 4d6c78d6743dc6584adc689ee4f4b3b222084578
+  metadata.gz: a8f3573257ed6d3ae187bd8cc9b3e37ce6787373
+  data.tar.gz: 7780812efb908e7e533e1e7a3299b529f281bda5
 SHA512:
-  metadata.gz: 957cbcdff72652ecdc09a8e29ec6725a5dfe05efef3ee538f21c266dc6a65bd42a2a602079abeeaa6340f0910923c724d43e538ef46ca8f3efc6538be18cda07
-  data.tar.gz: 288c16b848d8909554198f7bee6fff33f67776303fc7d4890650d9e83326cc5da084342e53434473b702de9094b4a459f6678990486b615a1156d76f81d3bf4f
+  metadata.gz: 51e0f4e6e61305cd9535de52ca2bd3366694f63b99a0ea43d20ac084571815be95683a035698983ea0aa7eaeb4f14bf903f97aecb7b32988df1d4d296ba0834d
+  data.tar.gz: b1890c8532ddce82b3da898d5aae18d1c5da929b328fd460df090949310d5c70ec2d3cd3f7c98bdb1b2315fd44875e60b4594e2f91de7a87c115bb94683edbab

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.5.0
+===
+- Works for a repository with a significant number of dependencies (~ 400).
+- Testing on staging server indicates bundle is installed correctly.
+
 0.0.4
 ===
 - Pass bundle path to `gem install`.

data/lib/prebundler/cli/install.rb

@@ -1,7 +1,8 @@
 require 'securerandom'
 require 'tmpdir'
 require 'parallel'
-require 'bundler'
+require 'fileutils'
+require 'yaml'
 
 module Prebundler
   module Cli
@@ -9,15 +10,34 @@ module Prebundler
       def run
         prepare
         install
+        update_bundle_config
+        generate_binstubs
         check
       end
 
       private
 
       def prepare
-        gem_list.each do |_, gem_ref|
+        FileUtils.mkdir_p(bundle_path)
+
+        gem_list.each do |name, gem_ref|
           next if backend_file_list.include?(gem_ref.tar_file)
           next unless member_of_installable_group?(gem_ref)
+
+          # Edge case: installation could fail if dependencies of this
+          # gem ref haven't been installed yet. They should have been
+          # prepared by this point because of the tsort logic in the
+          # gemfile class, but we need to actually install them in
+          # order to install this current gem. A good example is nokogiri,
+          # which can't build its native extension without mini-portile2.
+          gem_ref.dependencies.each do |dep|
+            if gem_list.gems[dep]
+              install_gem_ref(gem_list.gems[dep])
+            else
+              out.puts "Oops, couldn't find dependency #{dep}"
+            end
+          end
+
           install_gem(gem_ref) if gem_ref.installable?
         end
       end
@@ -79,17 +99,51 @@ module Prebundler
         config.storage_backend.store_file(dest_file, gem_ref.tar_file)
       end
 
+      def update_bundle_config
+        file = Bundler.app_config_path.join('config').to_s
+        config = File.exist?(file) ? YAML.load_file(file) : {}
+        config['BUNDLE_WITH'] = with_groups.join(':') unless with_groups.empty?
+        config['BUNDLE_WITHOUT'] = without_groups.join(':') unless without_groups.empty?
+        File.write(file, YAML.dump(config))
+      end
+
+      def generate_binstubs
+        out.puts 'Generating binstubs...'
+
+        gems_with_executables = gem_list.gems.values.select do |gem_ref|
+          next false unless member_of_installable_group?(gem_ref)
+          !gem_ref.executables.empty?
+        end
+
+        return if gems_with_executables.empty?
+
+        system "bundle binstubs #{gems_with_executables.map(&:name).join(' ')}"
+        system "bundle binstubs --force bundler"
+        out.puts 'Done generating binstubs'
+      end
+
       def check
-        system 'bundle check'
+        system "bundle check --gemfile #{gemfile_path}"
 
         if $?.exitstatus != 0
           out.puts 'Bundle not satisfied, falling back to `bundle install`'
-          system 'bundle install'
+          system "bundle install #{bundle_install_args}"
         end
       end
 
+      def bundle_install_args
+        [].tap do |args|
+          args << "--gemfile #{gemfile_path}"
+          args << "--with #{with_groups}" unless with_groups.empty?
+          args << "--without #{without_groups}" unless without_groups.empty?
+          args << "--jobs #{options[:jobs]}"
+        end.join(' ')
+      end
+
       def gem_list
-        @gem_list ||= Prebundler::GemfileInterpreter.interpret(gemfile_path, bundle_path)
+        @gem_list ||= Prebundler::GemfileInterpreter.interpret(
+          gemfile_path, bundle_path
+        )
       end
 
       def backend_file_list
@@ -115,14 +169,18 @@ module Prebundler
 
       def groups
         @groups ||= begin
-          all_groups = gem_list.flat_map { |_, gem_ref| gem_ref.groups }.uniq
-          with_groups = (options[:with] || '').split(',').map { |g| g.strip.to_sym }
-          without_groups = (options[:without] || '').split(',').map { |g| g.strip.to_sym }
-
-          groups = with_groups.empty? ? all_groups : with_groups
-          groups - without_groups
+          all_groups = gem_list.flat_map { |_, gem_ref| gem_ref.groups.to_a }.uniq
+          (all_groups + with_groups).uniq - without_groups
         end
       end
+
+      def with_groups
+        @with_groups ||= (options[:with] || '').split(/[:, ]/).map { |g| g.strip.to_sym }
+      end
+
+      def without_groups
+        @without_groups ||= (options[:without] || '').split(/[:, ]/).map { |g| g.strip.to_sym }
+      end
     end
   end
 end

data/lib/prebundler/gem_ref.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'set'
 
 module Prebundler
   class GemRef
@@ -12,13 +13,13 @@ module Prebundler
       end
     end
 
-    attr_reader :name, :bundle_path
+    attr_reader :name, :bundle_path, :groups
     attr_accessor :spec, :dependencies
 
     def initialize(name, bundle_path, options = {})
       @name = name
       @bundle_path = bundle_path
-      @groups = options[:groups]
+      @groups = Set.new(options[:groups])
       @source = options[:source]
       @dependencies = options[:dependencies]
     end
@@ -27,10 +28,6 @@ module Prebundler
       @dependencies ||= []
     end
 
-    def groups
-      @groups ||= []
-    end
-
     def source
       @source ||= DEFAULT_SOURCE
     end
@@ -44,8 +41,8 @@ module Prebundler
     end
 
     def install
-      system "gem install -N -i #{bundle_path} --ignore-dependencies --source #{source} #{name} -v #{version}"
-      $?.exitstatus == 0
+      system({ "GEM_HOME" => bundle_path }, "gem install -N --ignore-dependencies --source #{source} #{name} -v #{version}")
+      $?.exitstatus
     end
 
     def install_from_tar(tar_file)
@@ -57,11 +54,28 @@ module Prebundler
       tar_flags = File.exist?(tar_file) ? '-rf' : '-cf'
 
       system "tar -C #{bundle_path} #{tar_flags} #{tar_file} #{relative_gem_dir}"
-      system "tar -C #{bundle_path} -rf #{tar_file} #{relative_gemspec_dir}"
+
+      relative_gemspec_files.each do |relative_gemspec_file|
+        system "tar -C #{bundle_path} -rf #{tar_file} #{relative_gemspec_file}"
+      end
 
       if File.directory?(extension_dir)
         system "tar -C #{bundle_path} -rf #{tar_file} #{relative_extension_dir}"
       end
+
+      executables.each do |executable|
+        system "tar -C #{bundle_path} -rf #{tar_file} #{File.join('bin', executable)}"
+      end
+    end
+
+    def executables
+      gemspecs.flat_map(&:executables)
+    end
+
+    def gemspecs
+      @gemspecs ||= relative_gemspec_files.map do |relative_gemspec_file|
+        Bundler.load_gemspec(File.join(bundle_path, relative_gemspec_file))
+      end
     end
 
     def installable?
@@ -96,8 +110,8 @@ module Prebundler
       File.join('gems', id)
     end
 
-    def relative_gemspec_dir
-      File.join('specifications', gemspec_file)
+    def relative_gemspec_files
+      [File.join('specifications', gemspec_file)]
     end
 
     def tar_file

data/lib/prebundler/gemfile.rb

@@ -9,6 +9,10 @@ module Prebundler
 
     def initialize(gems)
       @gems = gems
+
+      gems.each do |_, gem_ref|
+        assign_groups(gem_ref, [])
+      end
     end
 
     def each
@@ -23,6 +27,16 @@ module Prebundler
 
     private
 
+    # propagate groups to dependencies
+    def assign_groups(gem_ref, seen)
+      gem_ref.dependencies.each do |dep|
+        next if seen.include?(dep)
+        next unless gems[dep]
+        gem_ref.groups.each { |group| gems[dep].groups << group }
+        assign_groups(gems[dep], seen + [dep])
+      end
+    end
+
     def tsort_each_node(&block)
       gems.keys.each(&block)
     end

data/lib/prebundler/gemfile_interpreter.rb

@@ -1,18 +1,21 @@
+require 'set'
+
 module Prebundler
   class GemfileInterpreter
-    def self.interpret(path, bundle_path)
-      Gemfile.new(new(path, bundle_path).gems)
+    def self.interpret(gemfile_path, bundle_path)
+      Gemfile.new(new(gemfile_path, bundle_path).gems)
     end
 
-    attr_reader :gems, :bundle_path
+    attr_reader :gems, :gemfile_path, :bundle_path
 
-    def initialize(path, bundle_path)
+    def initialize(gemfile_path, bundle_path)
       @gems = {}
-      @current_groups = []
+      @current_groups = [:global]
+      @gemfile_path = gemfile_path
       @bundle_path = bundle_path
-      instance_eval(File.read(path))
+      instance_eval(File.read(gemfile_path))
 
-      lockfile = Bundler::LockfileParser.new(File.read("#{path}.lock"))
+      lockfile = Bundler::LockfileParser.new(File.read("#{gemfile_path}.lock"))
 
       lockfile.specs.each do |spec|
         gems[spec.name] ||= GemRef.create(spec.name, bundle_path)
@@ -35,7 +38,7 @@ module Prebundler
     end
 
     def path(dir)
-      @current_path = dir
+      @current_path = File.join(File.dirname(gemfile_path), dir)
       yield if block_given?
       @current_path = nil
     end
@@ -49,7 +52,7 @@ module Prebundler
     def group(*groups)
       @current_groups = groups
       yield if block_given?
-      @current_groups = []
+      @current_groups = [:global]
     end
 
     def gemspec

data/lib/prebundler/git_gem_ref.rb

@@ -15,10 +15,13 @@ module Prebundler
     def initialize(name, bundle_path, options = {})
       super
       @strategy = options.include?(:git) ? :git : :github
-      @uri = options[@strategy]
     end
 
     def install
+      FileUtils.mkdir_p(spec_path)
+      FileUtils.mkdir_p(install_path)
+      FileUtils.mkdir_p(cache_path)
+
       return if File.exist?(cache_dir) || File.exist?(install_dir)
       system "git clone #{uri} \"#{cache_dir}\" --bare --no-hardlinks --quiet"
       return $? if $?.exitstatus != 0
@@ -55,11 +58,7 @@ module Prebundler
     end
 
     def uri
-      if strategy == :github
-        "git://github.com/#{@uri}.git"
-      else
-        @uri
-      end
+      spec.source.uri
     end
 
     alias_method :source, :uri
@@ -68,16 +67,26 @@ module Prebundler
       spec.source.revision
     end
 
+    def gemspecs
+      @gemspecs ||= gemspec_files.map do |gemspec_file|
+        Bundler.load_gemspec(gemspec_file)
+      end
+    end
+
     private
 
+    def gemspec_files
+      @gemspec_files ||= Dir[File.join(install_dir, '*.gemspec')]
+    end
+
     def copy_gemspecs
-      FileUtils.cp(Dir[File.join(install_dir, '*.gemspec')], spec_path)
+      FileUtils.cp(gemspec_files, spec_path)
     end
 
     # adapted from
     # https://github.com/bundler/bundler/blob/fea23637886c1b1bde471c98344b8844f82e60ce/lib/bundler/source/git.rb#L237
     def serialize_gemspecs
-      Dir[File.join(install_dir, '*.gemspec')].each do |path|
+      gemspec_files.each do |path|
         # Evaluate gemspecs and cache the result. Gemspecs
         # in git might require git or other dependencies.
         # The gemspecs we cache should already be evaluated.
@@ -100,7 +109,7 @@ module Prebundler
         # If there is no URI scheme, assume it is an ssh/git URI
         input = uri
       end
-      Digest::SHA1.hexdigest(input)
+      Bundler::SharedHelpers.digest(:SHA1).hexdigest(input)
     end
   end
 end

data/lib/prebundler/path_gem_ref.rb

@@ -10,7 +10,11 @@ module Prebundler
 
     def initialize(name, bundle_path, options = {})
       super
-      @path = options[:path]
+
+      # @TODO: Individual gem calls can also specify a path, which
+      # we don't currently handle. For now just use the gem's name
+      # to complete the path.
+      @path = File.join(options[:path], name)
     end
 
     def installable?
@@ -21,6 +25,20 @@ module Prebundler
       false
     end
 
+    def gemspecs
+      @gemspecs ||= gemspec_files.map do |gemspec_file|
+        Dir.chdir(File.dirname(gemspec_file)) do
+          Bundler.load_gemspec(File.basename(gemspec_file))
+        end
+      end
+    end
+
     alias_method :source, :path
+
+    private
+
+    def gemspec_files
+      @gemspec_files ||= Dir[File.join(path, '*.gemspec')]
+    end
   end
 end

data/lib/prebundler/version.rb

@@ -1,3 +1,3 @@
 module Prebundler
-  VERSION = '0.0.4'
+  VERSION = '0.5.0'
 end

data/prebundler.gemspec

@@ -17,9 +17,8 @@ Gem::Specification.new do |s|
   s.add_dependency 'parallel', '~> 1.0'
   s.add_dependency 'gli', '~> 2.0'
 
-  # @TODO: remove these, maybe move s3 support into separate gem?
+  # @TODO: move s3 support into separate gem
   s.add_dependency 'aws-sdk', '~> 2.0'
-  s.add_dependency 'pry-byebug'
 
   s.executables << 'prebundle'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prebundler
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Cameron Dutro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-29 00:00:00.000000000 Z
+date: 2017-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Gem dependency prebuilder
 email:
 - camertron@gmail.com