prebundler 0.11.6 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdf192e38f6f57692d64316b8b3d2ae645ee28a482c47e6e2bf1f14adfa283bc
-  data.tar.gz: 66c8f18686a990bcc1b222e4c76c6c7069793a92cd2f4bc1f102bcd2ed978b7c
+  metadata.gz: f1b4617c0c426fe9ac76ca98f6d9a80e849a6f8fb848817f7bfa6a9c5393b238
+  data.tar.gz: be95fbc3f0f4f39682c2ee929d9a3c8ea27fd8db087f4a0d3d9c2c39f87c4b98
 SHA512:
-  metadata.gz: 446f73025c6af47a142eba416e109805977fba9486a1120c6b726c20b3edddfd747ac459206b7780b75218a9abf6507ee502d50d6a4040b6de40769259af7f0b
-  data.tar.gz: 72e86b825eae349a11a6b606faba2cc6280a3b125415bcf1dd2df28f80dbc3312dfeb56de4fceafc4bbe0d017528aab45710857f912c5934f7c228e4f4e049da
+  metadata.gz: 244764066b66efdc4feb132dc3f94ed455cf141fdeb14b758d935e6dd1d8b960f3bdef4d2c783709a41fff7e30e6405e4871514d4220f757181af4e0951e72fe
+  data.tar.gz: 67d0a49d079f337e6522ed6765652f12e3fcd93546744f3ed0efae8e554e9f6604f8957118293c4d826985445048b7b65618cfbeb01a9af7b0a56b64a2cfa118

data/CHANGELOG.md

@@ -1,98 +1,97 @@
-0.11.6
-===
-- Fix bug causing native extension compile errors.
-- Fix bug causing executables to not be included in tarballs.
-- Fix bug (maybe introduced by bundler 2?) causing incorrect directory to be tarred. Directory can now include platform apparently.
-
-0.11.5
-===
-- Add `--retry` flag to CLI (currently does nothing).
-
-0.11.4
-===
-- Ensure .bundle/config directory exists before writing to it.
-
-0.11.3
-===
-- Support (well, add stubs for) `ruby` and `git_source` methods in Gemfiles.
-- Don't attempt to install gems we can't get a spec for.
-
-0.11.2
-===
-- Always run `bundle install` just in case.
-- Make sure `bundle check` is the _last_ thing that runs.
-
-0.11.1
-===
-- Exit with nonzero status code if fallback `bundle install` fails.
-
-0.11.0
-===
-- Allow the caller to pass in a s3 client for non-standard setups
-
-0.10.0
-===
-- Update aws-sdk client creation to be able to support non-aws s3 api endpoints (e.g. minio)
-
-0.9.1
-===
-- Woops, also use platform version when determining the gems that have already been built.
-- Fix the subsetter so it outputs gems inside correct source blocks.
-
-0.9.0
-===
-- Include platform version when uploading gem tarballs to the storage backend (this will cause bundles installed by previous versions of prebundler to be rebuilt).
-
-0.8.1
-===
-- Fix bug causing config to not be loaded.
-
-0.8.0
-===
-- Add the subset command for generating subsets of gemfiles.
-
-0.7.2
-===
-- Fix use of continuation token when listing remote files in S3 backend.
-
-0.7.1
-===
-- Support git-based gems with non-standard repo names.
-- Fix `bundle install` fallback (group args were wrong).
-
-0.7.0
-===
-- Add prefix option to install command.
-
-0.6.2
-===
-- Provide CLI option to generate binstubs.
-
-0.5.2
-===
-- Set `BUNDLE_GEMFILE` during prepare step so bundler doesn't complain when we try to call `Bundler.app_config_path`.
-
-0.5.1
-===
-- Remove pry-byebug require.
-
-0.5.0
-===
-- Works for a repository with a significant number of dependencies (~ 400).
-- Testing on staging server indicates bundle is installed correctly.
-
-0.0.4
-===
-- Pass bundle path to `gem install`.
-
-0.0.3
-===
-- Add --with and --without flags to install command.
-
-0.0.2
-===
-- Better CLI interface.
-
-0.0.1
-===
-- Birthday!
+## 0.13.0
+* Support the `eval_gemfile` function in gemfiles.
+* Avoid shelling out to the `tar` command.
+  - The GNU and BSD versions annoyingly don't accept the same flags, meaning Prebundler can succeed or fail depending on the system it's run on.
+  - All tar files are now read and written using pure Ruby.
+* Upgrade S3 client to non-EOL version.
+* Stop ignoring gems that don't match the current platform.
+  - Prebundler shells out to `gem install` when installing individual gems. Older versions of rubygems would fetch gems for the "ruby" platform when the `--ignore-dependencies` option was given, ignoring any platform-specific gems. This resulted in either a) unnecessarily building a bunch of native extensions, or b) installing a gem for the wrong platform (i.e. when no gem existed for the "ruby" platform, eg. helm-rb). I addressed the problem by instructing Prebundler to ignore gems with native extensions, relying on `bundle install` to fix them up. However, the bug has been fixed in modern versions of rubygems, so we can stop ignoring gems.
+* Run `bundle lock` before storing or installing gems to make sure the lockfile matches the Gemfile.
+
+## 0.12.0
+* Switch out ohai for ohey, which has many fewer dependencies.
+
+## 0.11.8
+* Don't store gems in the backend if they failed to install.
+* Use an absolute bundle path.
+* Only consider a gem from the lockfile if it matches the current platform.
+* Fix `#install` methods so they all return true/false.
+
+## 0.11.7
+* Fix bug causing platform-specific gems to be installed from source even if they were already present in the backend.
+
+## 0.11.6
+* Fix bug causing native extension compile errors.
+* Fix bug causing executables to not be included in tarballs.
+* Fix bug (maybe introduced by bundler 2?) causing incorrect directory to be tarred. Directory can now include platform apparently.
+
+## 0.11.5
+* Add `--retry` flag to CLI (currently does nothing).
+
+## 0.11.4
+* Ensure .bundle/config directory exists before writing to it.
+
+## 0.11.3
+* Support (well, add stubs for) `ruby` and `git_source` methods in Gemfiles.
+* Don't attempt to install gems we can't get a spec for.
+
+## 0.11.2
+* Always run `bundle install` just in case.
+* Make sure `bundle check` is the _last_ thing that runs.
+
+## 0.11.1
+* Exit with nonzero status code if fallback `bundle install` fails.
+
+## 0.11.0
+* Allow the caller to pass in a s3 client for non-standard setups
+
+## 0.10.0
+* Update aws-sdk client creation to be able to support non-aws s3 api endpoints (e.g. minio)
+
+## 0.9.1
+* Woops, also use platform version when determining the gems that have already been built.
+* Fix the subsetter so it outputs gems inside correct source blocks.
+
+## 0.9.0
+* Include platform version when uploading gem tarballs to the storage backend (this will cause bundles installed by previous versions of prebundler to be rebuilt).
+
+## 0.8.1
+* Fix bug causing config to not be loaded.
+
+## 0.8.0
+* Add the subset command for generating subsets of gemfiles.
+
+## 0.7.2
+* Fix use of continuation token when listing remote files in S3 backend.
+
+## 0.7.1
+* Support git-based gems with non-standard repo names.
+* Fix `bundle install` fallback (group args were wrong).
+
+## 0.7.0
+* Add prefix option to install command.
+
+## 0.6.2
+* Provide CLI option to generate binstubs.
+
+## 0.5.2
+* Set `BUNDLE_GEMFILE` during prepare step so bundler doesn't complain when we try to call `Bundler.app_config_path`.
+
+## 0.5.1
+* Remove pry-byebug require.
+
+## 0.5.0
+* Works for a repository with a significant number of dependencies (~ 400).
+* Testing on staging server indicates bundle is installed correctly.
+
+## 0.0.4
+* Pass bundle path to `gem install`.
+
+## 0.0.3
+* Add --with and --without flags to install command.
+
+## 0.0.2
+* Better CLI interface.
+
+## 0.0.1
+* Birthday!

data/lib/prebundler/cli/install.rb

@@ -17,6 +17,7 @@ module Prebundler
 
     class Install < Base
       def run
+        lock
         prepare
         install
         update_bundle_config
@@ -30,6 +31,10 @@ module Prebundler
 
       private
 
+      def lock
+        system "bundle lock --gemfile #{gemfile_path}"
+      end
+
       def prepare
         FileUtils.mkdir_p(bundle_path)
         ENV['BUNDLE_GEMFILE'] = gemfile_path
@@ -102,8 +107,12 @@ module Prebundler
           FileUtils.rm(dest_file)
         else
           out.puts "Installing #{gem_ref.id} from source"
-          gem_ref.install
-          store_gem(gem_ref, dest_file) if gem_ref.storable?
+
+          if gem_ref.install
+            store_gem(gem_ref, dest_file) if gem_ref.storable?
+          else
+            out.puts "Failed to install #{gem_ref.id} from source"
+          end
         end
       end
 
@@ -179,7 +188,7 @@ module Prebundler
       end
 
       def bundle_path
-        options.fetch(:'bundle-path')
+        File.expand_path(options.fetch(:'bundle-path'))
       end
 
       def config

data/lib/prebundler/gem_ref.rb

@@ -1,3 +1,4 @@
+require 'rubygems/package'
 require 'fileutils'
 require 'set'
 
@@ -48,33 +49,62 @@ module Prebundler
     end
 
     def install
+      # NOTE: the --platform argument used to not work when --ignore-dependencies
+      # was specified, but has been fixed in modern versions of rubygems. See:
+      # https://github.com/rubygems/rubygems/pull/2446 for a very long rabbit hole.
       Bundler.with_unbundled_env do
-        system({ "GEM_HOME" => bundle_path }, "gem install -N --ignore-dependencies --source #{source} #{name} -v #{version}")
+        system(
+          { "GEM_HOME" => bundle_path },
+          'gem install -N --ignore-dependencies '\
+            "--source #{source} #{name} "\
+            "--version #{version} "\
+            "--platform #{Bundler.local_platform.to_s}"
+        )
       end
 
-      $?.exitstatus
-    end
-
-    def install_from_tar(tar_file)
-      system "tar -C #{bundle_path} -xf #{tar_file}"
       $?.exitstatus == 0
     end
 
-    def add_to_tar(tar_file)
-      tar_flags = File.exist?(tar_file) ? '-rf' : '-cf'
-
-      system "tar -C #{bundle_path} #{tar_flags} #{tar_file} #{relative_gem_dir}"
-
-      relative_gemspec_files.each do |relative_gemspec_file|
-        system "tar -C #{bundle_path} -rf #{tar_file} #{relative_gemspec_file}"
+    def install_from_tar(tar_file)
+      File.open(tar_file) do |f|
+        Gem::Package::TarReader.new(f) do |tar|
+          tar.each do |entry|
+            path = File.join(bundle_path, entry.full_name)
+
+            if entry.directory?
+              FileUtils.mkdir_p(path)
+            else
+              FileUtils.mkdir_p(File.dirname(path))
+              File.open(path, 'wb') do |new_file|
+                new_file.write(entry.read)
+              end
+            end
+          end
+        end
       end
 
-      if File.directory?(extension_dir)
-        system "tar -C #{bundle_path} -rf #{tar_file} #{relative_extension_dir}"
-      end
+      true
+    rescue => e
+      return false
+    end
 
-      executables.each do |executable|
-        system "tar -C #{bundle_path} -rf #{tar_file} #{File.join(relative_gem_dir, 'bin', executable)}"
+    def add_to_tar(tar_file)
+      File.open(tar_file, 'wb') do |f|
+        Gem::Package::TarWriter.new(f) do |tar_io|
+          Dir.chdir(bundle_path) do
+            add_files_to_tar(
+              tar_io, Dir.glob(File.join(relative_gem_dir, '**', '*'))
+            )
+
+            add_files_to_tar(tar_io, relative_gemspec_files)
+
+            if File.directory?(extension_dir)
+              add_files_to_tar(
+                tar_io, Dir.glob(File.join(relative_extension_dir, '**', '*'))
+              )
+            end
+          end
+        end
       end
     end
 
@@ -105,7 +135,13 @@ module Prebundler
     end
 
     def install_dir
-      File.join(install_path, id)
+      @install_dir ||= begin
+        base = File.join(install_path, id)
+
+        find_platform_dir(base) do |dir|
+          File.directory?(dir)
+        end
+      end
     end
 
     def extension_dir
@@ -119,14 +155,10 @@ module Prebundler
     def relative_gem_dir
       @relative_gem_dir ||= begin
         base = File.join('gems', id)
-        platform = Bundler.local_platform.to_a
 
-        platform.size.downto(0) do |i|
-          dir = [base, *platform[0...i]].join('-')
-          return dir if File.directory?(File.join(bundle_path, dir))
+        find_platform_dir(base) do |dir|
+          File.directory?(File.join(bundle_path, dir))
         end
-
-        base
       end
     end
 
@@ -140,5 +172,29 @@ module Prebundler
       file = File.join(Bundler.local_platform.to_s, Prebundler.platform_version, Gem.extension_api_version.to_s, "#{id}.tar")
       prefix && !prefix.empty? ? File.join(prefix, file) : file
     end
+
+    private
+
+    def add_files_to_tar(tar_io, files)
+      files.each do |file|
+        next if File.directory?(file)
+
+        mode = File.stat(file).mode
+        tar_io.add_file(file, mode) do |new_file|
+          new_file.write(File.binread(file))
+        end
+      end
+    end
+
+    def find_platform_dir(base)
+      platform = Bundler.local_platform.to_a
+
+      platform.size.downto(0) do |i|
+        dir = [base, *platform[0...i]].join('-')
+        return dir if yield(dir)
+      end
+
+      base
+    end
   end
 end

data/lib/prebundler/gemfile_interpreter.rb

@@ -73,5 +73,9 @@ module Prebundler
     def gemspec
       # do nothing
     end
+
+    def eval_gemfile(path)
+      instance_eval(File.read(path))
+    end
   end
 end

data/lib/prebundler/git_gem_ref.rb

@@ -22,15 +22,15 @@ module Prebundler
       FileUtils.mkdir_p(install_path)
       FileUtils.mkdir_p(cache_path)
 
-      return if File.exist?(cache_dir) || File.exist?(install_dir)
+      return true if File.exist?(cache_dir) || File.exist?(install_dir)
       system "git clone #{uri} \"#{cache_dir}\" --bare --no-hardlinks --quiet"
-      return $? if $?.exitstatus != 0
+      return false if $?.exitstatus != 0
       system "git clone --no-checkout --quiet \"#{cache_dir}\" \"#{install_dir}\""
-      return $? if $?.exitstatus != 0
+      return false if $?.exitstatus != 0
       Dir.chdir(install_dir) { system "git reset --hard --quiet #{revision}" }
       serialize_gemspecs
       copy_gemspecs
-      $?
+      $?.exitstatus == 0
     end
 
     def to_gem

data/lib/prebundler/s3_backend.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'aws-sdk'
+require 'aws-sdk-s3'
 
 module Prebundler
   class S3Backend

data/lib/prebundler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Prebundler
-  VERSION = '0.11.6'
+  VERSION = '0.13.0'
 end

data/lib/prebundler.rb

@@ -1,4 +1,4 @@
-require 'ohai'
+require 'ohey'
 
 module Prebundler
   autoload :Cli,                'prebundler/cli'
@@ -27,13 +27,13 @@ module Prebundler
     end
 
     def platform_version
-      @platform_version ||= "#{system_info['platform']}-#{system_info['platform_version']}"
+      @platform_version ||= "#{platform.name}-#{platform.version}"
     end
 
     private
 
-    def system_info
-      @@system_info ||= Ohai::System.new.all_plugins(['platform'])
+    def platform
+      @platform ||= Ohey.current_platform
     end
   end
 end

data/prebundler.gemspec

@@ -14,10 +14,10 @@ Gem::Specification.new do |s|
   s.add_dependency 'bundler'
   s.add_dependency 'parallel', '~> 1.0'
   s.add_dependency 'gli', '~> 2.0'
-  s.add_dependency 'ohai', '~> 14.0'
+  s.add_dependency 'ohey', '~> 1.0'
 
   # @TODO: move s3 support into separate gem
-  s.add_dependency 'aws-sdk', '~> 2.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1.0'
 
   s.executables << 'prebundle'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prebundler
 version: !ruby/object:Gem::Version
-  version: 0.11.6
+  version: 0.13.0
 platform: ruby
 authors:
 - Cameron Dutro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-15 00:00:00.000000000 Z
+date: 2022-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,33 +53,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: ohai
+  name: ohey
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '14.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '14.0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
 description: Gem dependency prebuilder
 email:
 - camertron@gmail.com
@@ -129,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Gem dependency prebuilder