rubygems-precompiled 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 437a696f3109133c285f30281afb05a3a774ab5c
+  data.tar.gz: 1068539b91ba04f83dbd865c24aacbb048a0af00
+SHA512:
+  metadata.gz: 72e124fac7166f9528fd155452d89d8aa263db8dd8a859113af2e4c874124b0758afd1ec15a1be0f06c2c675481f5e23d0baf478e5b4960863f7c5807aa70537
+  data.tar.gz: 40ff597a7dbd0190c05aac335ad6cda38c6bf2b01a69d44568a69a1737a70913e88fbeb2123584a92a6e6ed338be3a6fc2188c7c233c7fb1fcbdd505a5357157

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+scratch/*

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rubygems-precompiled.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Thomas Haggett
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Rubygems::Precompiled
+
+This gem allows you to build the c-extensions of a ruby-gem on a host with build-tools installed, make the result available over HTTP, then use this on end machines that may not have build-tools installed.
+
+## Installation
+
+    $ gem install rubygems-precompiled
+
+## Usage
+
+There are two halves to this, the pre-compile on a build machine and the install
+on machines that don't have the build-tools available.
+
+### Build host
+
+Fetch a gem you're interested in:
+
+    gem fetch foobar -v '0.1.0'
+
+Pre-compile the gem into the correct folder structure:
+
+    gem precompile -a -o /output foobar-0.1.0.gem
+
+Which will write the file:
+
+    /output/ruby-1.9.3p448/x86_64-linux/foobar-0.1.0.tar.gz
+
+You need to make this directory structure available over HTTP, and available to the installation machines.
+
+### Installation host
+
+Configure the cache path (file:/// and http:// urls are supported at the moment) in `/etc/gemrc`:
+
+    precompiled_cache:
+     - http://some.server/ruby-gem-extensions/
+
+Install a gem using the cache:
+
+    gem install foobar -v '0.1.0'
+
+
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/features/installing_with_cache.feature

@@ -0,0 +1,47 @@
+Feature: Installing using a compiled-cache
+
+  Background:
+    Given I have wiped the folder "/tmp/precompiled-workroot"
+
+#  Scenario: Installing a compiled gem without the cache
+
+  Scenario: Installing a compiled gem with a cache miss
+    Given I use the gem configuration option
+      """
+      precompiled_cache:
+       - file:///tmp/precompiled-workroot/cache
+      """
+
+    When I execute "gem install --install-dir /tmp/precompiled-workroot/installroot spec/fixtures/compiled-gem.gem"
+    Then I should see "Building native extensions"
+    Then I should not see "Loading native extension from cache"
+
+    When I execute
+      """
+      echo "puts CompiledClass.new.test_method" | ruby -I/tmp/precompiled-workroot/installroot/gems/compiled-gem-0.0.1/lib -rtest_ext/test_ext
+      """
+
+    Then I should see "Hello, world!"
+
+  Scenario: Installing a compiled gem with a cache hit
+    Given I use the gem configuration option
+      """
+      precompiled_cache:
+       - file:///tmp/precompiled-workroot/cache
+       """
+
+    When I execute "gem precompile -o /tmp/precompiled-workroot/cache -a spec/fixtures/compiled-gem.gem"
+
+    And I execute "gem install --install-dir /tmp/precompiled-workroot/installroot spec/fixtures/compiled-gem.gem"
+
+    Then I should not see "Building native extensions"
+    Then I should see "Loading native extension from cache"
+
+    When I execute
+      """
+      echo "puts CompiledClass.new.test_method" | ruby -I/tmp/precompiled-workroot/installroot/gems/compiled-gem-0.0.1/lib -rtest_ext/test_ext
+      """
+
+    Then I should see "Hello, world!"
+
+

data/features/precompiling_gems.feature

@@ -0,0 +1,46 @@
+Feature: Pre-compiling gems
+
+  Background:
+    Given I have changed to a temporary directory containing "spec/fixtures/*.gem"
+
+  Scenario: Input validation
+    When I run the command "gem precompile"
+    Then I should see "Please specify a gem file on the command line, e.g. gem precompile foo-0.1.0.gem"
+    And the command should not return a success status code
+
+  Scenario: Pre-compiling a single gem
+    When I run the command "gem precompile simple-gem.gem"
+
+    Then I should see "The gem 'simple-gem' doesn't contain a compiled extension"
+    And the command should return a success status code
+
+  Scenario: Pre-compiling a single compiled gem
+    When I run the command "gem precompile compiled-gem.gem"
+
+    Then I should see "Compiling 'compiled-gem'..."
+    And the command should return a success status code
+
+  Scenario: Pre-compiling multiple gems
+    When I run the command "gem precompile *.gem"
+
+    Then I should see "The gem 'simple-gem' doesn't contain a compiled extension"
+    And I should see "Compiling 'compiled-gem'... done."
+    And the command should return a success status code
+
+  Scenario: Creating the flat output files
+    When I run the command "gem precompile *.gem"
+
+    Then the file "compiled-gem-0.0.1.tar.gz" should exist
+    And the file "simple-gem*.tar.gz" should not exist
+
+  Scenario: Writing to specific folder
+    When I run the command "gem precompile -o foo *.gem"
+
+    Then the folder "foo" should exist
+    And the file "foo/compiled-gem-0.0.1.tar.gz" should exist
+
+  Scenario: Creating architecture files
+    When I run the command "gem precompile -o foo -a *.gem"
+
+    Then the file "foo/ruby-*/x86_64-*/compiled-gem-0.0.1.tar.gz" should exist
+

data/features/steps/command_steps.rb

@@ -0,0 +1,64 @@
+require "tmpdir"
+require 'fileutils'
+require 'tempfile'
+
+Given /^I use the gem configuration option$/ do |string|
+  File.open(File.expand_path("~/.gemrc"), "a") do |gemconfig|
+    gemconfig.puts("# --PURGE FROM HERE ---")
+    gemconfig.write string
+  end
+
+end
+After do
+  original, _ = File.read(File.expand_path("~/.gemrc")).split("# --PURGE FROM HERE ---")
+  File.open(File.expand_path("~/.gemrc"), "w") do |gemconfig|
+    gemconfig.write original
+  end
+end
+
+def execute(command)
+    @last_stderr, @last_stdout = "", ""
+
+  stderr_r, stderr_w, stdout_r, stdout_w = [IO.pipe, IO.pipe].flatten
+  @command_env ||= {}
+
+  pid = spawn( @command_env, command, :out => stdout_w, :err => stderr_w)
+  _, status = Process.waitpid2(pid)
+
+  [stderr_w,stdout_w].each { |p| p.close }
+
+  @last_status = status.exitstatus
+  @last_stderr += stderr_r.read until stderr_r.eof?
+  @last_stdout += stdout_r.read until stdout_r.eof?
+end
+
+When /^I (?:run the command|execute) "(.*?)"$/ do |command|
+  execute(command)
+end
+When /^I (?:run the command|execute)$/ do |command|
+    execute(command)
+end
+
+Then /^I should( not)? see "(.*?)"( on (stdout|stderr))?$/ do |invert, expect, any, channel|
+  data = if channel.nil?
+    @last_stdout + @last_stderr
+  elsif channel == 'stdout'
+    @last_stdout
+  elsif channel == 'stderr'
+    @last_stderr
+  end
+
+  if invert
+    data.should_not include(expect)
+  else
+    data.should include(expect)
+  end
+end
+
+Then /^the command should( not)? return a success status code$/ do |invert|
+  if invert
+    @last_status.should_not == 0
+  else
+    @last_status.should == 0
+  end
+end

data/features/steps/file_steps.rb

@@ -0,0 +1,37 @@
+
+Given /^I have wiped the folder "(.*?)"$/ do |path|
+  FileUtils.rm_rf(path)
+end
+
+
+OriginalWorkingDirectory = Dir.pwd
+Before do
+  FileUtils.chdir(OriginalWorkingDirectory)
+end
+cleanup = []
+Given /^I have changed to a temporary directory(?: containing "(.*?)")?$/ do |glob|
+  directory = Dir.mktmpdir
+  cleanup << directory
+  if glob
+    files = Dir.glob(glob)
+    files.each do |file|
+      FileUtils.cp(file, File.join(directory, File.basename(file)))
+    end
+  end
+  FileUtils.chdir(directory)
+end
+After do
+  cleanup.each { |dir| FileUtils.rm_rf(dir) }
+end
+
+Then /^the folder "(.*?)" should exist$/ do |folder|
+  File.directory?(folder).should be_true
+end
+
+Then /^the file "(.*?)" should (not )?exist$/ do |file, invert|
+  if invert
+    Dir.glob(file).should be_empty
+  else
+    Dir.glob(file).should_not be_empty
+  end
+end

data/lib/rubygems/commands/precompile_command.rb

@@ -0,0 +1,45 @@
+require "rubygems/command"
+
+class Gem::Commands::PrecompileCommand < Gem::Command
+  def initialize
+    super "precompile", "Create a bundle containing the compiled artifacts for this platform", :output => Dir.pwd, :arch => false
+
+    add_option('-o PATH', '--output=PATH', 'The output directory for the generated bundle. Defaults to the current directory') do |path,options|
+      options[:output] = path
+    end
+
+    add_option('-a','--arch-dirs','Adds the architecture sub-folders to the output directory before writing') do |arch, options|
+      options[:arch] = true
+    end
+  end
+
+  def arguments
+    "GEMFILE       path to the gem file to compile"
+  end
+
+  def usage
+    "#{program_name} GEMFILE"
+  end
+
+  def execute
+    gemfiles = options[:args]
+
+    # no gem, no binary
+    if gemfiles.empty?
+      raise Gem::CommandLineError, "Please specify a gem file on the command line, e.g. #{program_name} foo-0.1.0.gem"
+    end
+
+    require "rubygems/precompiler"
+
+    gemfiles.each do |gemfile|
+      compiler = Gem::Precompiler.new(gemfile, options)
+      if compiler.has_extension?
+        $stderr.puts "Compiling '#{compiler.gem_name}'... "
+        compiler.compile
+        $stderr.puts "done."
+      else
+        $stderr.puts "The gem '#{compiler.gem_name}' doesn't contain a compiled extension"
+      end
+    end
+  end
+end

data/lib/rubygems/precompiled.rb

@@ -0,0 +1,118 @@
+require 'rubygems/precompiled/version'
+require 'rubygems/installer'
+require 'rubygems/ext/builder'
+require 'rubygems/package/tar_reader'
+require 'zlib'
+require 'fileutils'
+require 'net/http'
+require 'uri'
+require 'tempfile'
+
+class Gem::Installer
+
+  class BaseCache
+    def initialize(root_uri)
+      @root_uri = root_uri
+    end
+
+    def retrieve(spec)
+      raise "Must be overriden!"
+    end
+    def contains?(spec)
+      false
+    end
+    def cache_key(spec)
+      "/ruby-#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}/#{Gem::Platform.local.to_s}/#{spec.name}-#{spec.version}.tar.gz"
+    end
+  end
+
+  class FileCache < BaseCache
+    def path_for(spec)
+      File.join(@root_uri.path, cache_key(spec))
+    end
+    def contains?(spec)
+      File.exists?(path_for(spec))
+    end
+
+    def retrieve(spec)
+      yield path_for(spec)
+    end
+  end
+
+  class HttpCache < BaseCache
+    def uri_to_spec(spec)
+      URI.join(@root_uri, File.join(@root_uri.path, cache_key(spec)))
+    end
+    def contains?(spec)
+      uri = uri_to_spec(spec)
+      http = Net::HTTP.start(uri.host, uri.port)
+      http.head(uri.path).code == "200"
+    end
+    def retrieve(spec)
+      tempfile = Tempfile.new('cache-hit')
+      uri = uri_to_spec(spec)
+      http = Net::HTTP.start(uri.host, uri.port)
+      http.request_get(uri.path) do |resp|
+        resp.read_body do |segment|
+          tempfile.write(segment)
+        end
+        tempfile.close
+      end
+
+      yield tempfile
+      tempfile.delete
+    end
+  end
+
+  GemCache = {
+    'file' => FileCache,
+    'http' => HttpCache
+  }.freeze
+
+  # Private: A list of precompiled cache root URLs loaded from the rubyges configuration file
+  #
+  # Returns Array of BaseCache subclasses
+  def self.precompiled_caches
+    @@caches ||= [Gem.configuration['precompiled_cache']].flatten.compact.map do |cache_root|
+      cache_root = URI.parse(cache_root)
+      GemCache[cache_root.scheme].new(cache_root)
+    end
+  end
+
+  def build_extensions_with_cache
+    cache = Gem::Installer.precompiled_caches.find { |cache| cache.contains?(@spec) }
+
+    if cache
+      puts "Loading native extension from cache"
+      cache.retrieve(@spec) do |path|
+        overlay_tarball(path)
+      end
+    else
+      build_extensions_without_cache
+    end
+  end
+
+  alias_method :build_extensions_without_cache, :build_extensions
+  alias_method :build_extensions, :build_extensions_with_cache
+
+  # Private: Extracts a .tar.gz file on-top of the gem's installation directory
+  def overlay_tarball(tarball)
+    Zlib::GzipReader.open(tarball) do |gzip_io|
+      Gem::Package::TarReader.new(gzip_io) do |tar|
+        tar.each do |entry|
+          target_path = File.join(gem_dir, entry.full_name)
+          if entry.directory?
+            FileUtils.mkdir_p(target_path)
+          elsif entry.file?
+            FileUtils.mkdir_p(File.dirname(target_path))
+            File.open(target_path, "w") do |f|
+              f.write entry.read(1024*1024) until entry.eof?
+            end
+          end
+          entry.close
+        end
+      end
+    end
+  end
+
+end

data/lib/rubygems/precompiled/version.rb

@@ -0,0 +1,5 @@
+module Rubygems
+  module Precompiled
+    VERSION = "0.0.1"
+  end
+end

data/lib/rubygems/precompiler.rb

@@ -0,0 +1,108 @@
+require "rbconfig"
+require "tmpdir"
+require "rubygems/installer"
+require "fileutils"
+require 'rubygems/package/tar_writer'
+require 'zlib'
+require 'pathname'
+
+class Gem::Precompiler
+  include FileUtils
+
+  def initialize(gemfile, opts = {})
+    @installer = Gem::Installer.new(gemfile, opts.dup.merge(:unpack => true))
+    @target_dir = opts.fetch(:output, Dir.pwd)
+    @target_dir = File.join(@target_dir, arch_string) if opts.fetch(:arch, false)
+    @options = opts
+  end
+
+  # Public: Returns the name of hte gem
+  #
+  # Returns a string
+  def gem_name
+    @installer.spec.name
+  end
+
+  # Public: Does the gem actually have any compiled extensions?
+  #
+  # Returns boolean - true if the gem has a c-extension that needs building
+  def has_extension?
+    !@installer.spec.extensions.empty?
+  end
+
+  # Private: Yield the path to a temporary directory that will get deleted when
+  # the block returns
+  #
+  def tempdir
+    temp_dir = Dir.mktmpdir
+    yield temp_dir
+  ensure
+    rm_rf temp_dir
+  end
+
+  # Private: Return a string that uniquely keys this machines ruby version and architecture
+  #
+  # Returns string
+  def arch_string
+    "ruby-#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}/#{Gem::Platform.local.to_s}"
+  end
+
+  # Public: The filename of the compiled bundle for this gem
+  #
+  # Returns a string
+  def output_path
+    File.join(*[@target_dir, "#{@installer.spec.name}-#{@installer.spec.version}.tar.gz"].compact)
+  end
+
+  # Private: Return a list fo build-products in a given directory
+  #
+  # Returns an array of paths
+  def build_products(installer, path)
+    dlext = RbConfig::CONFIG["DLEXT"]
+    lib_dirs = installer.spec.require_paths.join(',')
+    Dir.glob("#{path}/{#{lib_dirs}}/**/*.#{dlext}")
+  end
+
+  # Public: Compile
+  #
+  #
+  def compile
+    FileUtils.mkdir_p(@target_dir)
+
+    tempdir do |path|
+      @installer.unpack(path)
+      @installer.build_extensions
+
+      targz_file(output_path) do |tar_writer|
+
+        build_products(@installer, path).each do |product_path|
+          product_path = Pathname.new(product_path)
+          relative_path = product_path.relative_path_from(Pathname.new(path))
+
+          stat = File.stat(product_path)
+          mode = stat.mode
+          size = stat.size
+
+          File.open(product_path, "r") do |source|
+
+            tar_writer.add_file_simple(relative_path.to_s, mode, size) do |dest|
+              dest.write source.read(1024*1024) until source.eof?
+            end
+          end
+
+        end
+
+      end
+    end
+  end
+
+  # Private: Yield a reference to a TarWriter that writes to
+  # the specified .tar.gz file
+  #
+  def targz_file(path, &block)
+    Zlib::GzipWriter.open(path) do |tar_file_io|
+      Gem::Package::TarWriter.new(tar_file_io, &block)
+    end
+  end
+
+end

data/lib/rubygems_plugin.rb

@@ -0,0 +1,4 @@
+require 'rubygems/precompiled'
+require 'rubygems/command_manager'
+
+Gem::CommandManager.instance.register_command :precompile

data/rubygems-precompiled.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rubygems/precompiled/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rubygems-precompiled"
+  spec.version       = Rubygems::Precompiled::VERSION
+  spec.authors       = ["Thomas Haggett"]
+  spec.email         = ["thomas-rubygemplugin@haggett.org"]
+  spec.description   = %q{RubyGems plugin to allow a gem's compiled extension to be pre-built and cached}
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/joshado/rubygems-precompiled"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "cucumber"
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: rubygems-precompiled
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Thomas Haggett
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: RubyGems plugin to allow a gem's compiled extension to be pre-built and
+  cached
+email:
+- thomas-rubygemplugin@haggett.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- features/installing_with_cache.feature
+- features/precompiling_gems.feature
+- features/steps/command_steps.rb
+- features/steps/file_steps.rb
+- features/steps/rubygem_steps.rb
+- lib/rubygems/commands/precompile_command.rb
+- lib/rubygems/precompiled.rb
+- lib/rubygems/precompiled/version.rb
+- lib/rubygems/precompiler.rb
+- lib/rubygems_plugin.rb
+- rubygems-precompiled.gemspec
+- spec/fixtures/gem-sources.tar.gz
+homepage: https://github.com/joshado/rubygems-precompiled
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: RubyGems plugin to allow a gem's compiled extension to be pre-built and cached
+test_files:
+- features/installing_with_cache.feature
+- features/precompiling_gems.feature
+- features/steps/command_steps.rb
+- features/steps/file_steps.rb
+- features/steps/rubygem_steps.rb
+- spec/fixtures/gem-sources.tar.gz