kameleon-builder 2.0.0 → 2.1.0

data/lib/kameleon/shell.rb

@@ -8,30 +8,41 @@ module Kameleon
     READ_CHUNK_SIZE = 1048576
     EXIT_TIMEOUT = 60
 
-    attr :exit_status, :process
+    attr :exit_status, :process, :shell_cmd
 
     def initialize(context_name, cmd, shell_workdir, local_workdir, kwargs = {})
-      @logger = Log4r::Logger.new("kameleon::[shell]")
+      @logger = Log4r::Logger.new("kameleon::[kameleon]")
       @cmd = cmd.chomp
       @context_name = context_name
       @local_workdir = local_workdir
       @shell_workdir = shell_workdir
-      @bashrc_file = "/tmp/kameleon_#{@context_name}_bash_rc"
-      @bash_history_file = "/tmp/kameleon_#{@context_name}_bash_history"
-      @bash_env_file = "/tmp/kameleon_#{@context_name}_bash_env"
-      change_dir_cmd = ""
-      if @shell_workdir
-        unless @shell_workdir.eql? "/"
-          change_dir_cmd = "mkdir -p #{@shell_workdir} &&"
-        end
-        change_dir_cmd = "#{change_dir_cmd} cd #{@shell_workdir} && "
-      end
+      @bashrc_file = "kameleon_#{@context_name}_bash_rc"
+      @bash_history_file = "kameleon_#{@context_name}_bash_history"
+      @bash_env_file = "kameleon_#{@context_name}_bash_env"
       @default_bashrc_file = File.join(Kameleon.source_root,
                                        "contrib",
                                        "kameleon_bashrc.sh")
-      bash_cmd = "bash --rcfile #{@bashrc_file}"
+      if @shell_workdir
+        @bashrc_file = File.join(@shell_workdir, @bashrc_file)
+        @bash_history_file = File.join(@shell_workdir, @bash_history_file)
+        @bash_env_file = File.join(@shell_workdir, @bash_env_file)
+      end
+
+      ## Changing the default bashrc if the cache is activated
+      @cache = Kameleon::Persistent_cache.instance
+      if @cache.activated? then
+        new_kameleon_bashrc = Tempfile.new('kameleon_bashrc').path
+        FileUtils.cp @default_bashrc_file, new_kameleon_bashrc
+        tpl = ERB.new(File.read(@cache.polipo_env))
+        polipo_env_content = tpl.result(binding)
+        File.open(new_kameleon_bashrc,'a') do |f|
+          f.puts(polipo_env_content)
+        end
+        @default_bashrc_file = new_kameleon_bashrc
+      end
+
       @shell_cmd = "source #{@default_bashrc_file} 2> /dev/null; "\
-                   "#{@cmd} -c '#{change_dir_cmd}#{bash_cmd}'"
+                   "#{@cmd} --rcfile #{@bashrc_file}"
       @logger.debug("Initialize shell (#{self})")
       # Injecting all variables of the options and assign the variables
       instance_variables.each do |v|
@@ -45,7 +56,11 @@ module Kameleon
     end
 
     def stop
-      @process.stop
+      @process.stop(0) unless @process.nil?
+    end
+
+    def started?
+      return !@process.nil?
     end
 
     def exited?
@@ -59,6 +74,7 @@ module Kameleon
 
     def send_file(source_path, remote_dest_path, chunk_size=READ_CHUNK_SIZE)
       copy_process, = fork("pipe")
+      copy_process.io.stdin << init_shell_cmd
       copy_process.io.stdin << "cat > #{remote_dest_path}\n"
       copy_process.io.stdin.flush
       open(source_path, "rb") do |f|
@@ -73,15 +89,32 @@ module Kameleon
     end
 
     def init_shell_cmd
+      ## log shell error message
+      error = read_io(@stderr)
+      init_stdout = read_io(@stdout)
+      @logger.error(error) unless error.empty?
+      @logger.info(init_stdout) unless init_stdout.empty?
       bashrc_content = ""
       if File.file?(@default_bashrc_file)
         tpl = ERB.new(File.read(@default_bashrc_file))
         bashrc_content = tpl.result(binding)
       end
       bashrc = Shellwords.escape(bashrc_content)
+      if @shell_workdir
+        unless @shell_workdir.eql? "/"
+          change_dir_cmd = "mkdir -p #{@shell_workdir} &&"
+        end
+        change_dir_cmd = "#{change_dir_cmd} cd #{@shell_workdir}"
+        change_dir_cmd = Shellwords.escape(change_dir_cmd)
+      end
       shell_cmd = "mkdir -p $(dirname #{@bashrc_file})\n"
       shell_cmd << "echo #{bashrc} > #{@bashrc_file}\n"
+      shell_cmd << "echo #{bashrc} > #{@bashrc_file}\n"
+      unless change_dir_cmd.nil?
+        shell_cmd << "echo #{change_dir_cmd} >> #{@bashrc_file}\n"
+      end
       shell_cmd << "source #{@bashrc_file}\n"
+      shell_cmd << "export KAMELEON_WORKDIR=$PWD\n"
       shell_cmd
     end
 
@@ -167,7 +200,8 @@ module Kameleon
     end
 
     def fork_and_wait
-      system(@shell_cmd)
+      process, _, _ = fork("inherit")
+      process.wait()
     end
 
     protected
@@ -214,6 +248,7 @@ module Kameleon
     def fork(io)
       command = ["bash", "-c", @shell_cmd]
       @logger.notice("Starting process: #{@cmd.inspect}")
+      @logger.debug("Starting shell process: #{ command.inspect}")
       ChildProcess.posix_spawn = true
       process = ChildProcess.build(*command)
       # Create the pipes so we can read the output in real time as

data/omnibus/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+.bundle
+.kitchen/
+.kitchen.local.yml
+vendor/bundle
+pkg/*
+.vagrant
+bin/*
+files/**/cache/
+vendor/cookbooks
+*.out

data/omnibus/.kitchen.yml

@@ -0,0 +1,25 @@
+driver:
+  name: vagrant
+  customize:
+    cpus: 2
+    memory: 2048
+  synced_folders:
+    - ['../../..', '/home/vagrant/kameleon']
+    - ['../../../pkg/%{instance_name}', '/var/cache/omnibus/pkg', 'create: true']
+
+provisioner:
+  require_chef_omnibus: 11.8.0
+
+platforms:
+  - name: ubuntu-12.04
+    run_list: apt::default
+  - name: centos-5.10
+
+suites:
+  - name: default
+    run_list: omnibus::default
+    attributes:
+      omnibus:
+        build_user:  vagrant
+        build_dir:   /home/vagrant/kameleon
+        install_dir: /opt/kameleon

data/omnibus/Berksfile

@@ -0,0 +1,9 @@
+source 'https://api.berkshelf.com'
+
+# The apt cookbook is required to bring the apt cache up-to-date on Ubuntu
+# systems, since the cache can become stale on older boxes.
+cookbook 'apt', '~> 2.0'
+
+# cookbook 'omnibus'
+# Uncomment to use the latest version of the Omnibus cookbook from GitHub
+cookbook 'omnibus', github: 'opscode-cookbooks/omnibus'

data/omnibus/Berksfile.lock

@@ -0,0 +1,25 @@
+DEPENDENCIES
+  apt (~> 2.0)
+  omnibus
+    git: git://github.com/opscode-cookbooks/omnibus.git
+    branch: master
+
+GRAPH
+  7-zip (1.0.2)
+    windows (>= 1.2.2)
+  apt (2.3.8)
+  build-essential (2.0.0)
+  chef-sugar (1.2.8)
+  chef_handler (1.1.5)
+  homebrew (1.5.4)
+  omnibus (1.2.5)
+    7-zip (~> 1.0)
+    build-essential (~> 2.0)
+    chef-sugar (~> 1.1)
+    homebrew (~> 1.5)
+    windows (~> 1.30)
+    wix (~> 1.1)
+  windows (1.30.2)
+    chef_handler (>= 0.0.0)
+  wix (1.1.0)
+    windows (>= 1.2.2)

data/omnibus/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Use Berkshelf for resolving cookbook dependencies
+gem 'berkshelf', '~> 3.0.0.beta'
+
+# Install omnibus software
+gem 'omnibus', '~> 3.0'
+gem 'omnibus-software', github: 'opscode/omnibus-software'
+
+# Use Test Kitchen with Vagrant for convering the build environment
+gem 'test-kitchen',    '~> 1.2'
+gem 'kitchen-vagrant', '~> 0.14'

data/omnibus/README.md

@@ -0,0 +1,94 @@
+kameleon Omnibus project
+========================
+This project creates full-stack platform-specific packages for
+`kameleon`!
+
+Installation
+------------
+You must have a sane Ruby 1.9+ environment with Bundler installed. Ensure all
+the required gems are installed:
+
+```shell
+$ bundle install --binstubs
+```
+
+Usage
+-----
+### Build
+
+You create a platform-specific package using the `build project` command:
+
+```shell
+$ bin/omnibus build project kameleon
+```
+
+The platform/architecture type of the package created will match the platform
+where the `build project` command is invoked. For example, running this command
+on a MacBook Pro will generate a Mac OS X package. After the build completes
+packages will be available in the `pkg/` folder.
+
+### Clean
+
+You can clean up all temporary files generated during the build process with
+the `clean` command:
+
+```shell
+$ bin/omnibus clean
+```
+
+Adding the `--purge` purge option removes __ALL__ files generated during the
+build including the project install directory (`/opt/kameleon`) and
+the package cache directory (`/var/cache/omnibus/pkg`):
+
+```shell
+$ bin/omnibus clean --purge
+```
+
+### Help
+
+Full help for the Omnibus command line interface can be accessed with the
+`help` command:
+
+```shell
+$ bin/omnibus help
+```
+
+Kitchen-based Build Environment
+-------------------------------
+Every Omnibus project ships will a project-specific
+[Berksfile](http://berkshelf.com/) and [Vagrantfile](http://www.vagrantup.com/)
+that will allow you to build your omnibus projects on all of the projects listed
+in the `.kitchen.yml`. You can add/remove additional platforms as needed by
+changing the list found in the `.kitchen.yml` `platforms` YAML stanza.
+
+This build environment is designed to get you up-and-running quickly. However,
+there is nothing that restricts you to building on other platforms. Simply use
+the [omnibus cookbook](https://github.com/opscode-cookbooks/omnibus) to setup
+your desired platform and execute the build steps listed above.
+
+The default build environment requires Test Kitchen and VirtualBox for local
+development. Test Kitchen also exposes the ability to provision instances using
+various cloud providers like AWS, DigitalOcean, or OpenStack. For more
+information, please see the [Test Kitchen documentation](http://kitchen.ci).
+
+Once you have tweaked your `.kitchen.yml` (or `.kitchen.local.yml`) to your
+liking, you can bring up an individual build environment using the `kitchen`
+command.
+
+```shell
+$ bundle exec kitchen converge ubuntu-12.04
+```
+
+Then login to the instance and build the project as described in the Usage
+section:
+
+```shell
+$ bundle exec kitchen login ubuntu-12.04
+[vagrant@ubuntu...] $ cd kameleon
+[vagrant@ubuntu...] $ bundle install
+[vagrant@ubuntu...] $ ...
+[vagrant@ubuntu...] $ ./bin/omnibus build project kameleon
+```
+
+For a complete list of all commands and platforms, run `kitchen list` or
+`kitchen help`.

data/omnibus/config/projects/kameleon.rb

@@ -0,0 +1,23 @@
+
+name 'kameleon'
+maintainer 'Oar Team'
+homepage 'http://kameleon.readthedocs.org'
+
+replaces        'kameleon'
+install_path    '/opt/kameleon'
+build_version Omnibus::BuildVersion.new.semver
+build_iteration 1
+
+# creates required build directories
+dependency 'preparation'
+
+# kameleon dependencies/components
+dependency "polipo"
+dependency "kameleon"
+
+# version manifest file
+dependency "version-manifest"
+
+
+exclude '\.git*'
+exclude 'bundler\/git'

data/omnibus/config/software/kameleon.rb

@@ -0,0 +1,24 @@
+# This is an example software definition for a Ruby project.
+#
+# Lots of software definitions for popular open source software
+# already exist in `opscode-omnibus`:
+#
+#  https://github.com/opscode/omnibus-software/tree/master/config/software
+#
+name "kameleon"
+default_version "2.0.0"
+
+dependency "ruby"
+dependency "bundler"
+dependency "rsync"
+
+source :git => "git://github.com/oar-team/kameleon.git"
+
+relative_path "kameleon"
+
+build do
+  command "git checkout #{default_version}"
+  bundle "install --path=#{install_dir}/embedded/service/gem"
+  command "mkdir -p #{install_dir}/embedded/service/kameleon"
+  command "#{install_dir}/embedded/bin/rsync -a --delete --exclude=.git/*** --exclude=.gitignore ./ #{install_dir}/embedded/service/kameleon/"
+end

data/omnibus/config/software/polipo.rb

@@ -0,0 +1,30 @@
+# This is an example software definition for a C project.
+#
+# Lots of software definitions for popular open source software
+# already exist in `opscode-omnibus`:
+#
+#  https://github.com/opscode/omnibus-software/tree/master/config/software
+#
+name "polipo"
+default_version "1.0.3"
+
+
+source :url => "http://freehaven.net/~chrisd/polipo/polipo-1.0.3.tar.gz",
+       :md5 => "a0b00ca01541cf77ff3d725c27cf68bb"
+
+relative_path 'polipo-1.0.3'
+
+prefix="#{install_dir}/embedded"
+libdir="#{prefix}/lib"
+
+env = {
+  "LDFLAGS" => "-L#{libdir} -I#{prefix}/include",
+  "CFLAGS" => "-L#{libdir} -I#{prefix}/include -fPIC",
+  "LD_RUN_PATH" => libdir
+}
+
+build do
+  command "sed -i 's/^LOCAL_ROOT = \/usr\/share\/polipo\/www/LOCAL_ROOT = \$\(PREFIX\)\/usr\/share\/polipo\/www/g' Makefile"
+  command "make -j #{max_build_jobs} PREFIX=#{prefix} all", :env => env
+  command "make -j #{max_build_jobs} PREFIX=#{prefix} install", :env => env
+end

data/omnibus/config/software/ruby.rb

@@ -0,0 +1,158 @@
+#
+# Copyright:: Copyright (c) 2012-2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "ruby"
+default_version "1.9.3-p484"
+
+dependency "zlib"
+dependency "ncurses"
+dependency "libedit"
+dependency "openssl"
+dependency "libyaml"
+dependency "libiconv"
+dependency "gdbm"
+dependency "libgcc" if (platform == "solaris2" and Omnibus.config.solaris_compiler == "gcc")
+
+version "1.9.3-p484" do
+  source md5: '8ac0dee72fe12d75c8b2d0ef5d0c2968'
+end
+
+source url: "http://cache.ruby-lang.org/pub/ruby/#{version.match(/^(\d+\.\d+)/)[0]}/ruby-#{version}.tar.gz"
+
+relative_path "ruby-#{version}"
+
+env =
+  case platform
+  when "mac_os_x"
+    {
+      # -Qunused-arguments suppresses "argument unused during compilation"
+      # warnings. These can be produced if you compile a program that doesn't
+      # link to anything in a path given with -Lextra-libs. Normally these
+      # would be harmless, except that autoconf treats any output to stderr as
+      # a failure when it makes a test program to check your CFLAGS (regardless
+      # of the actual exit code from the compiler).
+      "CFLAGS" => "-arch x86_64 -m64 -L#{install_dir}/embedded/lib -I#{install_dir}/embedded/include -I#{install_dir}/embedded/include/ncurses -O3 -g -pipe -Qunused-arguments",
+      "LDFLAGS" => "-arch x86_64 -L#{install_dir}/embedded/lib -I#{install_dir}/embedded/include -I#{install_dir}/embedded/include/ncurses"
+    }
+  when "solaris2"
+    {
+      "CFLAGS" => "-L#{install_dir}/embedded/lib -I#{install_dir}/embedded/include -O3 -g -pipe",
+      "LDFLAGS" => "-R#{install_dir}/embedded/lib -L#{install_dir}/embedded/lib -I#{install_dir}/embedded/include -static-libgcc",
+      "LD_OPTIONS" => "-R#{install_dir}/embedded/lib"
+    }
+  when "aix"
+    {
+      # see http://www.ibm.com/developerworks/aix/library/au-gnu.html
+      #
+      # specifically:
+      #
+      # "To use AIX run-time linking, you should create the shared object
+      # using gcc -shared -Wl,-G and create executables using the library
+      # by adding the -Wl,-brtl option to the link line. Technically, you
+      # can leave off the -shared option, but it does no harm and reduces
+      # confusion."
+      #
+      # AIX also uses -Wl,-blibpath instead of -R or LD_RUN_PATH, but the
+      # option is not additive, so requires /usr/lib and /lib as well (there
+      # is a -bsvr4 option to allow ld to take an -R flag in addition
+      # to turning on -brtl, but it had other side effects I couldn't fix).
+      #
+      # If libraries linked with gcc -shared have symbol resolution failures
+      # then it may be useful to add -bexpfull to export all symbols.
+      #
+      # -O2 optimized away some configure test which caused ext libs to fail
+      #
+      # We also need prezl's M4 instead of picking up /usr/bin/m4 which
+      # barfs on ruby.
+      #
+      "CC" => "xlc -q64",
+      "CXX" => "xlC -q64",
+      "LD" => "ld -b64",
+      "CFLAGS" => "-q64 -O -qhot -I#{install_dir}/embedded/include",
+      "CXXFLAGS" => "-q64 -O -qhot -I#{install_dir}/embedded/include",
+      "LDFLAGS" => "-q64  -L#{install_dir}/embedded/lib -Wl,-brtl -Wl,-blibpath:#{install_dir}/embedded/lib:/usr/lib:/lib",
+      "OBJECT_MODE" => "64",
+      "ARFLAGS" => "-X64 cru",
+      "M4" => "/opt/freeware/bin/m4",
+      "warnflags" => "-qinfo=por"
+    }
+  else
+    {
+      "CFLAGS" => "-I#{install_dir}/embedded/include -O3 -g -pipe",
+      "LDFLAGS" => "-Wl,-rpath,#{install_dir}/embedded/lib -L#{install_dir}/embedded/lib"
+    }
+  end
+
+build do
+  configure_command = ["./configure",
+                       "--prefix=#{install_dir}/embedded",
+                       "--with-out-ext=fiddle,dbm",
+                       "--enable-shared",
+                       "--enable-libedit",
+                       "--with-ext=psych",
+                       "--disable-install-doc"]
+
+  case platform
+  when "aix"
+    patch :source => "ruby-aix-configure.patch", :plevel => 1
+    patch :source => "ruby_aix_1_9_3_448_ssl_EAGAIN.patch", :plevel => 1
+    # --with-opt-dir causes ruby to send bogus commands to the AIX linker
+  when "freebsd"
+    configure_command << "--without-execinfo"
+    configure_command << "--with-opt-dir=#{install_dir}/embedded"
+  when "smartos"
+    # Opscode patch - someara@opscode.com
+    # GCC 4.7.0 chokes on mismatched function types between OpenSSL 1.0.1c and Ruby 1.9.3-p286
+    patch :source => "ruby-openssl-1.0.1c.patch", :plevel => 1
+
+    # Patches taken from RVM.
+    # http://bugs.ruby-lang.org/issues/5384
+    # https://www.illumos.org/issues/1587
+    # https://github.com/wayneeseguin/rvm/issues/719
+    patch :source => "rvm-cflags.patch", :plevel => 1
+
+    # From RVM forum
+    # https://github.com/wayneeseguin/rvm/commit/86766534fcc26f4582f23842a4d3789707ce6b96
+    configure_command << "ac_cv_func_dl_iterate_phdr=no"
+    configure_command << "--with-opt-dir=#{install_dir}/embedded"
+  else
+    configure_command << "--with-opt-dir=#{install_dir}/embedded"
+  end
+
+  # @todo expose bundle_bust() in the DSL
+  env.merge!({
+    "RUBYOPT"         => nil,
+    "BUNDLE_BIN_PATH" => nil,
+    "BUNDLE_GEMFILE"  => nil,
+    "GEM_PATH"        => nil,
+    "GEM_HOME"        => nil
+  })
+
+  # @todo: move into omnibus-ruby
+  has_gmake = system("gmake --version")
+
+  if has_gmake
+    env.merge!({'MAKE' => 'gmake'})
+    make_binary = 'gmake'
+  else
+    make_binary = 'make'
+  end
+
+  command configure_command.join(" "), :env => env
+  command "#{make_binary} -j #{max_build_jobs}", :env => env
+  command "#{make_binary} -j #{max_build_jobs} install", :env => env
+end