vagrant-openshift 1.0.9

data/lib/vagrant-openshift/templates/builder/Rakefile

@@ -0,0 +1,231 @@
+#--
+# Copyright 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+
+require 'rubygems'
+require 'pathname'
+require 'yaml'
+require 'tsort'
+require_relative 'lib/constants'
+require_relative 'lib/options'
+require_relative 'lib/rpm'
+require_relative 'lib/test'
+$stdout.sync = true
+$stderr.sync = true
+
+VALID_IP_ADDR_RE = Regexp.new('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')
+
+task :install_deps do
+  spec_files = RPM::get_packages(ignore_packages: OPTIONS[:ignore_packages], ignore_cache: true)
+
+  all_build_dependencies = []
+  all_inst_dependencies  = []
+  spec_files.each do |spec|
+    all_build_dependencies += spec[:build_deps]
+    all_inst_dependencies  += spec[:deps]
+  end
+
+  all_packages = all_build_dependencies + all_inst_dependencies
+
+  all_packages.map! do |name|
+    m = name.match(/(ruby193-)?rubygem[(]([a-z\-_]*)[)]/)
+    if m.nil?
+      name.strip
+    else
+      scl_prefix = m[1].nil? ? "" : m[1]
+      "#{scl_prefix}rubygem-#{m[2]}".strip
+    end
+  end
+
+  all_packages.delete_if { |name| name.match(/openshift/)}
+
+  RPM::install_rpms(all_packages)
+end
+
+task :build_all do
+  FileUtils.rm_rf("/tmp/tito/*")
+  spec_files = RPM::get_packages(remove_os_build_deps: true, ignore_packages: OPTIONS[:ignore_packages], ignore_cache: true)
+  RPM::build_packages(spec_files)
+end
+
+task :clean_rpms do
+  FileUtils.rm_rf("/tmp/tito/*")
+  RPM::uninstall_openshift
+end
+
+task :install_puppet do
+  Dir.chdir((Vagrant::Openshift::Constants.build_dir + "puppet-openshift_origin").to_s) do
+    FileUtils.rm_rf "pkg"
+    system("puppet module build .")
+    pkg = Dir.glob((Vagrant::Openshift::Constants.build_dir + "puppet-openshift_origin/pkg/openshift-openshift_origin*.tar.gz").to_s).first
+    system("puppet module uninstall openshift-openshift_origin")
+    system("puppet module install #{pkg}")
+    system("rm -rf #{Vagrant::Openshift::Constants.build_dir + "puppet-openshift_origin/pkg"}")
+  end
+end
+
+task :update_packages do
+  if  (Vagrant::Openshift::Constants.build_dir + ".built_packages").exist?
+    print "Updating packages\n"
+    RPM::updated_packages
+  else
+    print "Building all packages\n"
+    Rake::Task['build_all'].invoke
+    print "Installing openshift puppet module\n"
+    Rake::Task['install_puppet'].invoke
+    print "Applying openshift puppet recipe\n"
+    system("puppet apply --verbose #{Vagrant::Openshift::Constants.build_dir + 'configure_origin.pp'}")
+  end
+  Rake::Task['restart_services'].invoke
+end
+
+task :restart_services do
+  is_fedora=File.exists?("/etc/fedora-release")
+  system(Vagrant::Openshift::Constants.restart_services_cmd(is_fedora).join("\n"))
+  cmd = ""
+  OPTIONS[:additional_services].each do |service|
+    cmd += "service #{service} restart;"
+  end
+  system(cmd)
+
+  unless system("service openshift-tc status 2>/dev/null 1>/dev/null")
+    system("service openshift-tc reload")
+  end
+
+  # TODO: Replace this temporary fix with a complete post-install routine
+  `oo-admin-ctl-cartridge -c import-node --activate --obsolete && RAILS_ENV=test oo-admin-ctl-cartridge -c import-node --activate --obsolete`
+end
+
+task :checkout_tests do
+  print "Updating remote tests..."
+  system "rm -rf #{Vagrant::Openshift::Constants.build_dir}/openshift-test"
+  git_archive_commands = ''
+  Vagrant::Openshift::Constants.repos.each do |repo_name, url|
+    repo_dir = "#{Vagrant::Openshift::Constants.build_dir}/#{repo_name}-bare"
+    git_archive_commands += "pushd #{repo_dir} > /dev/null; git archive --prefix openshift-test/ --format=tar 'HEAD' | (cd #{Vagrant::Openshift::Constants.build_dir} && tar --warning=no-timestamp -xf -); popd > /dev/null; "
+  end
+  git_archive_commands += "pushd #{Vagrant::Openshift::Constants.build_dir}/rhc-bare > /dev/null; git archive --prefix openshift-test/rhc/ --format=tar 'HEAD' | (cd #{Vagrant::Openshift::Constants.build_dir} && tar --warning=no-timestamp -xf -); popd > /dev/null; "
+
+  system %{
+set -e;
+rm -rf #{Vagrant::Openshift::Constants.build_dir}/openshift-test
+#{git_archive_commands}
+mkdir -p /tmp/rhc/junit
+          }
+
+  system(%{
+rm -rf #{Vagrant::Openshift::Constants.build_dir}/openshift-test/tests;
+ln -s #{Vagrant::Openshift::Constants.build_dir}/openshift-test/controller/test/cucumber #{Vagrant::Openshift::Constants.build_dir}/openshift-test/tests
+})
+  print "done\n"
+end
+
+task :create_test_users do
+  system("mkdir -p /etc/openshift && touch /etc/openshift/htpasswd")
+  ['test_user@test.com', 'user_with_multiple_gear_sizes@test.com', 'user_with_extra_storage@test.com', 'user_with_certificate_capabilities@test.com']. each do |user|
+    unless system("/usr/bin/curl -f -k https://broker.example.com/broker/rest/user -u #{user}:test 2>/dev/null 1>/dev/null")
+
+      # Try to create the user record
+      print "Creating test user:  #{user}\n"
+      user_created   = false
+      create_retries = 0
+      while create_retries < 3 do
+        user_created = system("/usr/bin/htpasswd -b /etc/openshift/htpasswd #{user} test")
+        break if user_created
+        create_retries = create_retries + 1
+        print "Could not add user #{user} to htpasswd file on attempt #{create_retries}\n"
+        sleep 2
+      end
+      if not user_created
+        print "Unable to add user '#{user}' to /etc/openshift/htpasswd\n"
+        exit 1
+      end
+
+      # Test the user record via the REST API
+      rest_connected = false
+      rest_retries   = 0
+      while rest_retries < 3 do
+        rest_connected = system("/usr/bin/curl -f -k https://broker.example.com/broker/rest/user -u #{user}:test 2>/dev/null 1>/dev/null")
+        break if rest_connected
+        rest_retries = rest_retries + 1
+        print "Could not verify user #{user} via the REST API on attempt #{rest_retries}\n"
+        sleep 10
+      end
+      if not rest_connected
+        print "Unable to verify user #{user} via the REST API\n"
+        exit 1
+      end
+    else
+      print "#{user} already exists\n"
+    end
+  end
+
+  # Create a known test user with medium-sized gears
+  # This must be done before the deployment of application templates!
+  exitcode = system("/usr/sbin/oo-admin-ctl-user -l user_with_multiple_gear_sizes@test.com --addgearsize medium")
+  if(!exitcode)
+    print "Unable to assign medium gear capability to user_with_multiple_gear_sizes@test.com"
+    exit 1
+  end
+
+  # Create a test user with additional storage capabilities
+  exitcode = system("/usr/sbin/oo-admin-ctl-user -l user_with_extra_storage@test.com --setmaxuntrackedstorage 5 --setmaxgears 10")
+  if(!exitcode)
+    print "Unable to assign additional storage to user_with_extra_storage@test.com"
+    exit 1
+  end
+
+  # Create a test user with ssl certificates capabilities
+  exitcode = system("/usr/sbin/oo-admin-ctl-user -l user_with_certificate_capabilities@test.com --allowprivatesslcertificates true")
+  if(!exitcode)
+    print "Unable to assign ssl certificates capabilities to user_with_certificate_capabilities@test.com"
+    exit 1
+  end
+
+  unless system("test -e  /root/.ssh/id_rsa")
+    system("ssh-keygen -q -f /root/.ssh/id_rsa -P \"\" </dev/null")
+  end
+end
+
+task :idle_all_gears do
+  Test.new.idle_all_gears
+end
+
+task :run_tests do
+  options = {}
+  options[:fixme] = ENV["fixme"] == "true"
+  if ENV["extended"] == "true"
+    options[:gear_extended]   = ENV["gear"] == "true"
+    options[:node_extended]   = ENV["node"] == "true"
+    options[:cart_extended]   = ENV["cart"] == "true"
+    options[:broker_extended] = ENV["broker"] == "true"
+    options[:rhc_extended]    = ENV["rhc"] == "true"
+    options[:console_extended] = ENV["console"] == "true"
+    options[:node]    = false
+    options[:cart]    = false
+    options[:broker]  = false
+    options[:rhc]     = false
+    options[:console] = false
+  else
+    options[:gear]    = ENV["gear"] == "true"
+    options[:node]    = ENV["node"] == "true"
+    options[:cart]    = ENV["cart"] == "true"
+    options[:broker]  = ENV["broker"] == "true"
+    options[:rhc]     = ENV["rhc"] == "true"
+    options[:console] = ENV["console"] == "true"
+  end
+
+  Test.new.test(options)
+end

data/lib/vagrant-openshift/templates/builder/lib/rpm.rb

@@ -0,0 +1,248 @@
+#--
+# Copyright 2013 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+
+require_relative 'constants'
+require_relative 'options'
+require 'tempfile'
+require 'fileutils'
+require 'pathname'
+require 'yaml'
+
+class RPM
+  def self.get_packages(options={})
+    spec_cache = Pathname.new(Vagrant::Openshift::Constants.build_dir + ".spec_cache")
+    if spec_cache.exist? && !options[:ignore_cache]
+      return YAML.load(File.open(spec_cache.to_s))
+    end
+
+    parent_dir = Pathname.new(File.expand_path("__FILE__/../../"))
+
+    #find all spec files
+    spec_files = []
+    Vagrant::Openshift::Constants.repos.each do |name, url|
+      repo_dir = parent_dir + name
+      spec_files += Dir.glob((repo_dir + "**/*.spec").to_s).map{ |p| {path: p}}
+    end
+
+    #exclude ones that have not been tagged
+    source_package_names = []
+    spec_files.delete_if do |spec|
+      spec[:name] = `rpm -q --specfile --queryformat '%{NAME}\n' #{spec[:path]}`.split("\n")[0]
+      spec[:version] = `rpm -q --specfile --queryformat '%{VERSION}\n' #{spec[:path]}`.split("\n")[0]
+      source_package_names << spec[:name]
+      spec[:dir] = Pathname.new(spec[:path]).dirname
+      is_tagged = Dir.chdir(spec[:dir]) { system "git tag | grep '#{spec[:name]}' 2>&1 1>/dev/null" }
+      is_spec_bad = false
+      if is_tagged
+        begin
+          spec[:build_deps] = YAML.load(`./yum-listbuilddep #{spec[:path]}`.split("\n").last)
+          raise "Bad spec" if spec[:build_deps].class == Hash
+          spec[:deps] = `rpm -q --specfile --requires #{spec[:path]}`.split("\n")
+        rescue Exception => e
+          next true
+        end
+      else
+        puts "\n\nSkipping '#{spec[:name]}' in '#{spec[:dir]}' since it is not tagged.\n"
+        next true
+      end
+
+      options.has_key?(:ignore_packages) && options[:ignore_packages].include?(spec[:name])
+    end
+
+    if(options.has_key?(:remove_os_build_deps) && options[:remove_os_build_deps])
+      #remove deps provided by OS and keep only the ones that will be built from source
+      spec_files.each do |spec|
+        spec[:build_deps].delete_if{ |dep| not source_package_names.include? dep }
+      end
+    end
+
+    File.open(spec_cache.to_s, "w") do |file|
+      file.write(spec_files.to_yaml)
+    end
+
+    spec_files
+  end
+
+  def self.install_rpms(list)
+    print "Installing #{list.size} in chunks of 20 packages"
+    list.uniq.each_slice(20) do |sub_list|
+      system %{yum install -y --skip-broken "#{sub_list.join('" "')}"}
+    end
+  end
+
+  def self.uninstall_openshift
+    #undo pam.d sshd modification
+    if system("grep -n 'pam_openshift' /etc/pam.d/sshd")
+      file = Tempfile.new('temp')
+      begin
+        file.write(%{
+set /files/etc/pam.d/sshd/#comment[.='pam_openshift.so close should be the first session rule'] 'pam_selinux.so close should be the first session rule'
+ins 01 before /files/etc/pam.d/sshd/*[argument='close']
+set /files/etc/pam.d/sshd/01/type session
+set /files/etc/pam.d/sshd/01/control required
+set /files/etc/pam.d/sshd/01/module pam_selinux.so
+set /files/etc/pam.d/sshd/01/argument close
+
+set /files/etc/pam.d/sshd/#comment[.='pam_openshift.so open should only be followed by sessions to be executed in the user context'] 'pam_selinux.so open should only be followed by sessions to be executed in the user context'
+ins 02 before /files/etc/pam.d/sshd/*[argument='open']
+set /files/etc/pam.d/sshd/02/type session
+set /files/etc/pam.d/sshd/02/control required
+set /files/etc/pam.d/sshd/02/module pam_selinux.so
+set /files/etc/pam.d/sshd/02/argument[1] open
+set /files/etc/pam.d/sshd/02/argument[2] env_params
+
+rm /files/etc/pam.d/sshd/*[module='pam_openshift.so']
+rm /files/etc/pam.d/sshd/*[module='pam_namespace.so']
+rm /files/etc/pam.d/sshd/*[module='pam_cgroup.so']
+rm /files/etc/pam.d/sshd/*[module='pam_succeed_if.so']
+save
+      })
+        file.close
+        system("augtool --file #{file.path}")
+      ensure
+        file.unlink
+      end
+    end
+    FileUtils.rm_rf("/etc/openshift")
+
+    get_packages(ignore_cache: true).each_slice(10) do |spec_files_list|
+      sub_list = spec_files_list.map {|s| s[:name] }
+      system %{yum erase -y "#{sub_list.join('" "')}"}
+    end
+  end
+
+  def self.build_packages(spec_files, opts={})
+    built_specs = []
+
+    FileUtils.mkdir_p "/tmp/tito/noarch/"
+    FileUtils.mkdir_p (Vagrant::Openshift::Constants.build_dir + "origin-rpms").to_s
+    FileUtils.mkdir_p (Vagrant::Openshift::Constants.build_dir + "origin-srpms").to_s
+    (1..3).each do |phase|
+      print "Build phase #{phase}\n"
+      buildable = []
+
+      #install pre reqs if they have already been built
+      spec_files.each do |spec|
+        spec[:build_deps].delete_if do |dep|
+          if opts[:assume_deps_built] || built_specs.map{ |s| s[:name] }.include?(dep)
+            puts "\n    Installing...#{dep}"
+            raise "Unable to install package #{package.name}" unless system("rpm -Uvh --force /tmp/tito/noarch/#{dep}*.rpm")
+            true
+          else
+            false
+          end
+        end
+      end
+
+      buildable = spec_files.select{ |spec| spec[:build_deps].size == 0 }
+
+      buildable.each do |spec|
+        Dir.chdir(spec[:dir]) do
+          puts "\n#{'-'*60}"
+          system "rm -f /tmp/tito/noarch/#{spec[:name]}*.rpm"
+          system "rm -f /tmp/tito/#{spec[:name]}*.src.rpm"
+
+          raise "Unable to build #{spec[:name]}" unless system("tito build --rpm --test")
+
+          Dir.glob('/tmp/tito/x86_64/*.rpm').each {|file|
+            FileUtils.mv file, "/tmp/tito/noarch/"
+          }
+          Dir.glob("/tmp/tito/noarch/#{spec[:name]}*.rpm").each do |file|
+            FileUtils.rm_f (Vagrant::Openshift::Constants.build_dir + "origin-rpms/" + "#{spec[:name]}*.rpm").to_s
+            FileUtils.cp file, (Vagrant::Openshift::Constants.build_dir + "origin-rpms/").to_s
+          end
+          Dir.glob('/tmp/tito/*.src.rpm').each {|file|
+            FileUtils.rm_f (Vagrant::Openshift::Constants.build_dir + "origin-srpms/" + "#{spec[:name]}*.src.rpm").to_s
+            FileUtils.mv file, (Vagrant::Openshift::Constants.build_dir + "origin-srpms/").to_s
+          }
+
+          built_specs << spec
+          spec_files.delete(spec)
+        end
+      end
+    end
+
+    File.open((Vagrant::Openshift::Constants.build_dir + ".built_packages").to_s, "w") do |file|
+      file.write(tito_report.to_yaml)
+    end
+
+    print "Updating local repo for built sources"
+    Dir.chdir((Vagrant::Openshift::Constants.build_dir + "origin-rpms/").to_s) do
+      system("createrepo .")
+      system("yum clean all")
+    end
+
+    built_specs
+  end
+
+  def self.updated_packages
+    spec_files = get_updated_packages
+    built_packages = build_packages(spec_files, {assume_deps_built: true})
+    built_packages.each do |spec|
+      unless system("rpm -Uvh --force #{Vagrant::Openshift::Constants.build_dir + "origin-rpms"}/#{spec[:name]}*.rpm")
+        unless system("rpm -e --justdb --nodeps #{spec[:name]}; yum install -y #{Vagrant::Openshift::Constants.build_dir + "origin-rpms"}/#{spec[:name]}*.rpm")
+          print "Unable to install updated package #{spec[:name]}"
+          exit 1
+        end
+      end
+    end
+  end
+
+  private
+
+  def self.get_updated_packages
+    raise "Please perform full build with 'rake openshift:build_all' before running sync" unless (Vagrant::Openshift::Constants.build_dir + ".built_packages").exist?
+
+    spec_files = get_packages(remove_os_build_deps: true)
+    cur_report = tito_report
+    old_report = YAML.load(File.new((Vagrant::Openshift::Constants.build_dir + ".built_packages").to_s))
+
+    updated_packages = []
+    spec_files.each do |spec|
+      cur_package_report = cur_report[spec[:name]]
+      old_package_report = old_report[spec[:name]]
+
+      if (old_package_report.nil? && !cur_package_report.nil?) ||
+          (!cur_package_report.nil? && cur_package_report[:version] != old_package_report[:version]) ||
+          (!cur_package_report.nil? && cur_package_report[:untagged_revisions] != old_package_report[:untagged_revisions])
+        updated_packages << spec unless OPTIONS[:ignore_packages].include?(spec[:name])
+      end
+    end
+
+    updated_packages
+  end
+
+  def self.tito_report
+    packages = {}
+    Vagrant::Openshift::Constants.repos.each do |name, url|
+      Dir.chdir((Vagrant::Openshift::Constants.build_dir + name).to_s) do
+        tito_report = `tito report --untagged-commits`
+        tito_report = tito_report.split(/[-]+{10,100}[\n]/)
+        tito_report.shift
+
+        tito_report.each do |package_report|
+          package_report = package_report.split("\n")
+          m = package_report.shift.match(/([a-z0-9\-]+)-([\d\.\-]+)..HEAD/)
+          packages[m[1]] = {
+              version: m[2],
+              untagged_revisions: package_report.map{ |line| line.split(" ")[0] }
+          }
+        end
+      end
+    end
+    packages
+  end
+end