fhcap-cli 0.4.9 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1685de7948d976df40d6c15e0870a51f12393fc2
-  data.tar.gz: 4a812cdd3b1e2865105e0ee69ffc55e74f82881b
+  metadata.gz: 1b016a16a0016627aec93bbdfc076803d288ee32
+  data.tar.gz: d33df34b6a28e16b580c6399f1a748295a0e9a8a
 SHA512:
-  metadata.gz: f4ce224936b910162e7fcf0b2bd10f0f0d415a4fd35c523cebabdbfe860ccc74b9fc0d31c19290e088064e59926224be8cd0063df2c0658c5cdfafc6ee95fe0b
-  data.tar.gz: 7e0917eceb7b16367d7c71683f949c8bfd6c4e2398a5e8d7608b5e866507b5f12cf10c173442321eb40fc4e837c0e481159462a7bc29687253024309bf7bccc0
+  metadata.gz: 2926b8a7452e6faac59e8559e91189f37950a34138abeb258623898ec934d6beb91fbb23f4b12dfb72fa779244f636b321fcd7e2204a0331e7a050fd7d49074b
+  data.tar.gz: d1b00a5098e14a1fdea3284a0a3d47f72a2254699763a63dc8692c826eca3b4f75eda6e85eb447546a63477f5a6980101968f7660390aa637d076b45b90797a7

data/CHANGELOG.md

@@ -1,6 +1,19 @@
 
 ## Unreleased
 
+## 0.4.10
+
+* Disable the dynovm bind9 cache in all single node environments by default.
+* [RHMAP-11771] - Fix kitchen generate using a role (fhcap kitchen generate -r fhcap -d vagrant).
+* Add ability to override test suite directory via an env var (TEST_SUITE_DIR=/tmp fhcap kitchen generate).
+* Use purge option on knife download command and stop manually deleting local files.
+* Pin nokogiri gem to version 1.6.8.1 (Can be removed when we require a ruby version >= 2.1.0)
+* [RHMAP-11675] - Add low level chef provisioning ssh provider. Note: This does not include generate support for the 'ssh' provider type.
+* [RHMAP-11546] - Update kitchen config templates to include additional platforms (centos-6.5 and macosx-10.10 ).
+*  Ensure all thor errors result in a non zero exit status. Optionally allow run commands to exit on failure and update kitchen commands to exit on failure with non zero status.
+* [RHMAP-11003] - Update kitchen config templates to include all supported platforms (rhel-7.2, centos-7 and ubuntu-12.04 ).
+* [RHMAP-11001] - Updated test kitchen generation task to use the contents of the cookbook under tests kitchen.yml when generating the platforms section of the kitchen config.
+
 ## 0.4.9
 
 * [RHMAP-9021] - Add support for longer AWS Instance and ELB Ids (Patched with upstream changes made in chef-provisioning-aws gem)

data/Dockerfile

@@ -0,0 +1,37 @@
+FROM ruby:2.3
+
+# Consumed by atomic cli: http://www.projectatomic.io/docs/usr-bin-atomic/
+# E.g. atomic run tools/fhcap cluster list # where the image name is tools/fhcap
+LABEL RUN="docker run -ti --rm -e LOCAL_USER_ID=\$SUDO_UID -v "\$HOME"/fhcapd:/home/fh/fhcap -v "\$HOME"/fhcap/repos:/home/fh/fhcap/repos -v "\$HOME"/src/fheng:/home/fh/fhcap/src -v "\$HOME"/.ssh:/home/fh/.ssh IMAGE"
+
+WORKDIR /usr/src/app
+
+RUN curl -sSL https://get.docker.com/ | sh
+
+RUN apt-get update && apt-get -y --no-install-recommends install \
+    ca-certificates \
+    curl
+
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.4/gosu-$(dpkg --print-architecture)" \
+    && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.4/gosu-$(dpkg --print-architecture).asc" \
+    && gpg --verify /usr/local/bin/gosu.asc \
+    && rm /usr/local/bin/gosu.asc \
+    && chmod +x /usr/local/bin/gosu
+
+COPY Gemfile* fhcap-cli.gemspec /usr/src/app/
+COPY bin/fhcap /usr/src/app/bin/
+COPY lib/fhcap/version.rb /usr/src/app/lib/fhcap/
+
+RUN gem install bundler && \
+    bundle install
+
+COPY . /usr/src/app
+
+RUN chmod 777 /usr/src/app/Gemfile.lock
+
+VOLUME /home/fh/fhcap /home/fh/fhcap/repos /home/fh/fhcap/src /home/fh/.ssh
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

data/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Add local user (fh)
+# Either use the LOCAL_USER_ID if passed in at runtime or
+# fallback
+
+USER_ID=${LOCAL_USER_ID:-9001}
+
+useradd --shell /bin/bash -u $USER_ID -o -c "" fh
+
+exec /usr/local/bin/gosu fh fhcap "$@"

data/fhcap-cli.gemspec

@@ -16,10 +16,12 @@ Gem::Specification.new do |spec|
   spec.required_rubygems_version = '>= 2.6.3'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.bindir       = "bin"
+  spec.executables  = %w{fhcap}
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'nokogiri', '= 1.6.8.1'
   spec.add_dependency 'thor', '= 0.19.1'
   spec.add_dependency 'sshkey', '= 1.6.1'
   spec.add_dependency 'public_suffix', '= 1.5.2'
@@ -33,16 +35,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'java_properties', '= 0.0.4'
   spec.add_dependency 'archive-tar-minitar', '= 0.5.2'
   spec.add_dependency 'aws-sdk', '= 2.1.35'
+  spec.add_dependency 'ffi-yajl', '= 2.2.3'
+  spec.add_dependency 'rack', '= 1.6.4'
   spec.add_dependency 'chef', '= 12.4.3'
   spec.add_dependency 'chef-provisioning', '= 1.4.1'
   spec.add_dependency 'chef-provisioning-aws', '= 1.4.1'
   spec.add_dependency 'chef-provisioning-vagrant', '= 0.10.0'
   spec.add_dependency 'chef-provisioning-fog', '= 0.15.0'
+  spec.add_dependency 'chef-provisioning-ssh', '= 0.0.9'
   spec.add_dependency 'test-kitchen', '= 1.4.2'
   spec.add_dependency 'kitchen-sync', '= 2.1.1'
-  spec.add_dependency 'kitchen-vagrant', '= 0.14.0'
-  spec.add_dependency 'kitchen-ec2', '= 0.8.0'
-  spec.add_dependency 'kitchen-docker', '= 2.1.0'
+  spec.add_dependency 'kitchen-vagrant', '= 0.16.0'
+  spec.add_dependency 'kitchen-ec2', '= 1.2.0'
+  spec.add_dependency 'kitchen-docker', '= 2.6.0'
   spec.add_dependency 'kitchen-openstack','= 1.8.1'
 
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/cookbooks/provision/libraries/provision.rb

@@ -88,7 +88,21 @@ def openstack_bootstrap_options(org_name, environment, instance_options)
 end
 
 def openstack_converge_options(org_name, environment, instance_options)
-  {:chef_version => '11.10.4', :ohai_hints => { :openstack => {} }}
+  {:chef_version => '11.10.4', :ohai_hints => {:openstack => {}}}
+end
+
+#SSH
+
+def ssh_machine_options(org_name, environment, instance_name, instance_options)
+  {}
+end
+
+def ssh_bootstrap_options(org_name, environment, instance_options)
+  {}
+end
+
+def ssh_converge_options(org_name, environment, instance_options)
+  {:chef_version => '11.10.4'}
 end
 
 def key_pair_name_for(name)
@@ -146,7 +160,7 @@ def with_cluster_config(node, &block)
   block.call cluster_config
 end
 
-def with_cluster_instances(node,  &block)
+def with_cluster_instances(node, &block)
   env_regexp = node[:'env-regexp'] || '.'
   node_regexp = node[:'node-regexp'] || '.'
   with_cluster_config(node) do |cluster_config|

data/lib/cookbooks/provision/recipes/ssh.rb

@@ -0,0 +1,25 @@
+require 'chef/provisioning/ssh_driver'
+
+cluster_config = cluster_config_for(node)
+org_name = cluster_config[:id]
+
+repo_dir_name = File.basename(node[:local_repo_path])
+tmp_repo_dir = File.join('/tmp', repo_dir_name)
+
+link tmp_repo_dir do
+  to node[:local_repo_path]
+end
+
+node.override[:local_repo_path] = tmp_repo_dir
+
+repo_ssh_nodes_dir = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'ssh_nodes')
+local_key_pairs_dir = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'key_pairs')
+key_pair_name = key_pair_name_for(org_name)
+
+node.default[:machine_options][:transport_options]['ssh_options']['keys'] = [File.join(local_key_pairs_dir, "#{key_pair_name}.pem")]
+
+directory repo_ssh_nodes_dir do
+  recursive true
+end
+
+with_driver "ssh:#{repo_ssh_nodes_dir}"

data/lib/cookbooks/provision/recipes/ssh_cluster_create.rb

@@ -0,0 +1,4 @@
+include_recipe 'provision::common'
+include_recipe "provision::ssh"
+
+include_recipe "provision::cluster_create_instances"

data/lib/cookbooks/provision/recipes/ssh_cluster_destroy.rb

@@ -0,0 +1,4 @@
+include_recipe 'provision::common'
+include_recipe "provision::ssh"
+
+include_recipe "provision::cluster_destroy_instances"

data/lib/extensions/thor/actions.rb

@@ -0,0 +1,33 @@
+class Thor
+  module Actions
+    def run(command, config = {})
+      return unless behavior == :invoke
+
+      destination = relative_to_original_destination_root(destination_root, false)
+      desc = "#{command} from #{destination.inspect}"
+
+      if config[:with]
+        desc = "#{File.basename(config[:with].to_s)} #{desc}"
+        command = "#{config[:with]} #{command}"
+      end
+
+      say_status :run, desc, config.fetch(:verbose, true)
+
+      unless options[:pretend]
+        if config[:capture]
+          result = `#{command}`
+          status = $?.exitstatus
+        else
+          result = system("#{command}")
+          status = result ? 0 : 1
+        end
+
+        if config[:exit_on_failure]
+          status == 0 or exit status
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/fhcap.rb

@@ -3,7 +3,7 @@ require "pathname"
 
 module Fhcap
   GEM_DIR = File.expand_path '..', File.dirname(__FILE__)
-  TEMPLATE_NAMES = %w{single core-3node mbaas-3node core-mbaas-6node core-small-9node nginx-test single-blank farm-3node farm-single ose-single}.sort
+  TEMPLATE_NAMES = %w{single core-3node mbaas-3node core-mbaas-6node core-small-9node nginx-test single-blank farm-3node farm-5node farm-single ose-single}.sort
 
   class << self
     def source_root
@@ -11,4 +11,4 @@ module Fhcap
     end
   end
 
-end
+end

data/lib/fhcap/dns.rb

@@ -51,10 +51,10 @@ module Fhcap
       desc "list", "List DNS Records"
 
       def list
-        require "fhcap/tasks/dns/list"
-        Tasks::Dns::List.new(task_options(options.dup)).run
+        require "fhcap/tasks/dns/list_records"
+        Tasks::Dns::ListRecords.new(task_options(options.dup)).run
       end
 
     end
   end
-end
+end

data/lib/fhcap/kitchen.rb

@@ -3,6 +3,7 @@ require 'fhcap/thor_base'
 
 require 'pathname'
 require 'yaml'
+require 'deep_merge'
 
 module Fhcap
   module CLI
@@ -29,7 +30,7 @@ module Fhcap
 
       def generate
         require 'chef'
-        cookbooks = get_cookbooks(options, nil, repo_cookbook_paths('-cookbooks'))
+        cookbooks = get_cookbooks(options, [], repo_cookbook_paths('-cookbooks'))
         roles = options[:roles] || []
         driver = options[:driver]
         provisioner = options[:provisioner]
@@ -53,6 +54,15 @@ module Fhcap
             cookbook_kitchen_yaml = File.join(meta.root_dir, ".kitchen.yml")
             if File.exists? cookbook_kitchen_yaml
               cookbook_kitchen_config = YAML.load_file(cookbook_kitchen_yaml)
+
+              if cookbook_kitchen_config['platforms']
+                cookbook_kitchen_config['platforms'].map! do |platform|
+                  default_values = kitchen_config['platforms'].detect {|f| f["name"] == platform['name']} || {}
+                  default_values.merge(platform)
+                end
+                kitchen_config['platforms'] = cookbook_kitchen_config['platforms']
+              end
+
               cookbook_kitchen_config['suites'].each do |suite|
                 suite['name'] = "#{suite['name']}_#{cookbook}"
                 suite['run_list'].unshift 'recipe[feedhenry_utils::kitchen_docker]' if driver == 'docker'
@@ -87,22 +97,46 @@ module Fhcap
           if File.exists? role_file
             ::Chef::Config[:role_path] = File.join(repo_dir(:fhcap), 'roles')
             role = ::Chef::Role.from_disk r
-            run_list = role.run_list.expand('development')
+            run_list = role.run_list.expand('development', 'disk')
 
             role_cookbooks = run_list.recipes.collect do |recipe|
               recipe.split('::').first
             end.compact.uniq
 
-            suite = {}
-            suite['name'] = "default_#{r}"
-            suite['run_list'] = ["role[#{r}]"]
-            kitchen_config['suites'] << suite
+            suite_attributes = {}
+            suite_run_list = []
+            suite_run_list << 'recipe[feedhenry_utils::kitchen_docker]' if driver == 'docker'
+            suite_run_list << 'recipe[apt::default]'
+            suite_run_list << 'recipe[chef-solo-search]' if provisioner == 'chef_solo'
+            suite_run_list << 'role[test-kitchen_server]'
+            suite_run_list << "role[#{r}]"
 
             role_cookbooks.each do |c|
               get_cookbook_deps(c, cookbook_dependencies, {cwd: test_suite_dir})
+              meta = cookbook_loader.cookbooks_by_name[c]
+              if meta
+                cookbook_kitchen_yaml = File.join(meta.root_dir, ".kitchen.yml")
+                if File.exists? cookbook_kitchen_yaml
+                  cookbook_kitchen_config = YAML.load_file(cookbook_kitchen_yaml)
+                  default_values = cookbook_kitchen_config['suites'].detect {|f| f["name"] == 'default'} || {}
+                  default_attributes = default_values['attributes'] || {}
+                  suite_attributes.deep_merge!(default_attributes)
+                end
+              end
             end
 
-            link_cookbook_tests(role_cookbooks, options, r)
+            # The env overrides in some of the wrapper cookbooks cause issues when using roles, for now we will just remove them entirely.
+            if suite_attributes['feedhenry_common']
+              suite_attributes['feedhenry_common'].delete('env')
+            end
+
+            suite = {}
+            suite['name'] = "default_#{r}"
+            suite['run_list'] = suite_run_list
+            suite['attributes'] = suite_attributes
+            kitchen_config['suites'] << suite
+
+            link_cookbook_tests(role_cookbooks, options, r, 'default')
 
           else
             say "Unknown Role #{role_file}"
@@ -117,6 +151,15 @@ module Fhcap
 
         #3. Configure tests
         link_cookbook_tests(cookbooks, options)
+
+        #4. Add custom driver config
+        if driver == 'docker'
+          template(File.join("templates", "kitchen", "Dockerfile-rhel"), File.join(test_suite_dir, "Dockerfile-rhel"), {}, :force => true)
+          template(File.join("templates", "kitchen", "docker_id_rsa"), File.join(test_suite_dir, "docker_id_rsa"), {}, :force => true)
+          template(File.join("templates", "kitchen", "docker_id_rsa.pub"), File.join(test_suite_dir, "docker_id_rsa.pub"), {}, :force => true)
+        elsif driver == 'vagrant'
+          template(File.join("templates", "kitchen", "Vagrantfile-osx.rb"), File.join(test_suite_dir, "Vagrantfile-osx.rb"), {}, :force => true)
+        end
       end
 
       desc "clean", "Clean"
@@ -137,10 +180,11 @@ module Fhcap
           kitchen_options = options[:'kitchen-options'] || {}
           Fhcap::CLI::Kitchen.new.invoke(:generate, [], options) if options[:cookbooks] || options[:modified] || options[:roles]
           if is_kitchen_generated?
-            Dir.chdir test_suite_dir do
+            tsd = test_suite_dir
+            Dir.chdir tsd do
               cmd_options = options.dup
               cmd_options['instance'] = instance
-              cmd_options['kitchen-yaml'] = test_suite_kitchen_yml
+              cmd_options['kitchen-yaml'] = test_suite_kitchen_yml(tsd)
               cmd_options['kitchen-options'] = kitchen_options_for(
                   default_kitchen_options.merge(
                       {:'log-level' => options[:'log-level']}
@@ -173,11 +217,11 @@ module Fhcap
       end
 
       def test_suite_dir
-        File.join(config.default_dir, 'tmp', 'kitchen')
+        ENV['TEST_SUITE_DIR'] ? File.expand_path(ENV['TEST_SUITE_DIR']) : File.join(config.default_dir, 'tmp', 'kitchen')
       end
 
-      def test_suite_kitchen_yml
-        File.join(test_suite_dir, 'kitchen.yml')
+      def test_suite_kitchen_yml(tsd=test_suite_dir)
+        File.join(tsd, 'kitchen.yml')
       end
 
       def is_kitchen_generated?
@@ -194,7 +238,7 @@ module Fhcap
         end
       end
 
-      def link_cookbook_tests(cookbooks, options, role=nil)
+      def link_cookbook_tests(cookbooks, options, role=nil, suite='.', type='.')
         tests_parent_dir = File.join(test_suite_dir, 'test', 'integration')
         test_helpers_dir = File.join(repo_dir(:fhcap), 'test', 'helpers')
 
@@ -205,10 +249,15 @@ module Fhcap
             cookbook_test_suite_dir = File.join(cookbook_dir, '/test/integration')
 
             if Dir.exists? cookbook_test_suite_dir
-              Dir.entries(cookbook_test_suite_dir).select { |f| !File.directory? f }.each() do |test_suite|
+              #feedhenry_common/test/integration
+
+              Dir.entries(cookbook_test_suite_dir).select { |f| !File.directory? f }.select { |f| f =~ /#{suite}/ }.each() do |test_suite|
+                #default
 
                 cookbook_test_dir = File.join(cookbook_test_suite_dir, test_suite)
-                Dir.entries(cookbook_test_dir).select { |f| !File.directory? f }.each() do |test_type|
+                Dir.entries(cookbook_test_dir).select { |f| !File.directory? f }.select { |f| f =~ /#{type}/ }.each() do |test_type|
+                  #serverspec
+
                   tests_dir = File.join(tests_parent_dir, "#{test_suite}_#{role || cookbook}")
 
                   test_type_dir = "#{tests_dir}/#{test_type}"
@@ -250,7 +299,7 @@ module Fhcap
       end
 
       def run_cmd(cmd, options)
-        run(cmd)
+        run(cmd, exit_on_failure: true)
       end
 
     end

data/lib/fhcap/knife_helper.rb

@@ -80,8 +80,6 @@ module Fhcap
         block.call
       ensure
         thor.say "[Chef Zero Server] - Syncing nodes to #{local_repo}"
-        FileUtils.rm_rf(File.join(local_repo, 'nodes'))
-        FileUtils.rm_rf Dir.glob(File.join(local_repo, 'data_bags/aws_*'))
         knife_download(local_repo, 'local', ['/nodes'])
         knife_download(local_repo, 'local', ['/data_bags/aws_*'])
       end
@@ -116,6 +114,7 @@ module Fhcap
           knife_command.config[:chef_repo_path] = repo
           knife_command.configure_chef
           knife_command.config[:recurse] = true
+          knife_command.config[:purge] = true
           knife_command.run
         end
       end

data/lib/fhcap/providers_helper.rb

@@ -20,6 +20,7 @@ module Fhcap
 
     def provider_config(id)
       provider_type, provider_id = id.split(':')
+      return {} if provider_type == 'ssh'
       providers_config[provider_type.to_sym][provider_id.to_sym]
     end
 

data/lib/fhcap/repos_helper.rb

@@ -175,7 +175,7 @@ module Fhcap
 
     def modified?(path, since='origin/master')
       stat = Dir.chdir(path) do
-        `git diff #{since} -- #{path}`
+        `git diff #{since}...HEAD -- #{path}`
       end
       !stat.empty?
     end