kondate 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 412540d569ef8a803833d569dc21818df84581db
-  data.tar.gz: 3ed435ac4a9113b348453a19029ab47bb1f56550
+  metadata.gz: cee86599c07e6b8ab7d455f0547e1332be08ce04
+  data.tar.gz: 644a9d48e73a01679e55bec6fae04d8d074b2cb2
 SHA512:
-  metadata.gz: f137334098009c2d3ba28cd6a3f05fcad3638a5b2c3e8ae245f6126976c2cdd2a5817aed3c7e15fff0fbadc4e969a7776accb180d0c9ab201f9ce0efca844334
-  data.tar.gz: 4492126051228c4e52d86350bd5c023eadfdaa1db84160f9098feecc950f5e8ae3111bd8ab27792f950204b68e779506569ce2d05781bba28c18a21f3ee19d81
+  metadata.gz: 330c1056afd6e8ee5723d61f1bacd39bb65357b37f5a44c37be195b16ea5775d3110f23661e06fb5f0386abab07e3f99baefd6f9b97f913cfa598fc67203c6bf
+  data.tar.gz: f832e67244d4326b7cbc381f66062a4ca0f27b9755683a24ea7c879f585608404ce2bd4a3b7c938a2baa7fd23455fa6513dc419c75efef092d439a17a73438e7

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.0 (2016-11-18)
+
+Enhancements:
+
+* Add itamae-role and serverspec-role subcommands to run for multiple hosts in parallel
+
 # 0.2.1 (2016-11-14)
 
 Fixes:

data/README.md

@@ -50,12 +50,12 @@ $ bundle exec kondate serverspec <host>
 ├── bootstrap.rb      # itamae bootstrap
 ├── hosts.yml         # manages hostnames and its roles
 ├── properties        # manages run_lists and attributes
-│   ├── nodes         # host specific properties
+│   ├── nodes         # host specific properties (deprecated)
 │   ├── roles         # role properties
 │   └── environments  # environment properties
 ├── secrets           # manages secrets attributes such as passwords
 │   └── properties
-│       ├── nodes
+│       ├── nodes (deprecated)
 │       ├── roles
 │       └── environments
 ├── recipes           # itamae recipes
@@ -76,13 +76,13 @@ The default .kondate.conf looks like below:
 
 ```
 middlware_recipes_dir: recipes/middleware
-roles_recipes_dir: recipes/roles
+roles_recipes_dir: recipes/roles (deprecated)
 middleware_recipes_serverspec_dir: spec/middleware
 roles_recipes_serverspec_dir: spec/roles
-nodes_properties_dir: properties/nodes
+nodes_properties_dir: properties/nodes (deprecated)
 roles_properties_dir: properties/roles
 environments_properties_dir: properties/environments
-secret_nodes_properties_dir: secrets/properties/nodes
+secret_nodes_properties_dir: secrets/properties/nodes (deprecated)
 secret_roles_properties_dir: secrets/properties/roles
 secret_environments_properties_dir: secrets/properties/environments
 plugin_dir: lib
@@ -174,7 +174,7 @@ Secret properties are places to write confidential attributes.
 ```
 ├── secrets         # manages secrets attributes such as passwords
 │   └── properties
-│       ├── nodes
+│       ├── nodes (deprecated)
 │       ├── roles
 │       └── environments
 ```
@@ -258,7 +258,7 @@ See [templates/spec/spec_helper.rb](./lib/kondate/templates/spec/spec_helper.rb)
 
 ## Host Plugin
 
-The default reads `hosts.yml` to resolve roles of a host, but 
+The default reads `hosts.yml` to resolve roles of a host, but
 you may want to resolve roles from AWS EC2 `roles` tag, or
 you may want to resolve roles from your own host resolver API application.
 
@@ -299,6 +299,15 @@ module Kondate
         super
         raise ConfigError.new('file: path is not configured') unless config.path
         @path = config.path
+
+        @roles_of_hosts = YAML.load_file(@path)
+        @hosts_of_roles = {}
+        @roles_of_hosts.each do |host, roles|
+          roles.each do |role|
+            @hosts_of_roles[role] ||= []
+            @hosts_of_roles[role] << host
+          end
+        end
       end
 
       # @param [String] host hostname
@@ -310,7 +319,15 @@ module Kondate
       # @param [String] host hostname
       # @return [Array] array of roles
       def get_roles(host)
-        YAML.load_file(@path)[host]
+        @roles_of_hosts[host]
+      end
+
+      # @param [String] role role
+      # @return [Array] array of hosts
+      #
+      # Available from kondate >= 0.3.0
+      def get_hosts(role)
+        @hosts_of_roles[role]
       end
 
       # Optional

data/exe/itamae-kondate-role

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+CMD = File.join(File.dirname(__FILE__), 'kondate')
+if ARGV.size == 0 || ARGV[0] == "-h" || ARGV[0] == "--help"
+  exec CMD, 'help', 'itamae-role'
+else
+  exec CMD, 'itamae-role', *ARGV
+end

data/exe/serverspec-kondate-role

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+CMD = File.join(File.dirname(__FILE__), 'kondate')
+if ARGV.size == 0 || ARGV[0] == "-h" || ARGV[0] == "--help"
+  exec CMD, 'help', 'serverspec-role'
+else
+  exec CMD, 'serverspec-role', *ARGV
+end

data/kondate.gemspec

@@ -23,8 +23,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'serverspec'
   spec.add_dependency 'thor'
   spec.add_dependency 'highline'
-  spec.add_dependency "rake"
+  spec.add_dependency 'facter'
+  spec.add_dependency 'parallel'
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "rake"
 end

data/lib/kondate/cli.rb

@@ -1,12 +1,13 @@
 require 'thor'
 require 'yaml'
 require 'net/ssh'
-require 'rspec/core/rake_task'
 require "highline/import"
 require_relative '../kondate'
 require 'fileutils'
 require 'shellwords'
 require 'find'
+require 'facter'
+require 'parallel'
 
 module Kondate
   class CLI < Thor
@@ -55,11 +56,83 @@ module Kondate
     option :profile,                     :type => :string,  :default => nil, :desc => "[EXPERIMENTAL] Save profiling data", :banner => "PATH"
     option :recipe_graph,                :type => :string,  :default => nil, :desc => "[EXPERIMENTAL] Write recipe dependency graph in DOT", :banner => "PATH"
     def itamae(host)
-      builder, property_files = build_property_files(host)
+      property_files = build_property_files(host)
+      if proceed?(property_files)
+        exit(-1) unless do_itamae(host, property_files)
+      end
+    end
+
+    desc "itamae-role <role>", "Execute itamae for multiple hosts in the role"
+    option :role,                         :type => :array,   :default => []
+    option :recipe,                       :type => :array,   :default => []
+    option :debug,   :aliases => ["-d"],  :type => :boolean, :default => false
+    option :confirm,                      :type => :boolean, :default => true
+    option :vagrant,                      :type => :boolean, :default => false
+    option :profile,                      :type => :string,  :default => nil, :desc => "[EXPERIMENTAL] Save profiling data", :banner => "PATH"
+    option :recipe_graph,                 :type => :string,  :default => nil, :desc => "[EXPERIMENTAL] Write recipe dependency graph in DOT", :banner => "PATH"
+    option :parallel, :aliases => ["-p"], :type => :numeric, :default => Facter['processorcount'].value.to_i 
+    def itamae_role(role)
+      $stdout.puts "Number of parallels is #{@options[:parallel]}"
+      hosts = Kondate::Config.host_plugin.get_hosts(role)
+      if hosts.nil? or hosts.empty?
+        $stderr.puts 'No host'
+        exit(1)
+      end
+      $stdout.puts "Target hosts are [#{hosts.join(", ")}]"
+
+      property_files_of_hosts, summarized_property_files, hosts_of_roles = build_property_files_of_hosts(hosts)
+      if proceed?(summarized_property_files, hosts_of_roles)
+        successes = Parallel.map(hosts, in_processes: @options[:parallel]) do |host|
+          do_itamae(host, property_files_of_hosts[host])
+        end
+        exit(-1) unless successes.all?
+      end
+    end
 
+    desc "serverspec <host>", "Execute serverspec"
+    option :role,                        :type => :array,   :default => []
+    option :recipe,                      :type => :array,   :default => []
+    option :debug,   :aliases => ["-d"], :type => :boolean, :default => false
+    option :confirm,                     :type => :boolean, :default => true
+    option :vagrant,                     :type => :boolean, :default => false
+    def serverspec(host)
+      property_files = build_property_files(host)
+      if proceed?(property_files)
+        exit(-1) unless do_serverspec(host, property_files)
+      end
+    end
+
+    desc "serverspec-role <role>", "Execute serverspec for multiple hosts in the role"
+    option :role,                         :type => :array,   :default => []
+    option :recipe,                       :type => :array,   :default => []
+    option :debug,   :aliases => ["-d"],  :type => :boolean, :default => false
+    option :confirm,                      :type => :boolean, :default => true
+    option :vagrant,                      :type => :boolean, :default => false
+    option :parallel, :aliases => ["-p"], :type => :numeric, :default => Facter['processorcount'].value.to_i 
+    def serverspec_role(role)
+      $stdout.puts "Number of parallels is #{@options[:parallel]}"
+      hosts = Kondate::Config.host_plugin.get_hosts(role)
+      if hosts.nil? or hosts.empty?
+        $stderr.puts 'No host'
+        exit(1)
+      end
+      $stdout.puts "Target hosts are [#{hosts.join(", ")}]"
+
+      property_files_of_hosts, summarized_property_files, hosts_of_roles = build_property_files_of_hosts(hosts)
+      if proceed?(summarized_property_files, hosts_of_roles)
+        successes = Parallel.map(hosts, in_processes: @options[:parallel]) do |host|
+          do_serverspec(host, property_files_of_hosts[host])
+        end
+        exit(-1) unless successes.all?
+      end
+    end
+
+    private
+
+    def do_itamae(host, property_files)
+      ENV['RUBYOPT'] = "-I #{Config.plugin_dir} -r bundler/setup -r ext/itamae/kondate"
       property_files.each do |role, property_file|
-        ENV['TARGET_HOST'] = host
-        ENV['RUBYOPT'] = "-I #{Config.plugin_dir} -r bundler/setup -r ext/itamae/kondate"
+        next if property_file.nil?
         command = "bundle exec itamae ssh"
         command << " -h #{host}"
 
@@ -81,70 +154,97 @@ module Kondate
         command << " --recipe-graph=#{@options[:recipe_graph]}" if @options[:recipe_graph]
         command << " bootstrap.rb"
         $stdout.puts command
-        exit(-1) unless system(command)
+        return false unless system(command)
       end
+      true
     end
 
-    desc "serverspec <host>", "Execute serverspec"
-    option :role,                        :type => :array,   :default => []
-    option :recipe,                      :type => :array,   :default => []
-    option :debug,   :aliases => ["-d"], :type => :boolean, :default => false
-    option :confirm,                     :type => :boolean, :default => true
-    option :vagrant,                     :type => :boolean, :default => false
-    def serverspec(host)
-      builder, property_files = build_property_files(host)
-
+    def do_serverspec(host, property_files)
       ENV['RUBYOPT'] = "-I #{Config.plugin_dir} -r bundler/setup -r ext/serverspec/kondate"
       ENV['TARGET_VAGRANT'] = '1' if @options[:vagrant]
       property_files.each do |role, property_file|
-        RSpec::Core::RakeTask.new([host, role].join(':'), :recipe) do |t, args|
-          ENV['TARGET_HOST'] = host
-
-          ENV['TARGET_NODE_FILE'] = property_file
-          recipes = YAML.load_file(property_file)['attributes'].keys.map {|recipe|
-            File.join(Config.middleware_recipes_serverspec_dir, recipe)
-          }.compact
-          recipes << File.join(Config.roles_recipes_serverspec_dir, role)
-          t.pattern = '{' + recipes.join(',') + '}_spec.rb'
-        end
-
-        Rake::Task["#{host}:#{role}"].invoke(@options[:recipe])
+        next if property_file.nil?
+        recipes = YAML.load_file(property_file)['attributes'].keys.map {|recipe|
+          File.join(Config.middleware_recipes_serverspec_dir, recipe)
+        }.compact
+        recipes << File.join(Config.roles_recipes_serverspec_dir, role)
+        spec_files = recipes.map {|recipe| "#{recipe}_spec.rb"}.select! {|spec| File.exist?(spec) }
+
+        command = "TARGET_HOST=#{host.shellescape} TARGET_NODE_FILE=#{property_file.shellescape} bundle exec rspec"
+        command << " #{spec_files.map{|f| f.shellescape }.join(' ')}"
+        $stdout.puts command
+        return false unless system(command)
       end
+      true
     end
 
-    private
+    def proceed?(property_files, hosts_of_roles = {})
+      print_property_files(property_files, hosts_of_roles)
+      if property_files.values.compact.empty?
+        $stderr.puts "Nothing to run"
+        false
+      elsif @options[:confirm]
+        prompt = ask "Proceed? (y/n):"
+        prompt == 'y'
+      else
+        true
+      end
+    end
 
-    def build_property_files(host)
-      builder = PropertyBuilder.new(host)
-      roles   = builder.filter_roles(@options[:role])
+    def print_property_files(property_files, hosts_of_roles = {})
+      roles = property_files.keys
       if roles.nil? or roles.empty?
         $stderr.puts 'No role'
-        exit(1)
+        return
       end
-      $stdout.puts "roles: [#{roles.join(', ')}]"
+      $stdout.puts "Show property files for roles: [#{roles.join(", ")}]"
+
+      property_files.each do |role, property_file|
+        hosts = hosts_of_roles[role]
+        if hosts.nil? # itamae
+          $stdout.print "Show property file for role: #{role}"
+        else # itamae_role
+          $stdout.print "Show representative property file for role: #{role}"
+          $stdout.print " [#{hosts.join(", ")}]"
+        end
+
+        if property_file
+          $stdout.puts
+          $stdout.puts mask_secrets(File.read(property_file))
+        else
+          $stdout.puts " (does not exist, skipped)"
+        end
+      end
+    end
+
+    # @return [Hash] key value pairs whoses keys are roles and values are path (or nil)
+    def build_property_files(host)
+      builder = PropertyBuilder.new(host)
+      roles = builder.filter_roles(@options[:role])
 
       property_files = {}
       roles.each do |role|
         if path = builder.install(role, @options[:recipe])
           property_files[role] = path
-          $stdout.puts "# #{role}"
-          $stdout.puts mask_secrets(File.read(path))
         else
-          $stdout.puts "# #{role} (no attribute, skipped)"
+          property_files[role] = nil
         end
       end
 
-      if property_files.empty?
-        $stderr.puts "Nothing to run"
-        exit(1)
-      end
+      property_files
+    end
 
-      if @options[:confirm]
-        prompt = ask "Proceed? (y/n):"
-        exit(0) unless prompt == 'y'
+    def build_property_files_of_hosts(hosts)
+      summarized_property_files = {}
+      property_files_of_hosts = {}
+      hosts_of_roles = {}
+      hosts.each do |host|
+        property_files = build_property_files(host)
+        property_files_of_hosts[host] = property_files
+        property_files.each {|role, path| summarized_property_files[role] ||= path }
+        property_files.each {|role, path| (hosts_of_roles[role] ||= []) << host }
       end
-
-      [builder, property_files]
+      [property_files_of_hosts, summarized_property_files, hosts_of_roles]
     end
 
     def mask_secrets(str)

data/lib/kondate/host_plugin/base.rb

@@ -20,6 +20,14 @@ module Kondate
         raise NotImplementedError
       end
 
+      # @param [String] role role
+      # @return [Array] array of hosts
+      #
+      # Available from kondate >= 0.3.0
+      def get_hosts(role)
+        raise NotImplementedError
+      end
+
       # @param [String] host hostname
       # @return [Hash] arbitrary host information
       # def get_hostinfo(host)

data/lib/kondate/host_plugin/file.rb

@@ -12,6 +12,15 @@ module Kondate
         super
         raise ConfigError.new('file: path is not configured') unless config.path
         @path = config.path
+
+        @roles_of_hosts = YAML.load_file(@path)
+        @hosts_of_roles = {}
+        @roles_of_hosts.each do |host, roles|
+          roles.each do |role|
+            @hosts_of_roles[role] ||= []
+            @hosts_of_roles[role] << host
+          end
+        end
       end
 
       # @param [String] host hostname
@@ -23,7 +32,15 @@ module Kondate
       # @param [String] host hostname
       # @return [Array] array of roles
       def get_roles(host)
-        YAML.load_file(@path)[host]
+        @roles_of_hosts[host]
+      end
+
+      # @param [String] role role
+      # @return [Array] array of hosts
+      #
+      # Available from kondate >= 0.3.0
+      def get_hosts(role)
+        @hosts_of_roles[role]
       end
     end
   end

data/lib/kondate/property_builder.rb

@@ -136,9 +136,10 @@ module Kondate
       if property['attributes'].empty?
         nil
       else
-        Tempfile.open("kondate_") do |fp|
-          YAML.dump(property, fp)
-        end.path
+        fp = Tempfile.open("kondate_")
+        YAML.dump(property, fp)
+        fp.close
+        fp.path # should be removed when process finishes
       end
     end
   end

data/lib/kondate/version.rb

@@ -1,3 +1,3 @@
 module Kondate
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kondate
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - sonots
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2016-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: itamae
@@ -67,7 +67,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: facter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -108,13 +122,29 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Kondate is yet another nodes management framework for Itamae/Serverspec.
 email:
 - sonots@gmail.com
 executables:
 - itamae-kondate
+- itamae-kondate-role
 - kondate
 - serverspec-kondate
+- serverspec-kondate-role
 extensions: []
 extra_rdoc_files: []
 files:
@@ -130,8 +160,10 @@ files:
 - bin/console
 - bin/setup
 - exe/itamae-kondate
+- exe/itamae-kondate-role
 - exe/kondate
 - exe/serverspec-kondate
+- exe/serverspec-kondate-role
 - kondate.gemspec
 - lib/ext/itamae/attributes.rb
 - lib/ext/itamae/kondate.rb
@@ -183,7 +215,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Kondate is yet another nodes management framework for Itamae/Serverspec.