knife-stackbuilder 0.5.8 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aedb4132378ee68d7c8e56a485173de58d61a428
-  data.tar.gz: 9fa027941863f5f08a5bdc41944256fbf728a348
+  metadata.gz: 7ab807fc026606708f650e207c681ea3ffb82486
+  data.tar.gz: 71a6b26ab5974d2e1c117f04da246f150dcc4375
 SHA512:
-  metadata.gz: db532af7b3a449f18f47724bc7c098440cfd0f0aeda34eb6434701db013603295e3445168f9124cea1b49fc6ec21ae821275275d62b19f40f366101368d5c879
-  data.tar.gz: 72b4d0c1b30a929468d089d8530cd1063904eee27e4bb58b300a64d76fa1bbaa2787305229a79664a9bbfa624fb466e3643eacce75e5f6c403fabc0d7dd23f7c
+  metadata.gz: 53bac33ce0fe0a52524d8dbe36d18aadd34cdecd0a0bbe66cd2bf2ff393f2741cf08f745cde95c34c962e8a6db1e56ebae016a182ae199157bb94d2014985577
+  data.tar.gz: 52a0c878e71ee0f512fa3eb8b6213aa03a24a7772e9e2bc2c99dfc0324168998e2f96b6f6878de2c6040ae197ed22ce84bcc58414269d5a24c08c2b343e00507

data/lib/stackbuilder/chef/stack_container_node.rb

@@ -200,30 +200,26 @@ module StackBuilder::Chef
                             unless ENV['DOCKER_TLS_VERIFY']
                     end
 
-                    echo_output = @logger.info? || @logger.debug?
                     build_exists = @name==`docker images | awk '$1=="#{@name}" { print $1 }'`.strip
 
                     knife_cmd = Chef::Knife::ContainerDockerInit.new
 
                     # Run as a forked job (This captures all output and removes noise from output)
-                    run_jobs(knife_cmd, true, echo_output) do |k|
+                    knife_cmd.name_args = [ @name ]
 
-                        k.name_args = [ @name ]
+                    knife_cmd.config[:local_mode] = false
+                    knife_cmd.config[:base_image] = build_exists ? @name : @knife_config['image']
+                    knife_cmd.config[:force] = true
+                    knife_cmd.config[:generate_berksfile] = false
+                    knife_cmd.config[:include_credentials] = true
 
-                        k.config[:local_mode] = false
-                        k.config[:base_image] = build_exists ? @name : @knife_config['image']
-                        k.config[:force] = true
-                        k.config[:generate_berksfile] = false
-                        k.config[:include_credentials] = true
+                    knife_cmd.config[:dockerfiles_path] = @dockerfiles_build_dir
+                    knife_cmd.config[:run_list] = @knife_config['run_list']
 
-                        k.config[:dockerfiles_path] = @dockerfiles_build_dir
-                        k.config[:run_list] = @knife_config['run_list']
+                    knife_cmd.config[:encrypted_data_bag_secret] = IO.read(@env_key_file) \
+                        unless File.exist? (@env_key_file)
 
-                        k.config[:encrypted_data_bag_secret] = IO.read(@env_key_file) \
-                            unless File.exist? (@env_key_file)
-
-                        run_knife(k)
-                    end
+                    run_knife_forked(knife_cmd)
 
                     dockerfiles_named_path = @dockerfiles_build_dir + '/' + @name
 
@@ -282,21 +278,16 @@ module StackBuilder::Chef
                     # Run the build as a forked job (This captures all output and removes noise from output)
                     knife_cmd = Chef::Knife::ContainerDockerBuild.new
 
-                    job_results = run_jobs(knife_cmd, true, echo_output) do |k|
-
-                        k.name_args = [ @name ]
+                    knife_cmd.name_args = [ @name ]
 
-                        k.config[:run_berks] = false
-                        k.config[:force_build] = true
-                        k.config[:dockerfiles_path] = @dockerfiles_build_dir
-                        k.config[:cleanup] = true
-
-                        run_knife(k)
-                    end
+                    knife_cmd.config[:run_berks] = false
+                    knife_cmd.config[:force_build] = true
+                    knife_cmd.config[:dockerfiles_path] = @dockerfiles_build_dir
+                    knife_cmd.config[:cleanup] = true
 
-                    result = job_results[knife_cmd.object_id][0]
-                    if result.rindex('Chef run process exited unsuccessfully (exit code 1)') ||
-                        result.rindex(/The command \[.*\] returned a non-zero code:/)
+                    begin
+                        results = run_knife_forked(knife_cmd)
+                    rescue Exception => msg
 
                         if @logger.level>=::Logger::WARN
 
@@ -306,10 +297,10 @@ module StackBuilder::Chef
                             %x(docker rmi -f #{@name})
 
                             puts "Knife container build Chef convergence failed with an error."
-                            puts "#{job_results.first[0]}"
+                            puts "#{results[0]}"
                         end
 
-                        raise StackBuilder::Common::StackBuilderError, "Docker build of container #{@name} has errors."
+                        raise msg
                     end
 
                     puts 'Saving docker image for upload. This may take a few minutes.'

data/lib/stackbuilder/chef/stack_generic_node.rb

@@ -6,7 +6,7 @@ module StackBuilder::Chef
 
     class GenericNodeManager < StackBuilder::Chef::NodeManager
 
-        def create_vm(name, knife_config)
+        def create_vm(index, name, knife_config)
 
             create_class_name = knife_config['create']['class']
             raise ArgumentError, "Knife plugin's server 'create' class name not provided." \
@@ -21,16 +21,27 @@ module StackBuilder::Chef
                 knife_cmd.name_args = [ name ]
             end
 
+            if knife_config['create'].has_key?('pool_key')
+
+                pool_key = knife_config['create']['pool_key']
+
+                placement_pools = knife_config['placement_pools']
+                raise ArgumentError, "Knife plugin 'placement_pools' list was not provided." \
+                    if placement_pools.nil?
+
+                knife_cmd.config[pool_key.to_sym] = placement_pools[index % placement_pools.size]
+            end            
+
             config_knife(knife_cmd, knife_config['create']['options'] || { })
             config_knife(knife_cmd, knife_config['options'] || { })
 
             if knife_config['create']['synchronized']
                 @@sync ||= Mutex.new
                 @@sync.synchronize {
-                    run_knife(knife_cmd, knife_config['create']['retries'] || 0)
+                    run_knife_forked(knife_cmd)
                 }
             else
-                run_knife(knife_cmd, knife_config['create']['retries'] || 0)
+                run_knife_forked(knife_cmd)
             end
         end
 
@@ -59,10 +70,10 @@ module StackBuilder::Chef
             if knife_config['delete']['synchronized']
                 @@sync ||= Mutex.new
                 @@sync.synchronize {
-                    run_knife(knife_cmd, knife_config['delete']['retries'] || 0)
+                    run_knife_forked(knife_cmd)
                 }
             else
-                run_knife(knife_cmd, knife_config['delete']['retries'] || 0)
+                run_knife_forked(knife_cmd)
             end
         end
     end

data/lib/stackbuilder/chef/stack_node_manager.rb

@@ -67,13 +67,22 @@ module StackBuilder::Chef
         def create(index)
 
             name = "#{@node_id}-#{index}"
-            self.create_vm(name, @knife_config)
+            self.create_vm(index, name, @knife_config)
 
             node = Chef::Node.load(name)
             node.normal['stack_id'] = @id
             node.normal['stack_node'] = @name
             node.save
 
+            begin
+                # Wait for node to become available
+                node_search("name:#{name}", StackBuilder::Common::Config.timeouts[:QUERY_TIMEOUT])                
+
+            rescue Exception => msg
+                raise StackBuilder::Common::StackBuilderError, \
+                    "Error waiting for node named '#{name} to be indexed in Chef Server: #{msg}"
+            end
+
             unless @env_key_file.nil?
                 env_key = IO.read(@env_key_file)
                 knife_ssh( name,
@@ -88,7 +97,7 @@ module StackBuilder::Chef
             raise msg
         end
 
-        def create_vm(name, knife_config)
+        def create_vm(index, name, knife_config)
             raise StackBuilder::Common::NotImplemented, 'HostNodeManager.create_vm'
         end
 
@@ -135,7 +144,7 @@ module StackBuilder::Chef
                     "exit $result" )
             else
                 node = Chef::Node.load(name)
-                attributes.each { |k,v| node.override[k] = v }
+                attributes.each { |k,v| node.normal[k] = v }
                 node.save
             end
 
@@ -199,16 +208,9 @@ module StackBuilder::Chef
 
         def get_stack_node_resources
 
-            query = Chef::Search::Query.new
-
-            escaped_query = URI.escape(
-                "stack_id:#{@id} AND stack_node:#{@name}",
-                Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") )
-
-            results = query.search('node', escaped_query, nil, 0, 999999)
+            results = node_search("stack_id:#{@id} AND stack_node:#{@name}")
             @nodes = results[0]
-
-            results[2]
+            @nodes.size
         end
 
         def knife_ssh(name, cmd)
@@ -235,11 +237,32 @@ module StackBuilder::Chef
                 error = StackBuilder::Common::TeeIO.new($stderr)
 
                 @logger.info("Running '#{cmd}' on node 'name:#{name}'.")
-                run_knife(knife_cmd, 0, output, error)
+                run_knife(knife_cmd, output, error)
             else
                 run_knife(knife_cmd)
             end
         end
 
+        def node_search(search_query, timeout = 0)
+
+            query = Chef::Search::Query.new
+            escaped_query = URI.escape(search_query, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+
+            if timeout>0
+                @logger.info("Waiting '#{search_query}' to return results.")
+                results = Timeout::timeout(timeout) {
+
+                    while true do
+                        results = query.search('node', escaped_query, nil, 0, 999999)
+                        return results if results[0].size>0
+                    end
+                }
+            else
+                results = query.search('node', escaped_query, nil, 0, 999999)
+            end
+
+            results
+        end
+
     end
 end

data/lib/stackbuilder/chef/stack_provider.rb

@@ -37,7 +37,7 @@ module StackBuilder::Chef
             @id = id
 
             stack_environment = stack['environment']
-            raise ArgmentError, "Stack file is fixed to the environment '#{stack_environment}', " +
+            raise ArgumentError, "Stack file is fixed to the environment '#{stack_environment}', " +
                 " which it does not match the environment '#{@environment}' provided." \
                 unless stack_environment.nil? || stack_environment==@environment
 

data/lib/stackbuilder/chef/stack_vagrant_node.rb

@@ -6,7 +6,7 @@ module StackBuilder::Chef
 
     class VagrantNodeManager < StackBuilder::Chef::NodeManager
 
-        def create_vm(name, knife_config)
+        def create_vm(index, name, knife_config)
 
             handle_vagrant_box_additions(name, knife_config)
 
@@ -39,12 +39,15 @@ module StackBuilder::Chef
 
             @@sync ||= Mutex.new
             @@sync.synchronize {
-                run_knife(knife_cmd)
+                run_knife_forked(knife_cmd)
             }
         end
 
         def delete_vm(name, knife_config)
 
+            puts "deleting #{name}"
+            exit 1
+
             knife_cmd = Chef::Knife::VagrantServerDelete.new
             knife_cmd.name_args = [ name ]
             knife_cmd.config[:yes] = true
@@ -52,7 +55,7 @@ module StackBuilder::Chef
 
             @@sync ||= Mutex.new
             @@sync.synchronize {
-                run_knife(knife_cmd, 3)
+                run_knife_forked(knife_cmd)
             }
 
             handle_vagrant_box_cleanup(knife_config)

data/lib/stackbuilder/common/config.rb

@@ -4,6 +4,7 @@ module StackBuilder::Common
     
     class Config
 
+        QUERY_TIMEOUT = 60
         CACHE_TIMEOUT = 60
 
         class << self
@@ -18,6 +19,7 @@ module StackBuilder::Common
             # Determine timeouts
             config.timeouts = { } if config.timeouts.nil?
             config.timeouts[:CACHE_TIMEOUT] = CACHE_TIMEOUT unless config.timeouts.has_key?(:CACHE_TIMEOUT)
+            config.timeouts[:QUERY_TIMEOUT] = QUERY_TIMEOUT unless config.timeouts.has_key?(:QUERY_TIMEOUT)
             
             # Create cache folder
             config.enable_caching = false if config.enable_caching.nil?

data/lib/stackbuilder/common/helpers.rb

@@ -47,6 +47,8 @@ module StackBuilder::Common
                             previous_stderr, $stderr = $stderr, stderr
                             yield(job)
                             Marshal.dump([stdout.string, stderr.string], write)
+                        rescue Exception => msg
+                            Marshal.dump([stdout.string, stderr.string, msg], write)
                         ensure
                             $stdout = previous_stdout
                             $stderr = previous_stderr
@@ -369,42 +371,37 @@ module StackBuilder::Common
         end
 
         #
-        # Helper command to run Chef knife
+        # Helper commands to run Chef knife
         #
-        def run_knife(knife_cmd, retries = 0, output = StringIO.new, error = StringIO.new)
+        def run_knife(knife_cmd, output = StringIO.new, error = StringIO.new)
 
             knife_cmd.ui = Chef::Knife::UI.new(output, error, STDIN, knife_cmd.config) \
                 unless output.nil? && error.nil?
 
-            run = true
-            while run
-
-                begin
-                    knife_cmd.run
-                    run = false
+            knife_cmd.run
+            output.string
+        end
 
-                rescue Exception => msg
+        #
+        # Helper to run one or more Chef knife commands as a forked jobs
+        #
+        def run_knife_forked(*knife_cmd)
 
-                    if retries==0
+            logger = StackBuilder::Common::Config.logger
 
-                        if @logger.level>=::Logger::WARN
-                            puts "Knife execution failed with an error."
-                            puts "* StdOut from knife run: #{output.string}"
-                            puts "* StdErr from knife run: #{error.string}"
-                        end
+            job_results = run_jobs(knife_cmd, true, logger.info? || logger.debug?) do |k|
+                run_knife(k)
+            end
 
-                        @logger.debug(msg.backtrace.join("\n\t")) if Config.logger.debug?
-                        raise msg
-                    end
+            results = knife_cmd.collect do |k| 
 
-                    @logger.debug("Knife command #{knife_cmd} failed. Retrying after 2s.")
+                result = job_results[k.object_id]
+                raise result[2] unless result[2].nil?
 
-                    sleep 2
-                    retries -= 1
-                end
+                result[0]
             end
 
-            output.string
+            (results.size==1 ? results[0] : results)
         end
 
         #

data/lib/stackbuilder/stack/node_task.rb

@@ -211,29 +211,17 @@ module StackBuilder::Stack
 
             threads = [ ]
 
-            scale = (@deleted ? @manager.get_scale : @scale)
             if @targets.empty?
 
-                if scale > 0
-
-                    if @sync == "first"
-                        @manager.process(scale, events, self.parse_attributes(@attributes, 0))
-                        scale -= 1
-                    end
-
-                    if @sync == "all"
-                        scale.times do |i|
-                            @manager.process(i, events, self.parse_attributes(@attributes, i))
-                        end
-                    else
-                        scale.times do |i|
-                            spawn_processing(i, events, threads)
-                        end
-                    end
+                scale = (@deleted ? @manager.get_scale : @scale)
+                scale.times do |i|
+                    spawn_processing(i, events, threads)
                 end
             else
                 @targets.each do |t|
-                    t.scale.times do |i|
+                    
+                    scale = (@deleted ? t.manager.get_scale : t.scale)
+                    scale.times do |i|
                         spawn_processing(i, events, threads, t)
                     end
                 end
@@ -284,12 +272,13 @@ module StackBuilder::Stack
                 "#{orchestrate_events.collect { |e| e } .join(", ")}") if @logger.debug?
 
             if @sync==SYNC_ALL || (i==0 && @sync==SYNC_FIRST)
+                @resource_sync[i].wait if target.nil?
                 @manager.process(i, orchestrate_events, parse_attributes(@attributes, i), target_manager)
             else
-                @resource_sync[i].wait if target.nil?
                 threads << Thread.new {
 
                     begin
+                        @resource_sync[i].wait if target.nil?
                         @manager.process(i, orchestrate_events, parse_attributes(@attributes, i), target_manager)
 
                     rescue Exception => msg

data/lib/stackbuilder/version.rb

@@ -2,7 +2,7 @@
 
 module Knife
     module StackBuilder
-        VERSION = "0.5.8"
+        VERSION = "0.5.9"
         MAJOR, MINOR, TINY = VERSION.split('.')
     end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-stackbuilder
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
 platform: ruby
 authors:
 - Mevan Samaratunga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-06 00:00:00.000000000 Z
+date: 2015-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef