knife-stackbuilder 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 209a8ea8c3e0ab99bbe754f70003ed242e435052
-  data.tar.gz: 7e17e996e03499a4e9c7432ccd7354780d392b7f
+  metadata.gz: 278edc9ab005e30e5b5da72cd13e7e28a49df655
+  data.tar.gz: 0ee5c8dc89f98d4cfdcd6d0db3cd040789fa2a52
 SHA512:
-  metadata.gz: 65bc78b0ed9f4ae0a8a27f95734f5c3b84632c6d39188d8e3025e692051f43b04e2dd973e485308e89be641b7aa4d28dc616bcc3d40cc8852fb231a571d6cfcc
-  data.tar.gz: d5b245863b3a171884b481668482de92bdddcb49d10a36bdbf1db48d00093237c520fe44c9318edffbc33742460adbcfde0ba73186f8ba2e877d49825113a15e
+  metadata.gz: 463be1cca68ecdc686bb7f4e985b2e7f79a8e443605b142078a1fc2be0891327fa68b35be85a8283fdda7750ff789b5eaa38b299fa631b96eeba4de5cd2dc11a
+  data.tar.gz: 9336cb38c175d9cc6daaf14b70883adb13038c7ec1e7d323754e44b651d70309731ea908d85bdcd5f917b6f37a5f5e2e6592c182a33bbe2cd92922121830e340

data/README.md

@@ -1,80 +1,119 @@
 # Knife StackBuilder plugin
 
-## Usage
+Knife StackBuilder is a Chef Knife tool that can be used to orchestrate configuration across multiple nodes. It
+evolved from the need to simplify using Chef to build a clustered application services environment such as OpenStack.
+The plugin was built to:
 
-```
-knife stack initialize repo
-
-  Initializes or validates an existing stack repo. The stack repo should contain the 
-  following folders along with a Berksfile.
-  
-  * cookbooks
-  * environments
-  * secrets
-  * databags
-  * roles
-  * stacks
-
-  --path
-
-    Path to create and initialize the stack chef repository. If the repository already 
-    exists then any it will be validated. If the provided path begins with git:, http: 
-    or https:, it will be assumed to be a git repo. A branch/label may be specified
-    by preceding it with a : after the path.
-    
-    i.e. http://github.com/mevansam/mystack:tag1
-
-  --cert_path | --certs
-
-    If "--cert_path" is specified then it should point to a directory with a folder for 
-    each server domain name containing that server's certificate files. 
-    
-    If instead "--certs" are specified then it should provide a comma separated list of
-    server domain names for which self-signed certificates will be generated.
-
-  --envs
-    
-    Comma separated list of environments to generate.
-    
-  --cookbooks
-  
-    A comma separated list of cookbooks and their versions to be added to the Berksfile.
-    
-    i.e. "mysql:=5.6.1, wordpress:~> 2.3.0"
-```
+1. Describe a complex system topology using a YAML file
+2. Leverage knife cloud plugins to bootstrap cloud, virtual and baremetal nodes within the topology
+3. Leverage knife container to build and deploy docker containers using Chef cookbooks
+4. Re-use cookbooks from the [Chef Supermarket](http://supermarket.getchef.com)
+5. Leverage the Berkshelf workflow and not re-invent the wheel for developing Chef cookbooks
+6. Normalize the Chef environment and provide a means to externalize and parameterize configuration values
 
-```
-knife stack upload cookbook[s]
-```
+The plugin is very similar to Ansible and Saltstack, but is meant to be Chef centric. It you plan is to not use Chef
+cookbooks for configuration management, then this is not the tool for you. It differs from Chef metal in that the
+orchestration is driven by a set of directives captured as a YAML file. The advantage of describing the build in a
+YAML file is that it is easier to transform the topology description to another format such as Heat or Bosh if a
+decision is made down the road to move to a different infrastructure orchestration approach.
 
-```
-knife stack upload environment[s]
-```
+Check out the brief [tutorial](docs/how-to.md) on setting up a repository for a single node wordpress stack and building
+it. The [OpenStack HA Cookbook](https://github.com/mevansam/openstack-ha-cookbook) contains examples where the plugin
+is used to setup mult-node OpenStack environments in Vagrant, VMware etc. using the OpenStack StackForge cookbooks.
+
+## Overview
+
+The plugin extends the standard cookbook repository upload capabilities to provide an extensive variable substituion
+capability. This is done to enable templatizing the Chef artifacts to model a system which can be manipulated by
+variables in environment specific YAML files in the '```etc/```' folder which in turn can be overridden by shell
+variables.
+
+### Chef and Berkshelf Cookbook Repository Management
+
+This is nothing more than a wrapper of existing Chef and Berkshelf repository functionality. However, it adds a couple
+of key features that are helpful when externalizing and securing the environment for Chef.
+
+* Cookbooks
+
+    '```knife stack upload cookbooks```' simply invokes Berkshelf to upload the cookbooks specified in the Berksfile.
+
+* Data Bags and Encryption
+
+    '```knife stack upload data bags```' will upload the data bags found within the '```data_bags/```' folder. Folders
+    at the top-level of that folder will considered to be the data bags with the json files within them, the data bag
+    item and its content. A data bag instance will be created for each environment and encrypted with an environment
+    specific key found in the '```secrets/```' folder. So a data bag name will have the format '```[data bag
+    name]-[environment]```'.
+
+    Data bag content can be parameterized with the environment specific YAML file in the '```etc/```' folder. This
+    simplifies the handling of environment specific settings/secrets by externalizing them. Within a data bag folder
+    creating a content file within an environment specific folder will override any item content at the parent level.
+
+* Roles
+
+    '```knife stack upload roles```' will upload the roles within the '```roles/```'. This is similar to uploading
+    roles via the standard knife role method. However if required, role content can be paremeterized by referencing
+    shell environment variables.
+
+### Externalizing configuration values and order of evaluation
+
+As mentioned previously this plugin parameterizes the Chef environment in the '```environments/```' folder using a YAML
+file having the same name as the environment in the '```etc/```' folder. This same file is used to parameterize the
+stack file that describe the system topology. The YAML environment file can in turn be parameterized by pulling in
+values from the shell environment
+
+For example the following will propagate a value from the shell to the rest of the stack and Chef envrionment. Since ruby string variable expansion is used it is possible to reference '```ENV```' to pull shell environment directly into any YAML or JSON configuration file. You can reference a key-value in the yaml that has already been parsed via '```#{my['some key']}```'.
+
+In shell:
 
 ```
-knife stack upload role[s]
+export DOMAIN=knife-stackbuilder-dev.org
 ```
 
+in ```./etc/DEV.yml```:
+
 ```
-knife stack upload data bag[s]
+---
+domain: "#{ENV['DOMAIN']}"
+.
+.
 ```
 
+in ```./stack.yml```:
+
 ```
-knife stack upload repo
+---
+# Stack
+name: Stack1
+environment: DEV
+domain: "#{env['domain']}"
+.
+.
 ```
 
+in ```environments/DEV.rb```:
+
 ```
-knife stack build
+---
+name "DEV"
+description "Chef 'DEV' environment."
+env = YAML.load_file(File.expand_path('../../etc/DEV.yml', __FILE__))
+override_attributes(
+    'domain' => "#{env['domain']}",
+.
+.
 ```
 
-## Design
+The following diagram illustrates the relationships between the files in the repository and how they are
+parameterized.
 
-### Berkshelf Cookbook Repository Management
-
-### Externalizing configuration values and order of evaluation
+![Image of OpenStack HA Configuration File Structure]
+(docs/images/config_files.png)
 
 #### Requesting user input for non-persisted values
 
+
+
 #### Including common yaml configurations
 
 #### Processing node attributes
@@ -87,8 +126,10 @@ knife stack build
 
 ## To Do:
 
-* Use Chef Pushy instead of Knife SSH
-* Add option to execute Chef Push jobs on events
+* Use Chef Pushy instead of Knife SSH and add option to execute Chef Push jobs on events
+* Make encrypted data bag handling more robust using Use Chef Vault
+* The repo needs to detect changes to cookbooks, roles, data bags etc and upload only the changes
+* Load custom provider gems by inspecting the installed gems
 
 ## Contributing
 
@@ -112,4 +153,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-Author: Mevan Samaratunga (mevansam@gmail.com)
+Author | Email | Company
+-------|-------|--------
+Mevan Samaratunga | msamaratunga@pivotal.io | [Pivotal](http://www.pivotal.io)
+

data/lib/chef/knife/stack_build.rb

@@ -73,6 +73,10 @@ class Chef
 
                     env_vars = provider.get_env_vars
                     stack = StackBuilder::Common.load_yaml(stack_file, env_vars)
+                    merge_maps( stack, stack_overrides.end_with?('.json') ?
+                        JSON.load(File.new(stack_overrides, 'r')) : JSON.load(stack_overrides) ) \
+                        unless stack_overrides.nil?
+
                     puts("Stack file:\n#{stack.to_yaml}")
 
                 else

data/lib/stackbuilder/chef/repo.rb

@@ -89,6 +89,9 @@ module StackBuilder::Chef
                 # Create or copy certs
                 create_certs(certificates) unless certificates.nil?
             end
+
+            @build_path = @repo_path + '/.build'
+            FileUtils.mkdir_p(@build_path)
         end
 
         def upload_environments(environment = nil)
@@ -101,7 +104,7 @@ module StackBuilder::Chef
                 # TODO: Handle JSON environment files. JSON files should be processed similar to roles.
 
                 env_file = "#{@repo_path}/environments/#{env_name}.rb"
-                FileUtils.touch(env_file)
+                # FileUtils.touch(env_file)
 
                 knife_cmd.name_args = [ env_file ]
                 run_knife(knife_cmd)
@@ -411,7 +414,7 @@ module StackBuilder::Chef
                 @logger.debug("Uploading data bag '#{data_bag_item_name}' with contents:\n#{data_bag_item.to_yaml}")
 
                 tmpfile = "#{Dir.tmpdir}/#{data_bag_item_name}.json"
-                File.open("#{tmpfile}", 'w+') { |f| f.write(data_bag_item.to_json) }
+                File.open(tmpfile, 'w+') { |f| f.write(data_bag_item.to_json) }
 
                 knife_cmd = Chef::Knife::DataBagFromFile.new
                 knife_cmd.name_args = [ data_bag_name, tmpfile ]
@@ -439,7 +442,7 @@ module StackBuilder::Chef
             @logger.debug("Uploading role '#{role_name}' with contents:\n#{role_content.to_yaml}")
 
             tmpfile = "#{Dir.tmpdir}/#{role_name}.json"
-            File.open("#{tmpfile}", 'w+') { |f| f.write(role_content.to_json) }
+            File.open(tmpfile, 'w+') { |f| f.write(role_content.to_json) }
 
             knife_cmd = Chef::Knife::RoleFromFile.new
             knife_cmd.name_args = [ tmpfile ]

data/lib/stackbuilder/chef/stack_container_node.rb

@@ -12,20 +12,177 @@ module StackBuilder::Chef
 
             @env_file_path = repo_path + '/environments/' + environment + '.rb'
 
-            docker_image_dir = repo_path + '/.docker_images'
+            @dockerfiles_build_dir = repo_path + '/.build/docker/build'
+            FileUtils.mkdir_p(@dockerfiles_build_dir)
+
+            docker_image_dir = repo_path + '/.build/docker/' + environment
             FileUtils.mkdir_p(docker_image_dir)
-            @docker_image_path = docker_image_dir + '/' + @name + '.gz'
+
+            docker_image_filename = docker_image_dir + '/' + @name
+            @docker_image_path = docker_image_filename + '.gz'
+            @docker_image_target = docker_image_filename + '.target'
         end
 
         def process(index, events, attributes, target = nil)
 
+            return unless events.include?('create') || events.include?('install') || 
+                events.include?('configure') || events.include?('update')
+
+            target_node_instance = "#{target.node_id}-#{index}"
+            node = Chef::Node.load(target_node_instance)
+            ipaddress = node.attributes['ipaddress']
+
+            ssh = ssh_create(ipaddress, target.ssh_user,
+                target.ssh_password.nil? ? target.ssh_identity_file : target.ssh_password)
+
+            image_exists = @name==ssh_exec!(ssh, "docker images | awk '$1==\"@name\" { print $1 }'")[:out].strip
+
+            # Copy image file to target if it has changed or does not exist on target
+            if build_container(attributes) && !target.nil? &&
+                ( !File.exist?(@docker_image_target) ||
+                (File.mtime(@docker_image_path) > File.mtime(@docker_image_target)) ||
+                !image_exists )
+
+                puts "Uploading docker image to target '#{target_node_instance}'."
+                ssh.open_channel do |channel|
+
+                    channel.exec('gunzip | sudo docker load') do |ch, success|
+                        channel.on_data do |ch, data|
+                            res << data
+                        end
+
+                        channel.send_data IO.binread(@docker_image_path)
+                        channel.eof!
+                    end
+                end
+                ssh.loop
+
+                FileUtils.touch(@docker_image_target)
+            end
+
+            # Start container instances
+            if @knife_config.has_key?('container_start')
+
+                result = ssh_exec!(ssh, "sudo docker ps -a | awk '/#{@node_id}/ { print $0 }'")
+                raise StackBuilder::Common::StackBuilderError, "Error determining running containers for #{@name}: #{result[:err]}" \
+                    if result[:exit_code]>0
+
+                running_instances = result[:out].lines
+                if running_instances.size > @scale
+
+                    (running_instances.size - 1).downto(@scale) do |i|
+
+                        container_node_id = "#{@node_id}-#{i}"
+                        running_instance = running_instances.select{ |ri| ri[/#{@node_id}-\d+/,0]==container_node_id }
+                        container_id = running_instance.first[/^[0-9a-z]+/,0]
+
+                        result = ssh_exec!(ssh, "sudo docker rm -f #{container_id}")
+
+                        if result[:exit_code]==0
+
+                            remove_container_node_from_chef(container_node_id)
+
+                            container_port_map = Hash.new
+                            container_port_map.merge!(node.normal['container_port_map']) \
+                                unless node.normal['container_port_map'].nil?
+
+                            container_port_map.each do |k,v|
+                                container_port_map[k].delete(container_node_id)
+                            end
+
+                            node.normal['container_port_map'] = container_port_map
+                            node.save
+                        else
+                            @logger.error("Unable to stop container instance #{running_instances[i]}: #{result[:err]}")
+                        end
+                    end
+
+                elsif running_instances.size < @scale
+
+                    container_start = @knife_config['container_start']
+                    container_ports = container_start['ports']
+                    container_options = container_start['options']
+
+                    start_cmd = "sudo docker run -d "
+
+                    start_cmd += container_options + ' ' \
+                        unless container_options.nil?
+
+                    start_cmd += container_ports.collect \
+                        { |k,p| "-p :#{p=~/\d+\:\d+/ ? p.to_s : ':' + p.to_s}" }.join(' ') \
+                        unless container_ports.nil?
+
+                    running_instances.size.upto(@scale - 1) do |i|
+
+                        container_node_id = "#{@node_id}-#{i}"
+                        remove_container_node_from_chef(container_node_id)
+
+                        result = ssh_exec!( ssh,
+                            "#{start_cmd} --name #{container_node_id} " +
+                            "-h #{container_node_id} -e \"CHEF_NODE_NAME=#{container_node_id}\" #{@name}")
+
+                        if result[:exit_code]==0
+
+                            container_id = container_node_id
+
+                            container_port_map = Hash.new
+                            container_port_map.merge!(node.normal['container_port_map']) \
+                                unless node.normal['container_port_map'].nil?
+
+                            container_ports.each do |k,p|
+
+                                port_map = container_port_map[k]
+                                if port_map.nil?
+
+                                    port_map = { }
+                                    container_port_map[k] = port_map
+                                end
+
+                                if p=~/\d+\:\d+/
+                                    port_map[container_id] = p[/(\d+)\:\d+/, 1]
+                                else
+                                    result = ssh_exec!(ssh, "sudo docker port #{container_node_id} #{p}")
+
+                                    @logger.error( "Unable to get host port for " +
+                                        "'#{@node_id}-#{i}:#{p}': #{result[:err]}") \
+                                        if result[:exit_code]>0
+
+                                    port_map[container_id] = result[:out][/:(\d+)$/, 1]
+                                end
+                            end
+
+                            node.normal['container_port_map'] = container_port_map
+                            node.save
+                        else
+                            @logger.error("Unable to start container instance '#{@node_id}-#{i}': #{result[:err]}")
+                        end
+                    end
+                end
+
+            end
+
+            super(index, events, attributes, target)
+        end
+
+        def delete(index)
+
+            super(index)
+        end
+
+        private
+
+        def build_container(attributes)
+
             @@sync ||= Mutex.new
             @@sync.synchronize {
 
                 unless @build_complete ||
-                    (File.exist?(@docker_image_path) && File.exist?(@env_file_path) && \
+                    ( File.exist?(@docker_image_path) && File.exist?(@env_file_path) && \
                     File.mtime(@docker_image_path) > File.mtime(@env_file_path) )
 
+                    %x(docker images)
+                    raise ArgumentError, "Docker does not appear to be available." unless $?.success?
+
                     if is_os_x? || !is_nix_os?
 
                         raise ArgumentError, "DOCKER_HOST environment variable not set." \
@@ -36,121 +193,147 @@ module StackBuilder::Chef
                             unless ENV['DOCKER_TLS_VERIFY']
                     end
 
-                    begin
-                        build_role = Chef::Role.new
-                        build_role.name(@name + '_build')
-                        build_role.override_attributes(attributes)
-                        build_role.save
+                    echo_output = @logger.info? || @logger.debug?
+                    build_exists = @name==`docker images | awk '$1=="#{@name}" { print $1 }'`.strip
 
-                        dockerfiles_path = File.join(Dir.home, '/.knife/container')
+                    knife_cmd = Chef::Knife::ContainerDockerInit.new
 
-                        build_exists = @name==`docker images | awk '/#{@name}/ { print $1 }'`.strip
+                    # Run as a forked job (This captures all output and removes noise from output)
+                    run_jobs(knife_cmd, true, echo_output) do |k|
 
-                        knife_cmd = Chef::Knife::ContainerDockerInit.new
-                        knife_cmd.name_args = [ @name ]
+                        k.name_args = [ @name ]
 
-                        knife_cmd.config[:local_mode] = false
-                        knife_cmd.config[:base_image] = build_exists ? @name : @knife_config['image']
-                        knife_cmd.config[:force] = true
-                        knife_cmd.config[:generate_berksfile] = false
-                        knife_cmd.config[:include_credentials] = false
+                        k.config[:local_mode] = false
+                        k.config[:base_image] = build_exists ? @name : @knife_config['image']
+                        k.config[:force] = true
+                        k.config[:generate_berksfile] = false
+                        k.config[:include_credentials] = true
 
-                        knife_cmd.config[:dockerfiles_path] = dockerfiles_path
-                        knife_cmd.config[:run_list] = @knife_config['run_list'] + [ "role[#{build_role.name}]" ]
+                        k.config[:dockerfiles_path] = @dockerfiles_build_dir
+                        k.config[:run_list] = @knife_config['run_list']
 
-                        knife_cmd.config[:encrypted_data_bag_secret] = IO.read(@env_key_file) \
+                        k.config[:encrypted_data_bag_secret] = IO.read(@env_key_file) \
                             unless File.exist? (@env_key_file)
 
-                        run_knife(knife_cmd)
+                        run_knife(k)
+                    end
+
+                    dockerfiles_named_path = @dockerfiles_build_dir + '/' + @name
+
+                    # Create env key to add to the docker image
+                    FileUtils.cp(@env_key_file, dockerfiles_named_path + '/chef/encrypted_data_bag_secret')
+
+                    if @knife_config.has_key?('inline_dockerfile')
 
-                        if @knife_config.has_key?('inline_dockerfile')
+                        dockerfile_file = dockerfiles_named_path + '/Dockerfile'
+                        dockerfile = IO.read(dockerfile_file).lines
 
-                            dockerfile_path = dockerfiles_path + "/#{@name}/Dockerfile"
-                            docker_file = IO.read(dockerfile_path).lines
+                        dockerfile_new = [ ]
+
+                        log_level = (
+                            @logger.debug? ? 'debug' :
+                            @logger.info? ? 'info' :
+                            @logger.warn? ? 'warn' :
+                            @logger.error? ? 'error' :
+                            @logger.fatal? ? 'fatal' : 'error' )
+
+                        while dockerfile.size>0
+                            l = dockerfile.delete_at(0)
+
+                            if l.start_with?('RUN chef-init ')
+                                # Ensure node builds with the correct Chef environment and attributes
+                                dockerfile_new << l.chomp + " -E #{@environment} -l #{log_level}\n"
+
+                            elsif l.start_with?('CMD ')
+                                # Ensure node starts within the correct Chef environment and attributes
+                                dockerfile_new << l.gsub(/\"\]/,"\",\"-E #{@environment}\"]")
+
+                            else
+                                dockerfile_new << l
 
-                            docker_file_new = [ ]
-                            while docker_file.size>0
-                                l = docker_file.delete_at(0)
-                                docker_file_new << l
                                 if l.start_with?('FROM ')
-                                    docker_file_new += @knife_config['inline_dockerfile'].lines.map { |ll| ll.strip + "\n" }
-                                    break
+                                    # Insert additional custom Dockerfile build steps
+                                    dockerfile_new += @knife_config['inline_dockerfile'].lines.map { |ll| ll.strip + "\n" }
                                 end
                             end
-                            docker_file_new += docker_file
-
-                            File.open(dockerfile_path, 'w+') { |f| f.write(docker_file_new.join) }
                         end
+                        dockerfile_new += dockerfile
 
-                        knife_cmd = Chef::Knife::ContainerDockerBuild.new
-                        knife_cmd.name_args = [ @name ]
+                        File.open(dockerfile_file, 'w+') { |f| f.write(dockerfile_new.join) }
+                    end
 
-                        knife_cmd.config[:run_berks] = false
-                        knife_cmd.config[:force_build] = true
-                        knife_cmd.config[:dockerfiles_path] = dockerfiles_path
-                        knife_cmd.config[:cleanup] = true
+                    # Update first boot json file with attributes and services
+                    first_boot_file = dockerfiles_named_path + '/chef/first-boot.json'
+                    first_boot = JSON.load(File.new(first_boot_file, 'r')).to_hash
+                    first_boot.merge!(attributes)
 
-                        result = run_knife(knife_cmd)
+                    first_boot['container_service'] = @knife_config['container_services'] \
+                        if @knife_config.has_key?('container_services')
 
-                    ensure
-                        build_role.destroy unless build_role.nil?
-                    end
+                    File.open(first_boot_file, 'w+') { |f| f.write(first_boot.to_json) }
 
-                    # TODO: Errors are currently not detected as knife-container sends all chef-client output to stdout
-                    if result.rindex('Chef run process exited unsuccessfully (exit code 1)')
+                    # Run the build as a forked job (This captures all output and removes noise from output)
+                    knife_cmd = Chef::Knife::ContainerDockerBuild.new
 
-                        if @logger.level>=::Logger::WARN
-                            puts "Knife execution failed with an error."
-                            puts "#{result.string}"
-                        end
+                    job_results = run_jobs(knife_cmd, true, echo_output) do |k|
 
-                        `for i in $(docker ps -a | awk '/chef-in/ { print $1 }'); do docker rm -f $i; done`
-                        `for i in $(docker images | awk '/<none>/ { print $3 }'); do docker rmi $i; done`
+                        k.name_args = [ @name ]
 
-                        raise StackBuilderError, 'Container build has errors.'
+                        k.config[:run_berks] = false
+                        k.config[:force_build] = true
+                        k.config[:dockerfiles_path] = @dockerfiles_build_dir
+                        k.config[:cleanup] = true
+
+                        run_knife(k)
                     end
 
-                    `docker save #{@name} | gzip -9 > #{@docker_image_path}`
-                end
-                @build_complete = true
-            }
+                    result = job_results[knife_cmd.object_id][0]
+                    if result.rindex('Chef run process exited unsuccessfully (exit code 1)') ||
+                        result.rindex(/The command \[.*\] returned a non-zero code:/)
 
-            if @build_complete && !target.nil?
-
-                node = Chef::Node.load("#{target.node_id}-#{index}")
-                ipaddress = node.attributes['ipaddress']
-
-                if target.ssh_password.nil?
-                    ssh = Net::SSH.start(ipaddress, target.ssh_user,
-                        {
-                              :key_data => IO.read(target.ssh_identity_file),
-                              :user_known_hosts_file => "/dev/null"
-                        } )
-                else
-                    ssh = Net::SSH.start(ipaddress, target.ssh_user,
-                        {
-                              :password => target.ssh_password,
-                              :user_known_hosts_file => "/dev/null"
-                        } )
-                end
+                        if @logger.level>=::Logger::WARN
 
-                ssh.open_channel do |channel|
+                            %x(for i in $(docker ps -a | awk '/Exited \(\d+\)/ { print $1 }'); do docker rm -f $i; done)
+                            %x(for i in $(docker ps -a | awk '/chef-in/ { print $1 }'); do docker rm -f $i; done)
+                            %x(docker rmi -f $(docker images -q --filter "dangling=true"))
+                            %x(docker rmi -f #{@name})
 
-                    channel.exec('gunzip | sudo docker load') do |ch, success|
-                        channel.on_data do |ch, data|
-                            res << data
+                            puts "Knife container build Chef convergence failed with an error."
+                            puts "#{job_results.first[0]}"
                         end
 
-                        channel.send_data IO.binread(@docker_image_path)
-                        channel.eof!
+                        raise StackBuilder::Common::StackBuilderError, "Docker build of container #{@name} has errors."
                     end
+
+                    puts 'Saving docker image for upload. This may take a few minutes.'
+                    out = %x(docker save #{@name} | gzip -9 > #{@docker_image_path})
+
+                    raise StackBuilder::Common::StackBuilderError, \
+                        "Unable to save docker container #{@name}: #{out}" unless $?.success?
                 end
-                ssh.loop
-            end
+            }
 
-            super(index, events, attributes, target)
+            @build_complete = true
         end
 
+        def remove_container_node_from_chef(container_node_id)
+
+            begin
+                node = Chef::Node.load(container_node_id)
+                @logger.info("Deleting container node reference in Chef: #{container_node_id}")
+                node.destroy
+            rescue
+                # Do Nothing
+            end
+
+            begin
+                client = Chef::ApiClient.load(container_node_id)
+                @logger.info("Deleting container api client reference in Chef: #{container_node_id}")
+                client.destroy
+            rescue
+                # Do Nothing
+            end
+        end
 
     end
 end

data/lib/stackbuilder/chef/stack_node_manager.rb

@@ -26,6 +26,8 @@ module StackBuilder::Chef
             @name = node_config['node']
             @node_id = @name + '-' + @id
 
+            @environment = environment
+
             @run_list = node_config.has_key?('run_list') ? node_config['run_list'].join(',') : nil
             @run_on_event = node_config['run_on_event']
 
@@ -67,15 +69,10 @@ module StackBuilder::Chef
             name = "#{@node_id}-#{index}"
             self.create_vm(name, @knife_config)
 
-            knife_cmd = KnifeAttribute::Node::NodeAttributeSet.new
-            knife_cmd.name_args = [ name, 'stack_id', @id ]
-            knife_cmd.config[:type] = 'override'
-            run_knife(knife_cmd)
-
-            knife_cmd = KnifeAttribute::Node::NodeAttributeSet.new
-            knife_cmd.name_args = [ name, 'stack_node', @name ]
-            knife_cmd.config[:type] = 'override'
-            run_knife(knife_cmd)
+            node = Chef::Node.load(name)
+            node.normal['stack_id'] = @id
+            node.normal['stack_node'] = @name
+            node.save
 
             unless @env_key_file.nil?
                 env_key = IO.read(@env_key_file)
@@ -116,8 +113,6 @@ module StackBuilder::Chef
                 run_on_event = node_manager.run_on_event
             end
 
-            set_attributes(name, attributes)
-
             if (events.include?('configure') || events.include?('update')) && !run_list.nil?
 
                 log_level = (
@@ -138,6 +133,10 @@ module StackBuilder::Chef
                     "result=$?\n" +
                     "rm -f $TMPFILE\n" +
                     "exit $result" )
+            else
+                node = Chef::Node.load(name)
+                attributes.each { |k,v| node.override[k] = v }
+                node.save
             end
 
             run_on_event.each_pair { |event, cmd|
@@ -212,21 +211,6 @@ module StackBuilder::Chef
             results[2]
         end
 
-        def set_attributes(name, attributes, key = nil)
-
-            attributes.each do |k, v|
-
-                if v.is_a?(Hash)
-                    set_attributes(name, v, key.nil? ? k : key + '.' + k)
-                else
-                    knife_cmd = KnifeAttribute::Node::NodeAttributeSet.new
-                    knife_cmd.name_args = [ name, (key.nil? ? k : key + '.' + k), v.to_s ]
-                    knife_cmd.config[:type] = 'override'
-                    run_knife(knife_cmd)
-                end
-            end
-        end
-
         def knife_ssh(name, cmd)
 
             sudo = @knife_config['options']['sudo'] ? 'sudo -i su -c ' : ''

data/lib/stackbuilder/chef/stack_vagrant_node.rb

@@ -8,6 +8,8 @@ module StackBuilder::Chef
 
         def create_vm(name, knife_config)
 
+            handle_vagrant_box_additions(name, knife_config)
+
             knife_cmd = Chef::Knife::VagrantServerCreate.new
 
             knife_cmd.config[:chef_node_name] = name
@@ -53,6 +55,8 @@ module StackBuilder::Chef
                 run_knife(knife_cmd, 3)
             }
 
+            handle_vagrant_box_cleanup(knife_config)
+
         rescue Exception => msg
 
             if Dir.exist?(knife_cmd.config[:vagrant_dir] + '/' + name)
@@ -68,5 +72,79 @@ module StackBuilder::Chef
                 end
             end
         end
+
+        def handle_vagrant_box_additions(name, knife_config)
+
+            # Create add-on provider specific infrastructure
+            knife_options = knife_config['options']
+            provider = knife_options['provider']
+
+            if !provider.nil? && provider.start_with?('vmware')
+
+                vmx_customize = knife_options['vmx_customize']
+                unless vmx_customize.nil?
+
+                    # Build additional disks that will be added to
+                    # the VMware fusion/desktop VM when booted.
+
+                    disks = {}
+                    vagrant_disk_path = File.join(Dir.home, '/.vagrant/disks') + '/' + name
+                    FileUtils.mkdir_p(vagrant_disk_path)
+
+                    vmx_customize.split(/::/).each do |p|
+
+                        kv = p.split('=')
+                        k = kv[0].gsub(/\"/, '').strip
+                        v = kv[1].gsub(/\"/, '').strip
+
+                        if k.start_with?('scsi')
+                            ss = k.split('.')
+                            disks[ss[0]] ||= {}
+                            case ss[1]
+                                when 'fileName'
+                                    if v.start_with?('/')
+                                        disks[ss[0]]['fileName'] = v
+                                    else
+                                        vv = vagrant_disk_path + '/' + v
+                                        vmx_customize.gsub!(/#{v}/, vv)
+                                        disks[ss[0]]['fileName'] = vv
+                                    end
+                                when 'fileSize'
+                                    disks[ss[0]]['fileSize'] = v
+                            end
+                        end
+                    end
+
+                    # Create extra disks as unlike virtual box VMware fusion/workstation
+                    # will not create disks automatically based on configuration params
+
+                    vdiskmgr = %x(which vmware-vdiskmanager)
+                    vdiskmgr = "/Applications/VMware Fusion.app/Contents/Library/vmware-vdiskmanager" \
+                        if is_os_x? && vdiskmgr.empty?
+
+                    if File.exist?(vdiskmgr)
+
+                        run_jobs(disks.values) do |f|
+
+                            disk = f['fileName']
+                            @logger.info("Creating disk #{disk}.")
+
+                            %x("#{vdiskmgr}" -c -t 0 -s #{f['fileSize']} -a ide #{disk}) \
+                                unless File.exist?(f['fileName'])
+
+                            raise StackBuilder::Common::StackBuilderError, "Disk #{disk} could not be created." \
+                                unless File.exist?(disk)
+                        end
+                    else
+                        raise StackBuilder::Common::StackBuilderError,
+                            "Unable to determine path to vmware-vdiskmanager" +
+                            "to create the requested additional disk."
+                    end
+                end
+            end
+        end
+
+        def handle_vagrant_box_cleanup(knife_config)
+        end
     end
 end

data/lib/stackbuilder/common/helpers.rb

@@ -19,23 +19,68 @@ module StackBuilder::Common
         #
         # Runs the given execution list asynchronously if fork is supported
         #
-        def exec_forked(exec_list)
-            
-            if is_nix_os
-                p = []
-                exec_list.each do |data|
-                    p << fork {
-                        yield(data)
+        def run_jobs(jobs, wait = true, echo = false)
+
+            jobs = [ jobs ] unless jobs.is_a?(Array)
+            job_handles = { }
+
+            if is_nix_os?
+
+                jobs.each do |job|
+
+                    read, write = IO.pipe
+
+                    pid = fork {
+
+                        read.close
+
+                        if echo
+                            stdout = StackBuilder::Common::TeeIO.new($stdout)
+                            stderr = StackBuilder::Common::TeeIO.new($stderr)
+                        else
+                            stdout = StringIO.new
+                            stderr = StringIO.new
+                        end
+
+                        begin
+                            previous_stdout, $stdout = $stdout, stdout
+                            previous_stderr, $stderr = $stderr, stderr
+                            yield(job)
+                            Marshal.dump([stdout.string, stderr.string], write)
+                        ensure
+                            $stdout = previous_stdout
+                            $stderr = previous_stderr
+                        end
                     }
+                    write.close
+
+                    job_handles[job.object_id] = [ pid, read ]
                 end
-                p.each { |pid| Process.waitpid(pid) }
+            end
+
+            if wait
+                wait_jobs(job_handles)
             else
-                exec_list.each do |data|
-                    yield(data)
-                    printf("\n")
-                end
+                job_handles
             end
-            
+        end
+
+        #
+        # This should be called after run_jobs() with the returned handles
+        # if you want to wait for the forked jobs to complete and retrieve
+        # the results.
+        #
+        def wait_jobs(job_handles)
+
+            job_results = { }
+            job_handles.each do |job_id, handle|
+
+                result = Marshal.load(handle[1])
+                Process.waitpid(handle[0])
+                job_results[job_id] = result
+            end
+
+            job_results
         end
 
         #
@@ -324,7 +369,7 @@ module StackBuilder::Common
         end
 
         #
-        # Helper command to rin Chef knife
+        # Helper command to run Chef knife
         #
         def run_knife(knife_cmd, retries = 0, output = StringIO.new, error = StringIO.new)
 
@@ -385,5 +430,71 @@ module StackBuilder::Common
             $stderr = previous_stderr
         end
 
+        # Creates and ssh session to the given host using the given credentials
+        def ssh_create(host, user, key)
+
+            if key.start_with?('-----BEGIN RSA PRIVATE KEY-----')
+                ssh = Net::SSH.start(host, user,
+                    {
+                        :key_data => key,
+                        :user_known_hosts_file => "/dev/null"
+                    } )
+            elsif File.exist?(key)
+                ssh = Net::SSH.start(host, user,
+                    {
+                        :key_data => IO.read(key),
+                        :user_known_hosts_file => "/dev/null"
+                    } )
+            else
+                ssh = Net::SSH.start(host, user,
+                    {
+                        :password => key,
+                        :user_known_hosts_file => "/dev/null"
+                    } )
+            end
+
+            ssh
+        end
+
+        # Executes a remote shell command and returns exit status
+        def ssh_exec!(ssh, command)
+
+            stdout_data = ""
+            stderr_data = ""
+            exit_code = nil
+            exit_signal = nil
+
+            ssh.open_channel do |channel|
+                channel.exec(command) do |ch, success|
+                    unless success
+                        abort "FAILED: couldn't execute command (ssh.channel.exec)"
+                    end
+                    channel.on_data do |ch,data|
+                        stdout_data+=data
+                    end
+
+                    channel.on_extended_data do |ch,type,data|
+                        stderr_data+=data
+                    end
+
+                    channel.on_request("exit-status") do |ch,data|
+                        exit_code = data.read_long
+                    end
+
+                    channel.on_request("exit-signal") do |ch, data|
+                        exit_signal = data.read_long
+                    end
+                end
+            end
+            ssh.loop
+
+            {
+                :out => stdout_data,
+                :err => stderr_data,
+                :exit_code => exit_code,
+                :exit_signal => exit_signal
+            }
+        end
+
     end
 end

data/lib/stackbuilder/common/teeio.rb

@@ -5,25 +5,16 @@ module StackBuilder::Common
     #
     # Sends data written to an IO object to multiple outputs.
     #
-    class TeeIO < IO
+    class TeeIO < StringIO
 
         def initialize(output = nil)
-            @string_io = StringIO.new
+            super()
             @output = output
         end
 
-        def tty?
-            return false
-        end
-
         def write(string)
-            @string_io.write(string)
+            super(string)
             @output.write(string) unless @output.nil?
         end
-
-        def string
-            @string_io.string
-        end
-
     end
 end

data/lib/stackbuilder/stack/node_manager.rb

@@ -31,7 +31,6 @@ module StackBuilder::Stack
         end
 
         def delete(index)
-            raise StackBuilder::Common::NotImplemented, 'NodeManager.delete'
         end
 
     end

data/lib/stackbuilder/stack/node_task.rb

@@ -49,22 +49,14 @@ module StackBuilder::Stack
                     @sync = SYNC_NONE
             end
 
-            if node_config.has_key?('targets')
-
-                @logger.warn("Ignoring 'scale' attribute for '#{@name}' as that node has targets.") \
-                    if node_config.has_key?("scale")
-
-                @scale = 0
+            current_scale = manager.get_scale
+            if current_scale==0
+                @scale = (node_config.has_key?("scale") ? node_config["scale"] : 1)
             else
-                current_scale = manager.get_scale
-                if current_scale==0
-                    @scale = (node_config.has_key?("scale") ? node_config["scale"] : 1)
-                else
-                    @scale = current_scale
-                end
-
-                raise ArgumentError, "The scale for node \"#{@name}\" must be greater than 0." if @scale < 1
+                @scale = current_scale
             end
+
+            raise ArgumentError, "The scale for node \"#{@name}\" must be greater than 0." if @scale < 1
             @prev_scale = @scale
 
             @targets = [ ]
@@ -138,7 +130,7 @@ module StackBuilder::Stack
                     # Scale Down
 
                     delete_events = Set.new([ "stop", "uninstall" ])
-                    @scale.step(current_scale - 1).to_a.each do |i|
+                    @scale.step(current_scale - 1) do |i|
 
                         resource_sync = @resource_sync[i]
                         resource_sync.wait
@@ -208,9 +200,10 @@ module StackBuilder::Stack
                 end
 
                 @prev_scale = current_scale
-                @manager.set_scale(@scale)
             end
 
+            @manager.set_scale(@scale)
+
             threads
         end
 
@@ -219,25 +212,26 @@ module StackBuilder::Stack
             threads = [ ]
 
             scale = (@deleted ? @manager.get_scale : @scale)
-            if scale > 0
+            if @targets.empty?
 
-                if @sync == "first"
-                    @manager.process(scale, events, self.parse_attributes(@attributes, 0))
-                    scale -= 1
-                end
+                if scale > 0
 
-                if @sync == "all"
-                    scale.times do |i|
-                        @manager.process(i, events, self.parse_attributes(@attributes, i))
+                    if @sync == "first"
+                        @manager.process(scale, events, self.parse_attributes(@attributes, 0))
+                        scale -= 1
                     end
-                else
-                    scale.times do |i|
-                        spawn_processing(i, events, threads)
+
+                    if @sync == "all"
+                        scale.times do |i|
+                            @manager.process(i, events, self.parse_attributes(@attributes, i))
+                        end
+                    else
+                        scale.times do |i|
+                            spawn_processing(i, events, threads)
+                        end
                     end
                 end
-
-            elsif !@targets.empty?
-
+            else
                 @targets.each do |t|
                     t.scale.times do |i|
                         spawn_processing(i, events, threads, t)

data/lib/stackbuilder/stack/stack.rb

@@ -19,13 +19,14 @@ module StackBuilder::Stack
                 StackBuilder::Stack::NodeProvider." unless provider.is_a?(NodeProvider)
 
             @provider = provider
-            env_vars = provider.get_env_vars
 
+            env_vars = provider.get_env_vars
             stack = StackBuilder::Common.load_yaml(stack_file, env_vars)
-            @logger.debug("Initializing stack definition:\n #{stack.to_yaml}")
+            merge_maps( stack, overrides.end_with?('.json') ?
+                JSON.load(File.new(overrides, 'r')) : JSON.load(overrides) ) \
+                unless overrides.nil?
 
-            overrides = JSON.load(File.new(overrides, 'r')) unless overrides.nil? || !overrides.end_with?('.json')
-            merge_maps(stack, overrides) unless overrides.nil?
+            @logger.debug("Initializing stack definition:\n #{stack.to_yaml}")
 
             if id.nil?
                 @id = SecureRandom.uuid.gsub(/-/, '')

data/lib/stackbuilder/version.rb

@@ -2,7 +2,7 @@
 
 module Knife
     module StackBuilder
-        VERSION = "0.5.6"
+        VERSION = "0.5.7"
         MAJOR, MINOR, TINY = VERSION.split('.')
     end
 end

data/lib/stackbuilder.rb

@@ -13,6 +13,7 @@ require 'tmpdir'
 require 'tempfile'
 require "stringio"
 require 'json'
+require 'timeout'
 require 'openssl'
 require "net/ssh"
 require "net/ssh/multi"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-stackbuilder
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - Mevan Samaratunga
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-12 00:00:00.000000000 Z
+date: 2014-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12'
+- !ruby/object:Gem::Dependency
+  name: berkshelf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.1
 - !ruby/object:Gem::Dependency
   name: knife-attribute
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: knife-vagrant3
+  name: knife-vagrant2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="