zergrush 0.0.1

data/lib/zerg/generators/hivegen.rb

@@ -0,0 +1,63 @@
+#--
+
+# Copyright 2014 by MTN Sattelite Communications
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#++
+
+require 'thor/group'
+module Zerg
+    module Generators
+        class HiveGen < Thor::Group
+            include Thor::Actions
+
+            class_option :force, :type => :boolean
+
+            def self.source_root
+                File.join(File.dirname(__FILE__), "task")
+            end
+
+            def create_hive
+                load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+                empty_directory "#{File.join(load_path, "driver")}"
+            end
+
+            def copy_sample_task
+                load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+                opts = {
+                    :instances => 3,
+                    :drivertype => "vagrant",
+                    :providertype => "virtualbox",
+                    :baseboxpath => "http://files.vagrantup.com/precise32.box",
+                    :privatenetwork => true
+                }
+                template("template.ke", "#{File.join(load_path, "helloworld.ke")}", opts)
+
+                opts = {
+                    :instances => 3,
+                    :drivertype => "vagrant",
+                    :providertype => "aws",
+                    :baseboxpath => "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box",
+                    :privatenetwork => false
+                }
+                template("awstemplate.ke", "#{File.join(load_path, "helloaws.ke")}", opts)
+            end
+        end
+    end
+end

data/lib/zerg/generators/task/awstemplate.ke

@@ -0,0 +1,28 @@
+{
+    "instances": <%= config[:instances] %>,
+    "tasks": [
+        {
+            "type": "shell",
+            "inline": "echo \"ZERG RUSH PRIME!\""
+        }        
+    ],
+    "vm": {
+        "driver": {
+            "drivertype": "<%= config[:drivertype] %>",
+            "providertype": "<%= config[:providertype] %>",
+            "provider_options": [
+                "<%= config[:providertype] %>.instance_type = 't1.micro'",
+                "<%= config[:providertype] %>.access_key_id = \"#{ENV['AWS_ACCESS_KEY_ID']}\"",
+                "<%= config[:providertype] %>.secret_access_key = \"#{ENV['AWS_SECRET_ACCESS_KEY']}\"",
+                "<%= config[:providertype] %>.keypair_name = \"#{ENV['AWS_KEY_PAIR']}\"",
+                "<%= config[:providertype] %>.ami = 'ami-3fec7956'",
+                "<%= config[:providertype] %>.region = 'us-east-1'",
+                "<%= config[:providertype] %>.security_groups = [ \"#{ENV['AWS_SECURITY_GROUP']}\" ]",
+                "override.ssh.username = 'ubuntu'",
+                "override.ssh.private_key_path = \"#{ENV['AWS_PRIVATE_KEY_PATH']}\""
+            ]
+        },
+        "basebox": "<%= config[:baseboxpath] %>",
+        "private_network": <%= config[:privatenetwork] %>
+    }
+}

data/lib/zerg/generators/task/template.ke

@@ -0,0 +1,28 @@
+{
+    "instances": <%= config[:instances] %>,
+    "tasks": [
+        {
+            "type": "shell",
+            "inline": "echo \"ZERG RUSH!\""
+        }        
+    ],
+    "vm": {
+        "driver": {
+            "drivertype": "<%= config[:drivertype] %>",
+            "providertype": "<%= config[:providertype] %>",
+            "provider_options": [
+                "<%= config[:providertype] %>.gui = false",
+                "<%= config[:providertype] %>.memory = 256",
+                "# adjust for DNS weirdness in ubuntu 12.04",
+                "<%= config[:providertype] %>.customize ['modifyvm', :id,  '--natdnsproxy1', 'off']",
+                "<%= config[:providertype] %>.customize ['modifyvm', :id,  '--natdnshostresolver1', 'off']",
+                "# set virtio type on the NIC driver. Better performance for large traffic bursts",
+                "<%= config[:providertype] %>.customize ['modifyvm', :id,  '--nictype1', 'virtio']",
+                "<%= config[:providertype] %>.customize ['modifyvm', :id,  '--nictype2', 'virtio']",
+                "<%= config[:providertype] %>.customize ['modifyvm', :id,  '--nictype3', 'virtio']"
+            ]
+        },
+        "basebox": "<%= config[:baseboxpath] %>",
+        "private_network": <%= config[:privatenetwork] %>
+    }
+}

data/lib/zerg/hive.rb

@@ -0,0 +1,145 @@
+#--
+
+# Copyright 2014 by MTN Sattelite Communications
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#++
+
+require 'json'
+require 'awesome_print'
+require 'json-schema'
+require 'fileutils'
+require 'singleton'
+require 'highline/import'
+
+module Zerg
+    class Hive
+        include Singleton
+        attr_reader :hive
+        
+        def loaded
+            @loaded || false
+        end
+
+        def load
+            if loaded
+                return
+            end
+
+            load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+            abort("ERROR: '.hive' not found at #{load_path}. Run 'zerg init', change HIVE_CWD or run zerg from a different path.") unless File.directory?(load_path) 
+
+            # load all .ke files into one big hash
+            @hive = Hash.new
+            Dir.glob(File.join("#{load_path}", "*.ke")) do |ke_file|
+                # do work on files ending in .rb in the desired directory
+                begin 
+                    ke_file_hash = JSON.parse( IO.read(ke_file) )
+                    @hive[File.basename(ke_file, ".ke")] = ke_file_hash
+                rescue JSON::ParserError
+                    abort("ERROR: Could not parse #{ke_file}. Likely invalid JSON.")
+                end
+            end
+
+            @loaded = true
+        end
+
+        def self.list
+            instance.load
+
+            # iterate over hive configs and print out the names
+            puts  "Current hive tasks are:"
+
+            if instance.loaded == false
+                puts "No hive loaded!"
+                puts "FAILURE!"
+                return
+            end
+
+            if instance.hive.empty?()
+                puts "No tasks defined in hive."
+                return
+            end
+
+            puts "#{instance.hive.length} tasks in current hive:"
+            puts "#{instance.hive.keys.ai}"
+        end
+
+        def self.verify
+            load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+            abort("ERROR: '.hive' not found at #{load_path}. Run 'zerg init', change HIVE_CWD or run zerg from a different path.") unless File.directory?(load_path) 
+
+            Dir.glob(File.join("#{load_path}", "*.ke")) do |ke_file|
+                begin 
+                    ke_file_hash = JSON.parse( IO.read(ke_file) )
+
+                    # verify against schema.
+                    errors = JSON::Validator.fully_validate(File.join("#{File.dirname(__FILE__)}", "../../data/ke.schema"), ke_file_hash, :errors_as_objects => true)
+                    unless errors.empty?
+                        abort("ERROR: #{ke_file} failed validation. Errors: #{errors.ai}")
+                    end
+                rescue JSON::ParserError => err
+                    abort("ERROR: Could not parse #{ke_file}. Likely invalid JSON.")
+                end
+            end
+
+            puts "SUCCESS!"
+        end
+
+        def self.import(file, force)
+            load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+            abort("ERROR: '.hive' not found at #{load_path}. Run 'zerg init', change HIVE_CWD or run zerg from a different path.") unless File.directory?(load_path) 
+            abort("ERROR: '#{file}' not found!") unless File.exist?(file) 
+            abort("ERROR: '#{File.basename(file)}' already exists in hive!") unless !File.exist?(File.join(load_path, File.basename(file))) || force == true
+
+            # check the file against schema.
+            begin
+                ke_file_hash = JSON.parse( IO.read(file) )
+                errors = JSON::Validator.fully_validate(File.join("#{File.dirname(__FILE__)}", "../../data/ke.schema"), ke_file_hash, :errors_as_objects => true)
+                abort("ERROR: #{file} failed validation. Errors: #{errors.ai}") unless errors.empty?
+
+                FileUtils.cp(file, File.join(load_path, File.basename(file)))
+            rescue JSON::ParserError => err
+                abort("ERROR: Could not parse #{file}. Likely invalid JSON.")
+            end
+            puts "SUCCESS!"
+        end
+
+        def self.remove(taskname, force)
+            load_path = (ENV['HIVE_CWD'] == nil) ? File.join("#{Dir.pwd}", ".hive") : File.join("#{ENV['HIVE_CWD']}", ".hive")
+            abort("ERROR: '.hive' not found at #{load_path}. Run 'zerg init', change HIVE_CWD or run zerg from a different path.") unless File.directory?(load_path) 
+            abort("ERROR: '#{taskname}' not found!") unless File.exist?(File.join(load_path, "#{taskname}.ke")) 
+
+            # check the file against schema.
+            taskfile = File.join(load_path, "#{taskname}.ke")
+
+            agreed = true
+            if force != true
+                agreed = agree("Remove task #{taskname}?")
+            end
+
+            abort("Cancelled!") unless agreed == true
+
+            FileUtils.rm_rf(File.join(load_path, "driver", taskname))
+            FileUtils.rm(File.join(load_path, "#{taskname}.ke"))
+
+            puts "SUCCESS!"
+        end
+    end
+end

data/lib/zerg/runner.rb

@@ -0,0 +1,232 @@
+#--
+
+# Copyright 2014 by MTN Sattelite Communications
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#++
+
+require 'awesome_print'
+require 'fileutils'
+require 'erb'
+require 'rbconfig'
+
+module Zerg
+    class Runner
+
+        def check_provider(driver, provider)
+            if driver == "vagrant"
+                if provider == "aws"
+                    aws_pid = Process.spawn("vagrant plugin list | grep vagrant-aws")
+                    Process.wait(aws_pid)
+
+                    if $?.exitstatus != 0
+                        aws_pid = Process.spawn("vagrant plugin install vagrant-aws")
+                        Process.wait(aws_pid)
+                        abort("ERROR: vagrant-aws installation failed!") unless $?.exitstatus == 0
+                    end
+                elsif provider == "libvirt"
+                    abort("ERROR: libvirt is only supported on a linux host!") unless /linux|arch/i === RbConfig::CONFIG['host_os']
+                    
+                    libvirt_pid = Process.spawn("vagrant plugin list | grep vagrant-libvirt")
+                    Process.wait(libvirt_pid)
+
+                    if $?.exitstatus != 0
+                        libvirt_pid = Process.spawn("vagrant plugin install vagrant-libvirt")
+                        Process.wait(libvirt_pid)
+                        abort("ERROR: vagrant-libvirt installation failed! Refer to https://github.com/pradels/vagrant-libvirt to install missing dependencies, if any.") unless $?.exitstatus == 0
+                    end
+                end
+            end
+        end
+
+        # cross platform way of checking if command is available in PATH
+        def which(cmd)
+            exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+            ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+                exts.each { |ext|
+                exe = File.join(path, "#{cmd}#{ext}")
+                return exe if File.executable? exe
+                }
+            end
+          return nil
+        end
+
+        def process(taskname, task, debug)
+            puts ("Will perform task #{taskname} with contents:\n #{task.ai}")
+
+            # render driver template
+            renderer = DriverRenderer.new(
+                task["vm"], 
+                taskname, 
+                task["instances"], 
+                task["synced_folders"], 
+                task["tasks"])
+            
+            renderer.render
+
+            # do we need additional plugins?
+            task["tasks"].each { |task|
+                if task["type"] == "chef_client" || task["type"] == "chef_solo"
+                    omnibus_pid = Process.spawn("vagrant plugin list | grep vagrant-omnibus")
+                    Process.wait(omnibus_pid)
+
+                    if $?.exitstatus != 0
+                        omnibus_pid = Process.spawn("vagrant plugin install vagrant-omnibus")
+                        Process.wait(aws_pid)
+                        abort("ERROR: vagrant-omnibus installation failed!") unless $?.exitstatus == 0
+                    end
+                    break;
+                end
+            }
+
+            run(taskname, task["vm"]["driver"]["drivertype"], task["vm"]["driver"]["providertype"], task["instances"], debug)
+        end
+
+        def cleanup(taskname, task, debug)
+            abort("ERROR: Vagrant not installed!") unless which("vagrant") != nil
+            puts ("Will cleanup task #{taskname}...")
+
+            # TODO: generalize for multiple drivers
+            # render driver template
+            renderer = DriverRenderer.new(
+                task["vm"], 
+                taskname, 
+                task["instances"],
+                task["synced_folders"],  
+                task["tasks"])        
+            renderer.render
+
+            check_provider(task["vm"]["driver"]["drivertype"], task["vm"]["driver"]["providertype"])
+
+            # run vagrant cleanup
+            debug_string = (debug == true) ? " --debug" : ""
+            
+            for index in 0..task["instances"] - 1
+                cleanup_pid = Process.spawn(
+                    {
+                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", task["vm"]["driver"]["drivertype"], taskname),
+                        "VAGRANT_DEFAULT_PROVIDER" => task["vm"]["driver"]["providertype"]
+                    },
+                    "vagrant destroy zergling_#{index} --force#{debug_string}")
+                Process.wait(cleanup_pid)
+                abort("ERROR: vagrant failed!") unless $?.exitstatus == 0
+            end
+
+            cleanup_pid = Process.spawn(
+                {
+                    "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", task["vm"]["driver"]["drivertype"], taskname)
+                },
+                "vagrant box remove zergling_#{taskname}_#{task["vm"]["driver"]["providertype"]}#{debug_string} #{task["vm"]["driver"]["providertype"]}")
+            Process.wait(cleanup_pid)
+        end
+
+        def run(taskname, driver, provider, instances, debug)
+            # TODO: generalize to multiple drivers
+            abort("ERROR: Vagrant not installed!") unless which("vagrant") != nil
+
+            check_provider(driver, provider)
+
+            debug_string = (debug == true) ? " --debug" : ""                
+
+            # bring up all of the VMs first.
+            puts("Starting vagrant in #{File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname)}")
+            for index in 0..instances - 1
+                create_pid = Process.spawn(
+                    {
+                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname)
+                    },
+                    "vagrant up zergling_#{index} --no-provision --provider=#{provider}#{debug_string}")
+                Process.wait(create_pid)
+                
+                if $?.exitstatus != 0
+                    puts "ERROR: vagrant failed while creating one of the VMs. Will clean task #{taskname}:"
+                    self.class.clean(taskname, debug)
+                    abort("ERROR: vagrant failed!")
+                end
+            end
+
+            puts("Running tasks in vagrant virtual machines...")
+            # and provision them all at once (sort of)
+            provisioners = Array.new
+            provision_pid = nil
+            for index in 0..instances - 1
+                provision_pid = Process.spawn(
+                    {
+                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname),
+                        "VAGRANT_DEFAULT_PROVIDER" => "#{provider}"
+                    },
+                    "vagrant provision zergling_#{index}#{debug_string}")
+                provisioners.push({:name => "zergling_#{index}", :pid => provision_pid})
+            end
+
+            # wait for everything to finish...
+            errors = Array.new
+            lock = Mutex.new
+            provisioners.each { |provisioner| 
+                Thread.new { 
+                    Process.wait(provisioner[:pid]); 
+                    lock.synchronize do
+                        errors.push(provisioner[:name]) unless $?.exitstatus == 0    
+                    end
+                }.join 
+            }
+
+            puts("DONE! Halting all vagrant virtual machines...")
+            # halt all machines
+            halt_pid = nil
+            for index in 0..instances - 1
+                halt_pid = Process.spawn(
+                    {
+                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname),
+                        "VAGRANT_DEFAULT_PROVIDER" => "#{provider}"
+                    },
+                    "vagrant halt zergling_#{index}#{debug_string}")
+                Process.wait(halt_pid)
+                abort("ERROR: vagrant halt failed on machine zergling_#{index}!") unless $?.exitstatus == 0
+            end
+
+            abort("ERROR: Finished with errors in: #{errors.to_s}") unless errors.length == 0
+            puts("SUCCESS!")
+        end
+
+        def self.rush(task, debug)
+            # load the hive first
+            Zerg::Hive.instance.load
+
+            puts "Loaded hive. Looking for task #{task}..."
+            abort("ERROR: Task #{task} not found in current hive!") unless Zerg::Hive.instance.hive.has_key?(task) 
+
+            # grab the current task hash and parse it out
+            runner = Runner.new
+            runner.process(task, Zerg::Hive.instance.hive[task], debug);
+        end
+
+        def self.clean(task, debug)
+            # load the hive first
+            Zerg::Hive.instance.load
+
+            puts "Loaded hive. Looking for task #{task}..."
+            abort("ERROR: Task #{task} not found in current hive!") unless Zerg::Hive.instance.hive.has_key?(task) 
+
+            runner = Runner.new
+            runner.cleanup(task, Zerg::Hive.instance.hive[task], debug);
+            puts("SUCCESS!")
+        end
+    end
+end