zergrush 0.0.2 → 0.0.4

data/lib/zerg/runner.rb

@@ -23,212 +23,29 @@
 
 require 'awesome_print'
 require 'fileutils'
-require 'erb'
-require 'rbconfig'
 
 module Zerg
     class Runner
-
-        def check_provider(driver, provider)
-            if driver == "vagrant"
-                if provider == "aws"
-                    aws_pid = Process.spawn("vagrant plugin list | grep vagrant-aws")
-                    Process.wait(aws_pid)
-
-                    if $?.exitstatus != 0
-                        aws_pid = Process.spawn("vagrant plugin install vagrant-aws")
-                        Process.wait(aws_pid)
-                        abort("ERROR: vagrant-aws installation failed!") unless $?.exitstatus == 0
-                    end
-                elsif provider == "libvirt"
-                    abort("ERROR: libvirt is only supported on a linux host!") unless /linux|arch/i === RbConfig::CONFIG['host_os']
-                    
-                    libvirt_pid = Process.spawn("vagrant plugin list | grep vagrant-libvirt")
-                    Process.wait(libvirt_pid)
-
-                    if $?.exitstatus != 0
-                        libvirt_pid = Process.spawn("vagrant plugin install vagrant-libvirt")
-                        Process.wait(libvirt_pid)
-                        abort("ERROR: vagrant-libvirt installation failed! Refer to https://github.com/pradels/vagrant-libvirt to install missing dependencies, if any.") unless $?.exitstatus == 0
-                    end
-                end
-            end
-        end
-
-        # cross platform way of checking if command is available in PATH
-        def self.which(cmd)
-            exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-            ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
-                exts.each { |ext|
-                exe = File.join(path, "#{cmd}#{ext}")
-                return exe if File.executable? exe
-                }
-            end
-          return nil
-        end
-
-        def process(taskname, task, debug)
-            puts ("Will perform task #{taskname} with contents:\n #{task.ai}")
-
-            # render driver template
-            renderer = DriverRenderer.new(
-                task["vm"], 
-                taskname, 
-                task["instances"], 
-                task["synced_folders"], 
-                task["tasks"])
-            
-            renderer.render
-
-            # do we need additional plugins?
-            task["tasks"].each { |task|
-                if task["type"] == "chef_client" || task["type"] == "chef_solo"
-                    omnibus_pid = Process.spawn("vagrant plugin list | grep vagrant-omnibus")
-                    Process.wait(omnibus_pid)
-
-                    if $?.exitstatus != 0
-                        omnibus_pid = Process.spawn("vagrant plugin install vagrant-omnibus")
-                        Process.wait(aws_pid)
-                        abort("ERROR: vagrant-omnibus installation failed!") unless $?.exitstatus == 0
-                    end
-                    break;
-                end
-            }
-
-
-            run(taskname, task["vm"]["driver"]["drivertype"], task["vm"]["driver"]["providertype"], task["instances"], (task["vm"]["keepalive"] == nil) ? false : task["vm"]["keepalive"], debug)
-        end
-
-        def cleanup(taskname, task, debug)
-            puts ("Will cleanup task #{taskname}...")
-
-            # TODO: generalize for multiple drivers
-            # render driver template
-            renderer = DriverRenderer.new(
-                task["vm"], 
-                taskname, 
-                task["instances"],
-                task["synced_folders"],  
-                task["tasks"])        
-            renderer.render
-
-            check_provider(task["vm"]["driver"]["drivertype"], task["vm"]["driver"]["providertype"])
-
-            # run vagrant cleanup
-            debug_string = (debug == true) ? " --debug" : ""
-            
-            for index in 0..task["instances"] - 1
-                cleanup_pid = Process.spawn(
-                    {
-                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", task["vm"]["driver"]["drivertype"], taskname),
-                        "VAGRANT_DEFAULT_PROVIDER" => task["vm"]["driver"]["providertype"]
-                    },
-                    "vagrant destroy zergling_#{index} --force#{debug_string}")
-                Process.wait(cleanup_pid)
-                abort("ERROR: vagrant failed!") unless $?.exitstatus == 0
-            end
-
-            cleanup_pid = Process.spawn(
-                {
-                    "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", task["vm"]["driver"]["drivertype"], taskname)
-                },
-                "vagrant box remove zergling_#{taskname}_#{task["vm"]["driver"]["providertype"]}#{debug_string} #{task["vm"]["driver"]["providertype"]}")
-            Process.wait(cleanup_pid)
-        end
-
-        def halt(taskname, driver, provider, instances, debug)
-            puts("Halting all vagrant virtual machines...")
-            debug_string = (debug == true) ? " --debug" : ""  
-
-            # halt all machines
-            halt_pid = nil
-            for index in 0..instances - 1
-                halt_pid = Process.spawn(
-                    {
-                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname),
-                        "VAGRANT_DEFAULT_PROVIDER" => "#{provider}"
-                    },
-                    "vagrant halt zergling_#{index}#{debug_string}")
-                Process.wait(halt_pid)
-                abort("ERROR: vagrant halt failed on machine zergling_#{index}!") unless $?.exitstatus == 0
-            end
-        end
-
-        def run(taskname, driver, provider, instances, keepalive, debug)
-            check_provider(driver, provider)
-
-            debug_string = (debug == true) ? " --debug" : ""                
-
-            # bring up all of the VMs first.
-            puts("Starting vagrant in #{File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname)}")
-            for index in 0..instances - 1
-                create_pid = Process.spawn(
-                    {
-                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname)
-                    },
-                    "vagrant up zergling_#{index} --no-provision --provider=#{provider}#{debug_string}")
-                Process.wait(create_pid)
-                
-                if $?.exitstatus != 0
-                    puts "ERROR: vagrant failed while creating one of the VMs. Will clean task #{taskname}:"
-                    self.class.clean(taskname, debug)
-                    abort("ERROR: vagrant failed!")
-                end
-            end
-
-            puts("Running tasks in vagrant virtual machines...")
-            # and provision them all at once (sort of)
-            provisioners = Array.new
-            provision_pid = nil
-            for index in 0..instances - 1
-                provision_pid = Process.spawn(
-                    {
-                        "VAGRANT_CWD" => File.join("#{Dir.pwd}", ".hive", "driver", driver, taskname),
-                        "VAGRANT_DEFAULT_PROVIDER" => "#{provider}"
-                    },
-                    "vagrant provision zergling_#{index}#{debug_string}")
-                provisioners.push({:name => "zergling_#{index}", :pid => provision_pid})
-            end
-
-            # wait for everything to finish...
-            errors = Array.new
-            lock = Mutex.new
-            provisioners.each { |provisioner| 
-                Thread.new { 
-                    Process.wait(provisioner[:pid]); 
-                    lock.synchronize do
-                        errors.push(provisioner[:name]) unless $?.exitstatus == 0    
-                    end
-                }.join 
-            }
-
-            if keepalive == false
-                halt(taskname, driver, provider, instances, debug)
-            else
-                puts "Will leave instances running."
-            end
-
-            abort("ERROR: Finished with errors in: #{errors.to_s}") unless errors.length == 0
-            puts("SUCCESS!")
-        end
-
         def self.rush(task, debug)
-            abort("ERROR: Vagrant not installed!") unless which("vagrant") != nil
-
             # load the hive first
             Zerg::Hive.instance.load
 
             puts "Loaded hive. Looking for task #{task}..."
             abort("ERROR: Task #{task} not found in current hive!") unless Zerg::Hive.instance.hive.has_key?(task) 
 
-            # grab the current task hash and parse it out
-            runner = Runner.new
-            runner.process(task, Zerg::Hive.instance.hive[task], debug);
+            # rush!
+            begin
+                pmgr = ZergGemPlugin::Manager.instance
+                pmgr.load
+                driver = pmgr.create("/driver/#{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]}")
+                driver.rush Zerg::Hive.instance.load_path, task, Zerg::Hive.instance.hive[task], debug
+            rescue ZergGemPlugin::PluginNotLoaded
+                abort("ERROR: driver #{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]} not found. Did you install the plugin gem?")
+            end
+            puts("SUCCESS!")
         end
 
         def self.halt(task, debug)
-            abort("ERROR: Vagrant not installed!") unless which("vagrant") != nil
-
             # load the hive first
             Zerg::Hive.instance.load
 
@@ -236,22 +53,32 @@ module Zerg
             abort("ERROR: Task #{task} not found in current hive!") unless Zerg::Hive.instance.hive.has_key?(task) 
 
             # halt!
-            runner = Runner.new
-            runner.halt(task, Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"], Zerg::Hive.instance.hive[task]["vm"]["driver"]["providertype"], Zerg::Hive.instance.hive[task]["instances"], debug)
+            begin
+                pmgr = ZergGemPlugin::Manager.instance
+                pmgr.load
+                driver = pmgr.create("/driver/#{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]}")
+                driver.halt Zerg::Hive.instance.load_path, task, Zerg::Hive.instance.hive[task], debug
+            rescue ZergGemPlugin::PluginNotLoaded
+                abort("ERROR: driver #{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]} not found. Did you install the plugin gem?")
+            end
             puts("SUCCESS!")
         end
 
         def self.clean(task, debug)
-            abort("ERROR: Vagrant not installed!") unless which("vagrant") != nil
-
             # load the hive first
             Zerg::Hive.instance.load
 
             puts "Loaded hive. Looking for task #{task}..."
             abort("ERROR: Task #{task} not found in current hive!") unless Zerg::Hive.instance.hive.has_key?(task) 
 
-            runner = Runner.new
-            runner.cleanup(task, Zerg::Hive.instance.hive[task], debug);
+            begin
+                pmgr = ZergGemPlugin::Manager.instance
+                pmgr.load
+                driver = pmgr.create("/driver/#{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]}")
+                driver.clean Zerg::Hive.instance.load_path, task, Zerg::Hive.instance.hive[task], debug
+            rescue ZergGemPlugin::PluginNotLoaded
+                abort("ERROR: driver #{Zerg::Hive.instance.hive[task]["vm"]["driver"]["drivertype"]} not found. Did you install the plugin gem?")
+            end
             puts("SUCCESS!")
         end
     end

data/lib/zerg/version.rb

@@ -22,5 +22,5 @@
 #++
 
 module Zerg
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/zerg.gemspec

@@ -8,10 +8,11 @@ Gem::Specification.new do |s|
   s.authors     = ["MTN Satellite Communications"]
   s.email       = ["Marat.Garafutdinov@mtnsat.com"]
   s.homepage    = "https://github.com/MTNSatelliteComm/zerg"
+  s.license     = "MIT"
   s.summary     = "Zerg is a tool for launching an arbitrary number of virtual machines and running a task on all of them at once"
   s.description = "Zerg is a tool for launching an arbitrary number of virtual machines and running a task on all of them at once"
 
-  s.required_rubygems_version = ">= 1.3.6"
+  s.required_rubygems_version = ">= 2.0.0"
   s.rubyforge_project         = "zergrush"
 
   s.add_development_dependency "bundler", ">= 1.0.0"
@@ -24,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_dependency "json-schema"
   s.add_dependency "thor"
   s.add_dependency "highline"
+  s.add_dependency "zergrush_vagrant"
 
   s.files        = `git ls-files`.split("\n")
   s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zergrush
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - MTN Satellite Communications
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-14 00:00:00.000000000 Z
+date: 2014-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zergrush_vagrant
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Zerg is a tool for launching an arbitrary number of virtual machines
   and running a task on all of them at once
 email:
@@ -151,18 +165,14 @@ files:
 - LICENSE.txt
 - Rakefile
 - bin/zerg
-- data/driver/vagrant/bridging.template
-- data/driver/vagrant/hostonly.template
-- data/driver/vagrant/machine.template
-- data/driver/vagrant/main.template
-- data/driver/vagrant/provider.template
 - data/ke.schema
 - features/hive.feature
 - features/support/setup.rb
 - features/task.feature
 - lib/zerg.rb
 - lib/zerg/cli.rb
-- lib/zerg/driver_renderer.rb
+- lib/zerg/erbalize.rb
+- lib/zerg/gem_plugin.rb
 - lib/zerg/generators/hivegen.rb
 - lib/zerg/generators/task/awstemplate.ke
 - lib/zerg/generators/task/template.ke
@@ -172,7 +182,8 @@ files:
 - spec/zerg_spec.rb
 - zerg.gemspec
 homepage: https://github.com/MTNSatelliteComm/zerg
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -187,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.6
+      version: 2.0.0
 requirements: []
 rubyforge_project: zergrush
 rubygems_version: 2.2.2

data/data/driver/vagrant/bridging.template

@@ -1 +0,0 @@
-<%= machine_name %>.vm.network :public_network, :bridge => "<%= bridged_eth_description %>", :mac => <%= machine_mac %>, :use_dhcp_assigned_default_route => true

data/data/driver/vagrant/hostonly.template

@@ -1 +0,0 @@
-<%= machine_name %>.vm.network :private_network, ip: "192.168.50.<%= last_octet %>"

data/data/driver/vagrant/machine.template

@@ -1,22 +0,0 @@
-    # define the machine name
-    config.vm.define "<%= machine_name %>" do |<%= machine_name %>|
-    <%= bridge_specifics -%>
-    <%= hostonly_specifics %>
-<%= sync_folders_array -%>
-
-        # provisioning
-        tasks = JSON.parse(<%= tasks_array %>)
-        tasks.each { |task| 
-            if task["type"] == "chef_client" || task["type"] == "chef_solo"
-                <%= machine_name %>.omnibus.chef_version = :latest   
-            end
-
-            <%= machine_name %>.vm.provision "#{task["type"]}" do |provisioner| 
-                task.each do |key, value|
-                    next if key == "type"
-                    provisioner.send("#{key}=", value)                  
-                end
-            end
-        }
-    end
-    

data/data/driver/vagrant/main.template

@@ -1,11 +0,0 @@
-Vagrant.configure(2) do |config|    
-<%= provider_section %>
-
-    config.vagrant.host = :detect
-    config.vm.box_url = "<%= basebox_path %>"
-    config.vm.box = "<%= box_name %>"
-    config.vm.boot_timeout = 300
-    config.vm.graceful_halt_timeout = 60    
-<%= vm_defines -%>
-
-end

data/data/driver/vagrant/provider.template

@@ -1,3 +0,0 @@
-    config.vm.provider :<%= provider %> do |<%= provider %>, override|
-<%= provider_specifics -%>
-    end

data/lib/zerg/driver_renderer.rb

@@ -1,190 +0,0 @@
-#--
-
-# Copyright 2014 by MTN Sattelite Communications
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#++
-
-require 'awesome_print'
-require 'fileutils'
-require 'erb'
-require 'ostruct'
-require 'securerandom'
-
-module Zerg
-    class DriverRenderer
-        class Erbalize < OpenStruct
-            def self.erbalize_hash(template, sources)
-                Erbalize.new(sources).render(template)
-            end
-
-            def render(template)
-                ERB.new(template, nil, '-').result(binding)
-            end
-        end
-
-        # generate a virtualbox - compatible MAC address
-        def generateMACAddress()
-            firstChar = (0..255).map(&:chr).select{|x| x =~ /[0-9A-Fa-f]/}.sample(1).join
-            secondChar = (0..255).map(&:chr).select{|x| x =~ /[02468ACEace]/}.sample(1).join
-            restOfChars = (0..255).map(&:chr).select{|x| x =~ /[0-9A-Fa-f]/}.sample(10).join
-            return "#{firstChar}#{secondChar}#{restOfChars}"
-        end
-
-        def initialize(vm, name, instances, synced_folders, tasks )
-            @vm = vm
-            @name = name
-            @instances = instances
-            @tasks = tasks
-            @synced_folders = synced_folders
-        end
-
-        def render
-            puts ("Rendering driver templates...")
-
-            # TODO: generalize this processing better
-            abort("ERROR: Driver type '#{@vm["driver"]["drivertype"]} is not supported.") unless (@vm["driver"]["drivertype"] == "vagrant")
-
-            # load the template files
-            main_template = File.open(File.join("#{File.dirname(__FILE__)}", "..", "..", "data", "driver", "#{@vm["driver"]["drivertype"]}", "main.template"), 'r').read
-
-            # load the provider top level template
-            provider_parent_template = File.open(File.join("#{File.dirname(__FILE__)}", "..", "..", "data", "driver", "#{@vm["driver"]["drivertype"]}", "provider.template"), 'r').read
-
-            # load the machine details template
-            machine_template = File.open(File.join("#{File.dirname(__FILE__)}", "..", "..", "data", "driver", "#{@vm["driver"]["drivertype"]}", "machine.template"), 'r').read
-
-            # load the bridge details template
-            bridge_template = File.open(File.join("#{File.dirname(__FILE__)}", "..", "..", "data", "driver", "#{@vm["driver"]["drivertype"]}", "bridging.template"), 'r').read
-
-            # load the host only network details template
-            hostonly_template = File.open(File.join("#{File.dirname(__FILE__)}", "..", "..", "data", "driver", "#{@vm["driver"]["drivertype"]}", "hostonly.template"), 'r').read
-
-            # render templates....
-            # render provider details to string
-            # 
-            # render provider details into a string
-            provider_details_array = @vm["driver"]["provider_options"]
-            provider_details = ""
-            for index in 0..provider_details_array.length - 1
-                provider_details += "\t\t" + provider_details_array[index] + "\n"
-            end
-
-            # render provider parent
-            sources =  {
-                :provider => @vm["driver"]["providertype"],
-                :provider_specifics => provider_details
-            }
-            provider_parent_string = Erbalize.erbalize_hash(provider_parent_template, sources)
-
-            # render machine template
-            all_macs = Array.new
-            all_machines = ""
-            for index in 0..@instances - 1
-
-                # last ip octet offset for host only networking
-                ip_octet_offset = index
-
-                # inject randomized node_name into chef_client tasks
-                @tasks.each { |task| 
-                    if task["type"] == "chef_client"
-                        task["node_name"] = "zergling_#{index}_#{SecureRandom.hex(20)}"
-                    end
-                }
-
-                # tasks array rendered to ruby string. double encoding to escape quotes and allow for variable expansion
-                tasks_array = @tasks.to_json.to_json
-
-                # do we need the bridging template as well?
-                bridge_section = nil
-                if @vm.has_key?("bridge_description")
-                    # mac address to use?
-                    new_mac = ""
-                    begin
-                        new_mac = generateMACAddress()
-                    end while all_macs.include? new_mac
-
-                    sources = {
-                        :machine_mac => new_mac,
-                        :bridged_eth_description => @vm["bridge_description"]
-                    }
-                    bridge_section = Erbalize.erbalize_hash(bridge_template, sources)
-                end
-
-                # do we need the host only template as well?
-                hostonly_section = nil
-                if @vm["private_network"] == true
-                    sources = {
-                        :machine_name => "zergling_#{index}",
-                        :last_octet => ip_octet_offset + 4, # TODO: this is probably specific to virtualbox networking
-                    }
-                    hostonly_section = Erbalize.erbalize_hash(hostonly_template, sources)
-                end
-
-                # blah
-                folder_definitions = nil
-                if @synced_folders != nil
-                    folder_definitions = ""
-                    @synced_folders.each { |folder| 
-                        other_options = ""
-                        if folder.has_key?("options")
-                            folder["options"].each { |option|
-                                option.each do |key, value|
-                                    if value.is_a?(String)
-                                        other_options += ", :#{key} => \"#{value}\""
-                                    else
-                                        other_options += ", :#{key} => #{value}" 
-                                    end
-                                end
-                            } 
-                        end
-
-                        folder_definition = "zergling_#{index}.vm.synced_folder \"#{folder['host_path']}\", \"#{folder['guest_path']}\""
-                        folder_definition = "#{folder_definition}#{other_options}" unless other_options.empty?()
-                        folder_definitions += "\t\t#{folder_definition}\n"
-                    }
-                end
-
-                sources = {
-                    :machine_name => "zergling_#{index}",
-                    :bridge_specifics => bridge_section,
-                    :hostonly_specifics => hostonly_section,
-                    :tasks_array => tasks_array,
-                    :sync_folders_array => folder_definitions 
-                }.delete_if { |k, v| v.nil? }
-
-                machine_section = Erbalize.erbalize_hash(machine_template, sources)
-                all_machines += "\n#{machine_section}"
-            end
-
-            sources = {
-                :provider_section => provider_parent_string,
-                :basebox_path => @vm["basebox"],
-                :box_name => "zergling_#{@name}_#{@vm["driver"]["providertype"]}",
-                :vm_defines => all_machines
-            }
-            full_template = Erbalize.erbalize_hash(main_template, sources)
-
-            # write the file
-            puts ("Writing #{File.join("#{Dir.pwd}", ".hive", "driver", @vm["driver"]["drivertype"], @name, "Vagrantfile")}...")
-            FileUtils.mkdir_p(File.join("#{Dir.pwd}", ".hive", "driver", @vm["driver"]["drivertype"], @name))
-            File.open(File.join("#{Dir.pwd}", ".hive", "driver", @vm["driver"]["drivertype"], @name, "Vagrantfile"), 'w') { |file| file.write(full_template) }
-        end
-    end
-end