hazetug 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a5a2351211bc68264c5fe9b4780278bbbe5f419a
+  data.tar.gz: f1ecdb92eb86d8978754fb21da5bb0dcd38de525
+SHA512:
+  metadata.gz: e6ca974f9bc299115e6749def4b8b847fbdb8cbc57bc5570c6fcf54450617d5c0faffcb32600021bfba8ac25d70fe3cf3d75d6ac2e41a07fbe12876e3295f16a
+  data.tar.gz: 4f49bcae8fc1965fe9e35c153c69b51a4ddfa2d9228e6d37adfb5fae71824afd739f346ad53d6a6f0afee4a7a343e0b0841bbc7dd3c1161af369c78a8cabe40c

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+logs

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hazetug.gemspec
+gemspec
+gem 'fog', git: 'https://github.com/fog/fog.git'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Denis Barishev
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,188 @@
+# Hazetug
+
+Cloud provisioner and bootstrapper for DigitalOcean and Linode.
+Hazetug uses [fog cloud library](http://fog.io) to be able to easily append other cloud computes and *tugs* (bootstraps) hosts using:
+
+* Knife bootstrap.
+
+## Options
+
+### Cloud Computes specific options
+
+<table>
+    <tr>
+        <td><b>Option</b></td>
+        <td><b>Description</b></td>
+    </tr>
+    <tr>
+        <td><b><i>name</i></b></td>
+        <td>Name of a host to provision. Random names are possible using %rand(x)% macro.</td>
+    </tr>
+    <tr>
+        <td><b><i>location</i></b></td>
+        <td>Location in the cloud compute, namely data center.</td>
+    </tr>
+    <tr>
+        <td><b><i>flavor</i></b></td>
+        <td>Flavor of a provisioned host. Flavors are recognized by memory, use: 512mb, 4gb, etc to choose the proper one.</td>
+    </tr>
+    <tr>
+        <td><b><i>image</i></b></td>
+        <td>Distribution or image which is used for provisioning. Examples: ubuntu-12.04-x32, arch-linux-2014.14. If OS architecture is not specified then x64 is assumed.</td>
+    </tr>
+</table>
+
+### SSH Specific
+
+<table>
+    <tr>
+        <td><b>Option</b></td>
+        <td><b>Description</b></td>
+        <td><b>Default value</b></td>
+    </tr>
+    <tr>
+        <td><b><i>ssh_user</i></b></td>
+        <td>User used to during provisioning and for connecting via ssh.</td>
+        <td><i>root</i><td>
+    </tr>
+    <tr>
+        <td><b><i>ssh_password</i></b></td>
+        <td>Password for a provisioned node. Evaluated randomly for some computes.</td>
+        <td></td>
+    </tr>
+    <tr>
+        <td><b><i>ssh_port</i></b></td>
+        <td>Port used for ssh connection.</td>
+        <td><i>22</i><td>
+    </tr>
+    <tr>
+        <td><b><i>host_key_verify</i></b></td>
+        <td>Verifies host key, set to true to enable verification.</td>
+        <td><i>false</i><td>
+    </tr>
+</table>
+
+## Knife tug
+
+### Knife tug bootstrap options
+
+<table>
+    <tr>
+        <td><b>Option</b></td>
+        <td><b>Description</b></td>
+        <td><b>Default value</b></td>
+    </tr>
+    <tr>
+        <td><b><i>chef_validation_key</i></b></td>
+        <td>Validation key used to authenticate new nodes in the Chef Server.</td>
+        <td><i>validation.pem</i><td>
+    </tr>
+    <tr>
+        <td><b><i>chef_environment</i></b></td>
+        <td>Chef Environment used during bootstrap</td>
+        <td></td>
+    </tr>
+    <tr>
+        <td><b><i>chef_server_url</i></b></td>
+        <td>URL of the Chef Serer.</td>
+        <td></td>
+    </tr>
+</table>
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'hazetug'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hazetug
+
+## Usage
+
+### Configuration file
+
+Create *~/.hazetug* configuration file, with the content like:
+
+```yaml
+default:
+  linode_api_key: YOUR_LINODE_API_KEY
+  linode_ssh_keys:
+    - ~/.ssh/linode.pem (Change with your path, it also might be missing)
+  digitalocean_api_key: DIGITALOCEAN_API_KEY
+  digitalocean_client_id: DIGITALOCEAN_CLIENT_ID
+  digitalocean_ssh_keys:
+    - ~/.ssh/digitalocean.pem (Change with your path)
+```
+
+### Tasks
+
+Hazetug bootstrap task file is yaml file as well, it consists of two sections **global** and **bootstrap**. Global section sets default variables used by hazetug, each bootstraped host from bootstrap list sets variables specific to it thus redefining global defaults. Let's have a look at a sample task file:
+
+```
+chef_server_url: 'https://mychefserver.uri'
+chef_environment: staging
+chef_version: 11.14.6
+ruby_version: ruby-2.1.2
+ssh_password: my-password-on-api-nodes
+bootstrap:
+  - name: api-testbox-%rand(6)%
+    number:   2
+    location: london
+    flavor:   2gb
+    image:    ubuntu-14.04-x64
+    run_list: ["role[api]"]
+```
+
+From the example above we can see various variables used by hazetug they are common for all bootstrapped nodes, that's why it's reasonable to locate them in the global. However each variable has three layer hierarchy more details look into the [Variables Priority](README.md#variables-priority) section.
+
+### Variables priority
+
+Hazetug uses 3-level priority for flexible variable choosing. Priority in the ascending order is the following: variable from the global section -> variable set via command option -> variable in the bootstrap list entity.
+All variables are merged using this 3-level priority.
+
+
+### Command Line and Invocation
+
+### Bootstrap using knife
+
+Help for linode compute is given bellow:
+
+```
+NAME
+    knife - Bootstraps server using Knife
+
+SYNOPSIS
+    hazetug.rb [global options] linode bootstrap knife [command options] task.yaml
+
+COMMAND OPTIONS
+    -v, --variables=arg   - Set variable or comma-seperated list of variables (var1_1=hello) (default: none)
+    -n, --number=arg      - Set number of created nodes, value from yaml is honored (default: 1)
+    -c, --concurrency=arg - Set concurrency value, i.e. number of hosts bootstraped simultaneously (default: 1)
+    -b, --bootstrap=arg   - Set path to knife bootstrap.erb file (default: bootstrap.erb)
+```
+
+All variables are passed to the bootstrap template and are available using the hazetug hash like - `hazetug[:variable_name]`. Amongst variables described here in the options sections, hazetug also passes useful variables such as ***compute_name***, ***public_ip_address***, ***private_ip_address*** if those are available.
+
+#### Examples
+
+* Provisioning and bootstrapping 5 nodes, each 3 of them will be processed simultaneously:
+  
+  `hazetug digitalocean bootstrap knife -n 5 -c 3 -b api.erb api-task.yaml`
+  
+* Redefining validation_key and chef_version:
+  
+  `hazetug digitalocean bootstrap knife -v validation_key=/tmp/validation.pem,chef_version=11.12.4 api.erb api-task.yaml`
+
+## Contributing
+
+1. Fork it ( http://github.com/dennybaa/hazetug/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/hazetug.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hazetug/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hazetug"
+  spec.version       = Hazetug::VERSION
+  spec.authors       = ["Denis Barishev"]
+  spec.email         = ["denz@twiket.com"]
+  spec.summary       = %q{Cloud provisoner tool}
+  spec.description   = %q{Hazetug uses fog cloud library and he}
+  spec.homepage      = "https://github.com/dennybaa/hazetug"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_dependency "psych"
+  spec.add_dependency "fog"
+  spec.add_dependency "chef", ">= 11.10.0"
+  spec.add_dependency "gli"
+  spec.add_dependency "agent"
+end

data/lib/hazetug/cli/action.rb

@@ -0,0 +1,30 @@
+require 'hazetug/ui'
+
+class Hazetug
+  class CLI
+    class Action
+      attr_reader :data
+
+      def ui
+        Hazetug::UI.instance
+      end
+
+      def pass(hash={})
+        @data = hash.dup
+        self
+      end
+
+      class << self
+        def inherited(child)
+          action = child.name.split('::').last.downcase.to_sym
+          @actions ||= {}
+          @actions[action] = child
+        end
+
+        def [](value)
+          @actions[value]
+        end
+      end
+    end
+  end
+end

data/lib/hazetug/cli/bootstrap.rb

@@ -0,0 +1,88 @@
+require 'agent'
+require 'hazetug/cli/action'
+require 'hazetug/tug'
+require 'hazetug/task'
+
+
+class Hazetug
+  class CLI
+    class Bootstrap < Action
+      def execute
+        concurrency = data[:opts][:concurrency].to_i || 1
+        queue = channel!(Object, concurrency)
+        waitgroup = Agent::WaitGroup.new
+        bootstrap_list do |haze, tug|
+          queue << nil; waitgroup.add(1)
+          block = method(:provision_and_bootstrap).to_proc
+          go!(haze, tug, queue, waitgroup, &block)
+        end
+        waitgroup.wait
+      end
+
+      def task
+        yaml_task = data[:args].shift
+        @task ||= Hazetug::Task.load_from_file(yaml_task)
+      end
+
+      def provision_and_bootstrap(haze, tug, channel, waitgroup)
+        haze.provision
+        tug.bootstrap({
+          args:  data[:args],
+          opts:  data[:opts],
+          gopts: data[:gopts]
+        })
+      rescue
+        # Exeception will be lost, since we run inside goproc,
+        # ie. as soon as waitgroup is empty all process exit.
+        puts $!.inspect
+        puts $@
+      ensure
+        waitgroup.done
+        channel.receive
+      end
+
+      def bootstrap_list(&block)
+        return if block.nil?
+        task.hosts_to_bootstrap(env_split) do |conf|
+          num = conf[:number] || data[:opts][:number].to_i || 1
+          if convert_rand_name(conf[:name]) == conf[:name] && num > 1
+            ui.fatal "Can't bootstrap several hosts with the same name"
+            raise ArgumentError, "%rand(x)% expected"
+          end
+          (1..num).each do
+            newconf = conf.dup
+            newconf[:name] = convert_rand_name(conf[:name])
+            haze = Hazetug::Haze[data[:compute_name]].new(newconf)
+            # Ensure a dynamic password loaded back from haze
+            if haze.config[:ssh_password]
+              newconf[:ssh_password] = haze.config[:ssh_password]
+            end
+            tug  = Hazetug::Tug[data[:tug_name]].new(newconf, haze)
+            block.call(haze, tug)
+          end
+        end
+      end
+
+      def env_split
+        @env_split ||= begin
+          env = {}
+          arr = data[:opts][:env]
+          if arr
+            arr.each do |eq|
+              k, v = eq.split('=')
+              env[k] = v
+            end
+          end
+          env
+        end
+      end
+
+      def convert_rand_name(name)
+        name.sub(/%rand\((\d+)\)%/) do |m|
+          rand(36**$1.to_i).to_s(36)
+        end
+      end
+
+    end
+  end
+end

data/lib/hazetug/cli.rb

@@ -0,0 +1,74 @@
+require 'gli'
+require 'hazetug/cli/bootstrap'
+
+class Hazetug
+  class CLI
+    include GLI::App
+
+    def run(args)
+      define_cli
+      super
+    end
+
+    private
+
+    def define_cli
+      sort_help :manually
+      switch [:v, :verbose]
+
+      {
+        digital_ocean: 'DigitalOcean compute',
+        linode:        'Linode Compute'
+      }.each do |compute, compute_desc|
+        desc compute_desc
+        default_command :linode
+
+        command compute.to_s.gsub(/_/,'') do |compute_cmd|
+
+          compute_cmd.desc 'Provisions and bootstraps server'
+          compute_cmd.command :bootstrap do |op|
+
+            op.desc 'Bootstraps server using Knife'
+            op.arg_name 'task.yaml'
+            op.command :knife do |tug|
+              tug.arg_name nil
+
+              tug.flag [:v, :variables], :must_match => Array,
+                :desc => 'Set variable or comma-seperated list of variables (var1_1=hello)'
+
+              tug.flag [:n, :number], :default_value => 1,
+                :desc => 'Set number of created nodes, value from yaml is honored'
+
+              tug.flag [:c, :concurrency], :default_value => 1,
+                :desc => 'Set concurrency value, i.e. number of hosts bootstraped simultaneously'
+
+              tug.flag [:b, :bootstrap], :default_value => 'bootstrap.erb',
+                :desc => 'Set path to knife bootstrap.erb file'
+
+              tug.action do |gopts, opts, args|
+
+                if args.empty?
+                  commands[:help].execute({},{},tug.name_for_help)
+                  exit 0
+                end
+
+                act = CLI::Action[:bootstrap].new
+                act.pass(
+                  tug_name: :knife,
+                  compute_name: compute,
+                  cli:   tug,
+                  gopts: gopts,
+                  opts:  opts,
+                  args:  args
+                ).execute
+              end
+            end
+
+          end
+
+        end
+      end
+    end
+
+  end
+end

data/lib/hazetug/compute.rb

@@ -0,0 +1,80 @@
+require 'fog/core'
+require 'fog/xml'
+require 'fog/json'
+require 'fog/core/parser'
+
+class Hazetug
+  class Compute
+    autoload :Linode, 'fog/linode'
+    autoload :DigitalOcean, 'fog/digitalocean'
+
+    module CloudMixin
+      CLOUD_MODELS = [
+        :locations,
+        :flavors,
+        :images
+      ]
+
+      def cloud?
+        true
+      end
+
+      def cloud_models
+        Hazetug::Compute::CloudMixin::CLOUD_MODELS
+      end
+
+      CLOUD_MODELS.each do |method_name|
+        define_method(method_name) do
+          collection = self.class.const_get(:CollectionMap) || {}
+          map_method = collection[method_name] ? collection[method_name] : method_name
+          fog.send(map_method)
+        end
+      end
+    end
+
+    class Base
+      def initialize(*args)
+        klass = self.class.name.split('::').last
+        @fog = Fog::Compute.const_get(klass).new(*args)
+      end
+
+      def method_missing(method_name, *args, &block)
+        # cache all missing methods calls
+        self.class.module_eval <<-EOS, __FILE__, __LINE__
+          def #{method_name}(*args, &block)
+            fog.#{method_name}(*args, &block)
+          end
+          EOS
+        fog.send(method_name, *args, &block)
+      end
+
+      class << self
+        def collection_map(mapping)
+          @collection_map = mapping
+          unless const_defined?(:CollectionMap)
+            module_eval "CollectionMap = @collection_map"
+          end
+        end
+      end
+
+      private
+
+      attr_reader :fog
+    end
+
+    class Linode < Base
+      include CloudMixin
+      collection_map({
+        :locations => :data_centers
+      })
+    end
+
+    class DigitalOcean < Base
+      include CloudMixin
+      collection_map({
+        :locations => :regions
+      })
+    end
+
+  end
+end

data/lib/hazetug/config.rb

@@ -0,0 +1,26 @@
+require 'psych'
+
+class Hazetug
+  class Config
+    PATH = File.expand_path("~/.hazetug")
+
+    class << self
+      def load
+        Fog.credentials_path = PATH
+        @credentials ||= begin
+          if File.exist?(PATH)
+            Psych.load_file(PATH)[Fog.credential.to_s]
+          else
+            {}
+          end
+        end
+      end
+
+      def [](key)
+        @credentials[key]
+      end
+    end
+
+  end
+end
+Hazetug::Config.load

data/lib/hazetug/haze/cloud_server.rb

@@ -0,0 +1,71 @@
+class Hazetug
+  class Haze
+    module CloudServer
+
+      def provision_server
+        ui.info "[#{compute_name}] creating server #{config[:name]}"
+        @server = compute.servers.create(server_args)
+        server.wait_for { ready? }
+        ui.info "[#{compute_name}] server #{config[:name]} created, ip: #{server.ssh_ip_address}"
+      end
+
+      def wait_for_ssh
+        ssh_options = {}
+        ssh_opts = Hazetug::Tug.ssh_options_from(config)
+        ssh_options[:password] = ssh_opts[:ssh_password]
+        ssh_options[:paranoid] = ssh_opts[:host_key_verify] || false
+        ssh_options[:keys] = ssh_opts[:ssh_keys] || Hazetug.ssh_keys(compute_name)
+        server.username = ssh_opts[:ssh_user]
+        server.ssh_port = ssh_opts[:ssh_port]
+        server.ssh_options = ssh_options
+        ui.info "[#{compute_name}] waiting for active ssh on #{server.ssh_ip_address}"
+        server.wait_for(30) { sshable? }
+      rescue Fog::Errors::TimeoutError
+        ui.error "[#{compute_name}] ssh failed to #{config[:name]}, ip: #{server.ssh_ip_address}"
+      end
+
+      def lookup(model, *args)
+        collection_method = model.to_s + 's'
+        compare_method = "compare_#{model}?"
+        found = compute.send(collection_method).select do |o|
+          self.send(compare_method, o, *args)
+        end
+        if found.size > 1
+          ui.error "More than one #{model} found for #{config[model]}"
+        elsif found.empty?
+          ui.fatal "#{model} not found for #{config[model]}"
+          raise ArgumentError, "Wrong argument #{config[model]}"
+        end
+        found.first
+      end
+
+      def memory_in_megabytes(string)
+        mult = 1
+        mult = 1024 if string.match(/gb$/i)
+        string.to_i * mult
+      end
+
+      def image_from_string(string)
+        string.downcase.sub(/\s*lts\s*/, '').
+          sub(RE_BITS, '').
+          sub(/\s+$/, '').
+          gsub(/ /, '-')
+      end
+
+      # Bits from string, when can't be fetched default is 64
+      def bits_from_string(string)
+        m = string.match(Haze::RE_BITS)
+        m ? m.captures.compact.first.to_s.to_i : 64
+      end
+
+      def server_args
+        @server_args ||= create_server_args
+      end
+
+      def create_server_args
+        raise NotImplementedError, "#create_server_arguments not implemented"
+      end
+
+    end
+  end
+end