hadouken 0.1.4.pre

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+
+gem "net-ssh"
+gem "net-ssh-multi"
+gem "yajl-ruby"
+
+group :development do
+  gem "shoulda", ">= 0"
+  gem "mocha",   ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov",    ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,32 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    mocha (0.9.12)
+    net-ssh (2.2.1)
+    net-ssh-gateway (1.1.0)
+      net-ssh (>= 1.99.1)
+    net-ssh-multi (1.1)
+      net-ssh (>= 2.1.4)
+      net-ssh-gateway (>= 0.99.0)
+    rake (0.9.2.2)
+    rcov (0.9.10)
+    shoulda (2.11.3)
+    yajl-ruby (1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  mocha
+  net-ssh
+  net-ssh-multi
+  rcov
+  shoulda
+  yajl-ruby

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Matt Knopp
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+hadouken - soon
+
+### running
+
+    ./serviceie.rb --interactive \ 
+                 --level debug \
+                 --environment production \
+                 --history /opt/deploys   \
+                 --artifact https://artifacts/latest.tgz
+
+
+### serviceie.rb
+
+    Hadouken::Runner.run!
+      plan      = Hadouken::Plan.new
+      plan.name = "serviceie"
+      plan.user = "serviceie"
+      plan.base = "/opt/serviceie"
+   
+      # define some groups 10x10
+      #
+      plan.add_group :web, :range => (1..10), :pattern => 'serviceie-web-%02d.example.com'
+      plan.add_group :api, :range => (1..10), :pattern => 'serviceie-api-%02d.example.com'
+
+
+      # download latest.tgz from our artifact repository
+      # runs in parallel on all hosts
+      #
+      plan.tasks.add Hadouken::Strategy::ByHost.new(plan)
+      plan.tasks.add "curl -sSfL -output /tmp/latest.tgz #{artifact}"
+      plan.tasks.add "mv /tmp/latest.tgz #{plan.base}/latest.tgz"
+
+
+      # runs commands depth first on the api hosts, two at a time
+      # - restart service
+      # - verify  service
+      #
+      plan.tasks.add Hadouken::Strategy::ByHost.new(plan, :max_hosts => 2, :traversal => :depth)
+      plan.tasks.add "restart serviceie-api", :group => :api
+      plan.tasks.add Proc.new { |opts|
+        host = opts[:host]
+        10.times do
+          response = Typheous::Request.get("http://#{host}:8081/healthcheck")
+          break if response.status_code == 200
+        end
+      }, :group => :api
+
+      
+      # finally restart the webs as fast as possible
+      #
+      plan.tasks.add Hadouken::Strategy::ByHost.new(plan)
+      plan.tasks.add "restart windard-web, :group => :web
+    end
+
+

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name        = "hadouken"
+  gem.homepage    = "http://github.com/mhat/hadouken"
+  gem.license     = "MIT"
+  gem.summary     = %Q{run commands over ssh in a way that makes sense for deploying artifacts}
+  gem.description = %Q{run commands over ssh in a way that makes sense for deploying artifacts}
+  gem.email       = ["mknopp@yammer-inc.com, cgray@yammer-inc.com"]
+  gem.authors     = ["Matt Knopp", "Chris Gray"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs    << 'lib' << 'test'
+  test.pattern  = 'test/**/test_*.rb'
+  test.verbose  = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs      << 'test'
+  test.pattern    = 'test/**/test_*.rb'
+  test.verbose    = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+#require 'rdoc/task'
+#Rake::RDocTask.new do |rdoc|
+#  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+#
+#  rdoc.rdoc_dir = 'rdoc'
+#  rdoc.title = "hadouken #{version}"
+#  rdoc.rdoc_files.include('README*')
+#  rdoc.rdoc_files.include('lib/**/*.rb')
+#end

data/VERSION

@@ -0,0 +1 @@
+0.1.4.pre

data/hadouken.gemspec

@@ -0,0 +1,84 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "hadouken"
+  s.version = "0.1.4.pre"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Matt Knopp", "Chris Gray"]
+  s.date = "2012-01-27"
+  s.description = "run commands over ssh in a way that makes sense for deploying artifacts"
+  s.email = ["mknopp@yammer-inc.com, cgray@yammer-inc.com"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "hadouken.gemspec",
+    "lib/hadouken.rb",
+    "lib/hadouken/executor.rb",
+    "lib/hadouken/ext/net_ssh_multi_session_actions.rb",
+    "lib/hadouken/group.rb",
+    "lib/hadouken/groups.rb",
+    "lib/hadouken/host.rb",
+    "lib/hadouken/plan.rb",
+    "lib/hadouken/runner.rb",
+    "lib/hadouken/strategy/base.rb",
+    "lib/hadouken/strategy/by_group.rb",
+    "lib/hadouken/strategy/by_group_parallel.rb",
+    "lib/hadouken/strategy/by_host.rb",
+    "lib/hadouken/task.rb",
+    "lib/hadouken/tasks.rb",
+    "test/helper.rb",
+    "test/test_hadouken.rb"
+  ]
+  s.homepage = "http://github.com/mhat/hadouken"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.11"
+  s.summary = "run commands over ssh in a way that makes sense for deploying artifacts"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<net-ssh>, [">= 0"])
+      s.add_runtime_dependency(%q<net-ssh-multi>, [">= 0"])
+      s.add_runtime_dependency(%q<yajl-ruby>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<net-ssh>, [">= 0"])
+      s.add_dependency(%q<net-ssh-multi>, [">= 0"])
+      s.add_dependency(%q<yajl-ruby>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<net-ssh>, [">= 0"])
+    s.add_dependency(%q<net-ssh-multi>, [">= 0"])
+    s.add_dependency(%q<yajl-ruby>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/hadouken.rb

@@ -0,0 +1,29 @@
+module Hadouken; end;
+module Hadouken::Strategy; end;
+
+require 'rubygems'
+require 'fileutils'
+require 'yajl'
+require 'uri'
+require 'net/ssh/multi'
+
+require 'hadouken/executor'
+require 'hadouken/group'
+require 'hadouken/groups'
+require 'hadouken/host'
+require 'hadouken/plan'
+require 'hadouken/runner'
+require 'hadouken/strategy/base'
+require 'hadouken/strategy/by_host'
+require 'hadouken/strategy/by_group'
+require 'hadouken/strategy/by_group_parallel'
+require 'hadouken/task'
+require 'hadouken/tasks'
+
+require 'hadouken/ext/net_ssh_multi_session_actions'
+
+module Hadouken
+  def self.logger
+    @@logger ||= Logger.new(STDOUT)
+  end
+end

data/lib/hadouken/executor.rb

@@ -0,0 +1,236 @@
+class Hadouken::Executor
+
+  class Phase
+    attr_accessor :strategy
+    attr_accessor :tasks
+    def initialize
+      @tasks = []
+    end
+  end
+
+  attr_reader :plan
+
+  def self.run!(plan)
+    exec = Hadouken::Executor.new(plan)
+    exec.phases
+    exec.session!
+
+    if Hadouken::Hosts.any?
+      exec.execute!
+    else
+      Hadouken.logger.error "No hosts have been defined, this deploy is boring!"
+    end
+  end
+
+
+  def initialize(plan)
+    @plan    = plan
+    @session = Net::SSH::Multi.start
+
+    ## TODO: find a better place for this
+    unless @plan.tasks.first.is_a?(Hadouken::Strategy::Base)
+      raise RuntimeError, "first task in plan is not a strategy"
+    end
+  end
+
+
+  def phases
+    return @phases if @phases
+    @phases = []
+    plan.tasks.each do |task|
+      if task.is_a?(Hadouken::Strategy::Base)
+        @phases << Phase.new
+        @phases.last.strategy = task
+      end
+ 
+      if task.is_a?(Hadouken::Task::Base)
+        @phases.last.tasks << task
+      end
+    end
+
+    @phases
+  end 
+
+  def session
+    @session ||= session!
+  end
+
+  def session!
+    @session = Net::SSH::Multi.start(:on_error => Proc.new{ |server|
+      host = Hadouken::Hosts.get(server.host)
+      Hadouken.logger.debug "error with #{server.host}, disabling"
+      host.history.add "ssh.connection.new", :fail
+      host.disable!
+    })
+
+    plan.groups.each do |group|
+      group.hosts.each do |host|
+        unless host.server
+          Hadouken.logger.debug "session.use #{plan.user}@#{host}"
+          server      = session.use "#{plan.user}@#{host}"
+          host.server = server
+        end
+      end
+    end
+
+    @session
+  end
+
+  def execute!
+    # the heavy lifting: pivot our structure 
+    #   from tasks : task : hosts[]
+    #   to   hosts : host : tasks[]
+    phases.each_with_index do |phase, phase_index|
+      strategy         = phase.strategy
+      hosts_with_tasks = {}
+      host_sets        = strategy.host_strategy
+      Hadouken.logger.info "idx:#{phase_index}, strategy=#{strategy}"
+
+      ## assign work
+      host_sets.each do |host_set|
+        Hadouken.logger.info "hosts=#{host_set.join(', ')}"
+      
+        host_set.each do |host|
+          hosts_with_tasks[host] ||= []
+
+          phase.tasks.each do |task|
+            # if this is not a group task then assign it to the host OR if
+            # this is a group task and the host is part of the task-group,
+            # then assign the task to the host
+            if !task.group? || (task.group? && task.group.has_host?(host))
+              hosts_with_tasks[host] <<  task
+            end
+          end
+        end
+      end
+
+      case strategy.traversal
+        when :breadth then execute_breadth_traversal! host_sets, hosts_with_tasks
+        when :depth   then execute_depth_traversal!   host_sets, hosts_with_tasks
+        else raise RuntimeError, "unknown tranversal=#{strategy.traversal}"
+      end
+    end
+  end
+
+  private
+  def execute_depth_traversal!(host_sets, hosts_with_tasks)
+    # run all of the commands assigned to the hosts in a host_set then 
+    # move on to the next host set. rinse. repeat.
+
+    host_sets.each do |host_set|
+      while hosts_with_tasks.values_at(*host_set.map).select{|t| t.any?}.any?
+        channels = []
+        host_set.each do |host|
+          # not all hosts will necessarily have the same number of tasks
+          next unless hosts_with_tasks[host].any?
+          next unless task = hosts_with_tasks[host].shift
+
+          case task
+            when Hadouken::Task::Callback then Hadouken.logger.debug "callback for #{host}"
+            when Hadouken::Task::Command  then Hadouken.logger.debug "session.on(#{host}).exec(#{task.command})"
+          end
+
+          if ! plan.dry_run?
+            if ! host.enabled?
+              case task
+                when Hadouken::Task::Callback then host.history.add task.to_s,    :noop
+                when Hadouken::Task::Command  then host.history.add task.command, :noop
+              end
+            else
+              case task
+              when Hadouken::Task::Callback
+                ret = task.call({:host => host})
+                host.history.add task.to_s, ret
+                host.disable! unless ret == 0
+
+              when Hadouken::Task::Command
+                Hadouken.logger.info "running #{task.command} on #{host}"
+                channels << [task.command, session.on(host.server).hadouken_exec(task.command)]
+              end
+            end
+          end
+        end
+
+        # wait for the work assigned to complete before performing more work.
+        wait_on_channels(channels)
+
+      end # while hosts_with_tasks
+    end # host_sets.each
+  end 
+
+
+  def execute_breadth_traversal!(host_sets, hosts_with_tasks)
+    # perform whatever tasks have been assigned; i try to do as much as
+    # possible in parallel within the terms of the current strategy.
+
+    while hosts_with_tasks.any?
+
+      host_sets.each do |host_set|
+        channels = []
+        host_set.each do |host|
+          if hosts_with_tasks.has_key?(host)
+
+            unless task = hosts_with_tasks[host].shift
+              # remove the host from hosts-with-tasks when there are no more tasks!
+              hosts_with_tasks.delete(host)
+            else
+              case task
+                when Hadouken::Task::Callback then Hadouken.logger.debug "callback for #{host}"
+                when Hadouken::Task::Command  then Hadouken.logger.debug "session.on(#{host}).exec(#{task.command})"
+              end
+
+              if ! plan.dry_run?
+                if ! host.enabled?
+                  case task
+                    when Hadouken::Task::Callback then host.history.add task.to_s,    :noop
+                    when Hadouken::Task::Command  then host.history.add task.command, :noop
+                  end
+                else 
+                  case task
+                  when Hadouken::Task::Callback
+                    ret = task.call({:host => host})
+                    host.history.add task.to_s, ret
+                    host.disable! unless ret == 0
+                  when Hadouken::Task::Command
+                    Hadouken.logger.info "running #{task.command} on #{host}"
+                    channels << [task.command, session.on(host.server).hadouken_exec(task.command)]
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        # wait for the work assigned to complete before performing more work.
+        wait_on_channels(channels)
+
+      end # host_sets.each
+    end # while
+  end
+
+  def wait_on_channels(channels)
+    if channels.count > 0
+      Hadouken.logger.info "waiting for #{channels.count} commands to execute"
+      return if plan.dry_run?
+
+      session.loop
+      channels.each do |command, channel|
+        channel.each do |subchannel|
+          host = Hadouken::Hosts.get(subchannel[:host])
+          host.history.add(command, subchannel[:exit_status], subchannel[:stdout], subchannel[:stderr])
+          if plan.interactive?
+            Hadouken.logger.info "[STDOUT] - #{host.name}: %s" % [ subchannel[:stdout].join("\n") ] if subchannel[:stdout]
+            Hadouken.logger.warn "[STDERR] - #{host.name}: %s" % [ subchannel[:stderr].join("\n") ] if subchannel[:stderr]
+          end
+
+          unless subchannel[:exit_status] == 0
+            Hadouken.logger.debug "got status=#{subchannel[:exit_status]} on #{subchannel[:host]}"
+            host.disable!
+          end
+        end
+      end
+    end 
+  end
+
+
+end