dust-deploy 0.12.2 → 0.13.0

data/bin/dust

@@ -8,6 +8,7 @@ require 'erb'
 require 'fileutils'
 require 'ipaddress'
 require 'dust'
+require 'colorize'
 
 module  Dust
   class Deploy < Thor::Runner
@@ -15,72 +16,167 @@ module  Dust
     default_task :list
     check_unknown_options!
 
-    desc 'deploy [--yaml server.yaml] [--filter key=value,value2] [--recipes recipe1 recipe2] [--proxy host:port]',
-         'deploy all recipes to the node(s) specified in server.yaml or to all nodes defined in ./nodes/'
+    desc 'deploy', 'deploy all recipes to the node(s) specified in server.yaml or to all nodes defined in ./nodes/'
 
-    method_options :yaml => :string, :filter => :hash, :recipes => :array, :proxy => :string,
-                   :restart => :boolean, :reload => :boolean
+    method_option 'yaml', :type => :string, :desc => 'use only this server.yaml'
+    method_option 'filter', :type => :hash, :desc => 'only deploy to these hosts (e.g. environment:staging)'
+    method_option 'recipes', :type => :array, :desc => 'only deploy these recipes'
+    method_option 'proxy', :type => :string, :desc => 'socks proxy to use'
+    method_option 'restart', :type => :boolean, :desc => 'restart services after deploy'
+    method_option 'reload', :type => :boolean, :desc => 'reload services after deploy'
+    method_option 'parallel', :type => :boolean, :desc => 'deploy to all hosts at the same time using threads'
+    method_option 'summary', :type => :string, :desc => 'print summary of all events (all, warning, failed)'
 
     def deploy
       return unless check_dust_dir
       initialize_thorfiles
       Dust.print_failed 'no servers match this filter' if load_servers.empty?
 
-      run_recipes 'deploy'
+      # set global variables
+      $summary = options['summary']
+      $parallel = options['parallel']
+      $summary = 'all' if $parallel and not $summary
+
+      threads = []
+      @nodes.each_with_index do |node, i|
+        if $parallel
+          threads[i] = Thread.new do
+            run_recipes node, 'deploy'
+            Thread.current['hostname'] = node['hostname']
+          end
+        else
+          run_recipes node, 'deploy'
+        end
+      end
+
+      if $parallel
+        print 'waiting for servers: '
+        threads.each { |t| t.join; print t['hostname'].blue + ' ' }
+        puts
+      end
+
+      display_summary($summary) if $summary
     end
 
 
-    desc 'status [--yaml server.yaml] [--filter key=value,value2] [--recipes recipe1 recipe2] [--proxy host:port]',
-         'display status of recipes specified by filter'
+    desc 'status', 'display status of recipes specified by filter'
 
-    method_options :yaml => :string, :filter => :hash, :recipes => :array, :proxy => :string
+    method_option 'yaml', :type => :string, :desc => 'use only this server.yaml'
+    method_option 'filter', :type => :hash, :desc => 'only deploy to these hosts (e.g. environment:staging)'
+    method_option 'recipes', :type => :array, :desc => 'only deploy these recipes'
+    method_option 'proxy', :type => :string, :desc => 'socks proxy to use'
+    method_option 'parallel', :type => :boolean, :desc => 'deploy to all hosts at the same time using threads'
+    method_option 'summary', :type => :string, :desc => 'print summary of all events (all, warning, failed)'
 
     def status
       return unless check_dust_dir
       initialize_thorfiles
       Dust.print_failed 'no servers match this filter' if load_servers.empty?
 
-      run_recipes 'status'
+      # set global variables
+      $summary = options['summary']
+      $parallel = options['parallel']
+      $summary = 'all' if $parallel and not $summary
+
+      threads = []
+      @nodes.each_with_index do |node, i|
+        if $parallel
+          threads[i] = Thread.new do
+            run_recipes node, 'status'
+            Thread.current['hostname'] = node['hostname']
+          end
+        else
+          run_recipes node, 'status'
+        end
+      end
+
+      if $parallel
+        print 'waiting for servers: '
+        threads.each { |t| t.join; print t['hostname'].blue + ' ' }
+        puts
+      end
+
+      display_summary($summary) if $summary
     end
 
 
-    desc 'system_update [--yaml server.yaml] [--filter key=value,vale2] [--proxy host:port]',
-         'perform a full system upgrade (using aptitude, emerge, yum)'
+    desc 'system_update', 'perform a full system upgrade (using aptitude, emerge, yum)'
 
-    method_options :yaml => :string, :filter => :hash, :proxy => :string
+    method_option 'yaml', :type => :string, :desc => 'use only this server.yaml'
+    method_option 'filter', :type => :hash, :desc => 'only deploy to these hosts (e.g. environment:staging)'
+    method_option 'proxy', :type => :string, :desc => 'socks proxy to use'
+    method_option 'parallel', :type => :boolean, :desc => 'deploy to all hosts at the same time using threads'
+    method_option 'summary', :type => :string, :desc => 'print summary of all events (all, warning, failed)'
 
     def system_update
       return unless check_dust_dir
       initialize_thorfiles
       Dust.print_failed 'no servers match this filter' if load_servers.empty?
 
-      @nodes.each do |node|
-        # connect to server
-        server = Server.new node
-        next unless server.connect
-        server.system_update
-        server.disconnect
+      # set global variables
+      $summary = options['summary']
+      $parallel = options['parallel']
+      $summary = 'all' if $parallel and not $summary
+
+      threads = []
+      @nodes.each_with_index do |node, i|
+        if $parallel
+          threads[i] = Thread.new do
+            run_system_update(node)
+            Thread.current['hostname'] = node['hostname']
+          end
+        else
+          run_system_update(node)
+        end
+      end
+
+      if $parallel
+        print 'waiting for servers: '
+        threads.each { |t| t.join; print t['hostname'].blue + ' ' }
+        puts
       end
+
+      display_summary($summary) if $summary
     end
 
 
-    desc 'exec <command> [--yaml server.yaml] [--filter key=value,vale2] [--proxy host:port]',
-         'run a command on the server'
+    desc 'exec <command>', 'run a command on the server'
 
-    method_options :yaml => :string, :filter => :hash, :proxy => :string
+    method_option 'yaml', :type => :string, :desc => 'use only this server.yaml'
+    method_option 'filter', :type => :hash, :desc => 'only deploy to these hosts (e.g. environment:staging)'
+    method_option 'proxy', :type => :string, :desc => 'socks proxy to use'
+    method_option 'parallel', :type => :boolean, :desc => 'deploy to all hosts at the same time using threads'
+    method_option 'summary', :type => :string, :desc => 'print summary of all events (all, warning, failed)'
 
     def exec cmd, yaml=''
       return unless check_dust_dir
       initialize_thorfiles
       Dust.print_failed 'no servers match this filter' if load_servers.empty?
 
-      @nodes.each do |node|
-        # connect to server
-        server = Server.new node
-        next unless server.connect
-        server.exec cmd, :live => true
-        server.disconnect
+      # set global variables
+      $summary = options['summary']
+      $parallel = options['parallel']
+      $summary = 'all' if $parallel and not $summary
+
+      threads = []
+      @nodes.each_with_index do |node, i|
+        if $parallel
+          threads[i] = Thread.new do
+            run_exec(node, cmd)
+            Thread.current['hostname'] = node['hostname']
+          end
+        else
+          run_exec(node, cmd)
+        end
+      end
+
+      if $parallel
+        print 'waiting for servers: '
+        threads.each { |t| t.join; print t['hostname'].blue + ' ' }
+        puts
       end
+
+      display_summary($summary) if $summary
     end
 
 
@@ -111,53 +207,83 @@ module  Dust
     end
 
     # run specified recipes in the given context
-    def run_recipes context
-      @nodes.each do |node|
-        # skip this node if there are no recipes found
-        next unless node['recipes']
-          
-        recipes = generate_recipes node, context
-
-        # skip this node unless we're actually having recipes to cook
-        next if recipes.empty?
-
-        # connect to server
-        server = Server.new node
-        next unless server.connect
-
-        # runs the method with the recipe name, defined and included in recipe/*.rb
-        # call recipes for each recipe that is defined for this node
-        recipes.each do |recipe, config|
-          ::Dust.print_recipe recipe
-          send recipe, 'prepare', server, recipe, context, config, options
-          puts
-        end
+    def run_recipes(node, context)
+      # skip this node if there are no recipes found
+      return unless node['recipes']
+
+      recipes = generate_recipes node, context
+
+      # skip this node unless we're actually having recipes to cook
+      return if recipes.empty?
 
-        server.disconnect
+      # connect to server
+      node['server'] = Server.new node
+      return unless node['server'].connect
+
+      # runs the method with the recipe name, defined and included in recipe/*.rb
+      # call recipes for each recipe that is defined for this node
+      recipes.each do |recipe, config|
+        send recipe, 'prepare', node['server'], recipe, context, config, options
       end
+
+      node['server'].disconnect
+    end
+
+    def run_system_update(node)
+      node['server'] = Server.new(node)
+      return unless node['server'].connect
+      node['server'].system_update
+      node['server'].disconnect
+    end
+
+    def run_exec(node, cmd)
+      node['server'] = Server.new(node)
+      return unless node['server'].connect
+      node['server'].exec(cmd, :live => true)
+      node['server'].disconnect
     end
-    
+
     # generate list of recipes for this node
     def generate_recipes node, context
-      recipes = {}  
+      recipes = {}
       node['recipes'].each do |recipe, config|
-      
+
         # in case --recipes was set, skip unwanted recipes
-        next unless options[:recipes].include?(recipe) if options[:recipes]
-      
+        next unless options['recipes'].include?(recipe) if options['recipes']
+
         # skip disabled recipes
         next if config == 'disabled' or config.is_a? FalseClass
-      
+
         # check if method and thor task actually exist
         k = Thor::Util.find_by_namespace recipe
         next unless k
         next unless k.method_defined? context
-      
+
         recipes[recipe] = config
       end
       recipes
     end
-    
+
+    def display_summary(level)
+      puts "\n\n------------------------------ SUMMARY ------------------------------".red unless $parallel
+
+      @nodes.each do |node|
+        messages = node['server'].messages.collect(level)
+        next if messages.empty?
+
+        node['server'].messages.print_hostname_header(node['hostname'])
+
+        # display non-recipe messages first
+        msgs = messages.delete '_node'
+        msgs.each { |m| print m } if msgs
+
+        # display messages from recipes
+        messages.each do |recipe, msgs|
+          node['server'].messages.print_recipe_header(recipe)
+          msgs.each { |m| print m }
+        end
+      end
+    end
 
     # overwrite thorfiles to look for tasks in the recipes directories
     def thorfiles(relevant_to=nil, skip_lookup=false)
@@ -171,11 +297,11 @@ module  Dust
       # if the argument is empty, load all yaml files in the ./nodes/ directory
       # if the argument is a directory, load yaml files in this directory
       # if the argument is a file, load the file.
-      if options[:yaml]
-        if File.directory? options[:yaml]
-          yaml_files = Dir["#{options[:yaml]}/**/*.yaml"]
-        elsif File.exists? options[:yaml]
-          yaml_files = options[:yaml]
+      if options['yaml']
+        if File.directory? options['yaml']
+          yaml_files = Dir["#{options['yaml']}/**/*.yaml"]
+        elsif File.exists? options['yaml']
+          yaml_files = options['yaml']
         end
       else
         yaml_files = Dir['./nodes/**/*.yaml']
@@ -195,7 +321,7 @@ module  Dust
         # if there is not hostname field in the yaml file,
         # treat this node file as a template, and skip to the next one
         next unless node['hostname']
-  
+
         # look for the inherits field in the yaml file,
         # and merge the templates recursively into this node
         if node['inherits']
@@ -225,7 +351,7 @@ module  Dust
           end
 
           # pass command line proxy option
-          n['proxy'] = options[:proxy] if options[:proxy]
+          n['proxy'] = options['proxy'] if options['proxy']
 
           # add this node to the global node array
           @nodes.push n unless filtered? n
@@ -239,10 +365,10 @@ module  Dust
     def filtered? node
 
       # if filter is not specified, instantly return false
-      return false unless options[:filter]
+      return false unless options['filter']
 
       # remove items if other filter arguments don't match
-      options[:filter].each do |k, v|
+      options['filter'].each do |k, v|
         next unless v # skip empty filters
 
         # filter if this node doesn't even have the attribute

data/changelog.md

@@ -1,6 +1,43 @@
 Changelog
 =============
 
+0.13.0
+------------
+
+-  introduces --parallel and --summary
+   --summary -> warning e.g. shows a summary with all errors and warnings after completion
+
+   --parallel -> deploy to all hosts in parallel, using threads
+
+-  switches to new messaging system. using ::Dust.pring_* methods is now deprecated
+   please migrate your recipes to the new @node.messages.add() system
+
+    ::Dust.print_msg('checking something')
+    ::Dust.print_ok
+
+    msg = @node.messages.add('checking something')
+    msg.ok
+
+
+    ::Dust.print_ok('this went well')
+
+    @node.message.add('this went well').ok
+
+
+    ::Dust.print_message('executing something')
+    ::Dust.print_result(ret)
+
+    msg = @node.message.add('executing something')
+    msg.parse_result(ret)
+    
+
+-  redis doesn't configure sysctl anymore, please use sysctl redis template
+
+    recipes:
+      sysctl:
+        templates: redis
+
+
 0.12.2
 ------------
 

data/lib/dust.rb

@@ -1,5 +1,6 @@
 require 'dust/helper'
 require 'dust/print_status'
+require 'dust/messaging'
 require 'dust/server'
 require 'dust/recipe'
 

data/lib/dust/messaging.rb

@@ -0,0 +1,140 @@
+require 'colorize'
+
+module Dust
+
+  class Messages
+    attr_reader :current_recipe
+
+    def initialize
+      @store = {}
+
+      # store non-recipe messages in _node
+      start_recipe('_node')
+    end
+
+    def add(msg, options = {})
+      m = Message.new(msg, options)
+      @store[@current_recipe] << m
+      m
+    end
+
+    def start_recipe(recipe)
+      @current_recipe = recipe
+
+      # display recipe header, unless we're starting non-recipe messages
+      print_recipe_header(recipe) if recipe != '_node' and not $parallel 
+      
+      @store[@current_recipe] = []
+    end
+    
+    # print hostname
+    def print_hostname_header(host)
+      puts "\n\n[ #{host} ]".blue
+    end
+    
+    # print recipe name
+    def print_recipe_header(recipe)
+      puts "\n|#{recipe}|".green
+    end
+
+    def collect(level = 'all')
+      case level
+      when 'all'
+        l = [ 'none', 'ok', 'warning', 'failed' ]
+      when 'warning'
+        l = [ 'warning', 'failed' ]
+      when 'failed'
+        l = [ 'failed' ]
+      end
+
+      errors = {}
+      @store.each do |recipe, messages|
+        messages.each do |msg|
+          if l.include? msg.status
+            errors[recipe] ||= []
+            errors[recipe] << msg.text
+          end
+        end
+      end
+
+      errors
+    end
+
+  end
+
+  class Message
+    attr_reader :text, :status
+
+    def initialize(msg = '', options = {})
+      # merge default options
+      @options = { :quiet => false, :indent => 1 }.merge options
+
+      # autoflush
+      $stdout.sync = true
+
+      # just return if quiet mode is on
+      unless @options[:quiet]
+        # default status is 'message'
+        @status = 'none'
+
+        @text = indent + msg
+        print @text unless $parallel
+      end
+    end
+
+    def ok(msg = '')
+      unless @options[:quiet]
+        @text << msg + ' [ ok ]'.green + "\n"
+        puts msg + ' [ ok ]'.green unless $parallel
+        @status = 'ok'
+      end
+
+      true
+    end
+
+    def warning(msg = '')
+      unless @options[:quiet]
+        @text << msg + ' [ warning ]'.yellow + "\n"
+        puts msg + ' [ warning ]'.yellow unless $parallel
+        @status = 'warning'
+      end
+
+      true
+    end
+
+    def failed(msg = '')
+      unless @options[:quiet]
+        @text << msg + ' [ failed ]'.red + "\n"
+        puts msg + ' [ failed ]'.red unless $parallel
+        @status = 'failed'
+      end
+
+      false
+    end
+
+    def parse_result(ret)
+      return ok if ret == 0 or ret.is_a? TrueClass
+      failed
+    end
+
+    # prints stdout in grey and stderr in red (if existend)
+    def print_output(ret)
+      @text << indent + ret[:stdout].chomp.green + "\n" unless ret[:stdout].empty?
+      @text << indent + ret[:stderr].chomp.red + "\n" unless ret[:stderr].empty?
+
+      print @text unless $parallel
+    end
+    
+
+    private
+
+    # indent according to @options[:indent]
+    # indent 0
+    #  - indent 1
+    #    - indent 2
+    def indent
+      return '' if @options[:quiet] or @options[:indent] == 0
+      ' ' + '  ' * (@options[:indent] - 1) + '- '
+    end
+  end
+end