makitzo 0.1.2

data/lib/makitzo/logging/collector.rb

@@ -0,0 +1,170 @@
+module Makitzo; module Logging
+  THREAD_HOST_KEY = 'makitzo.logging.host'
+  
+  # A logger which collects all log messages and displays a summary by host
+  class Collector
+    include Colorize
+    
+    attr_accessor :use_color
+    def use_color?; !!@use_color; end
+
+    def initialize
+      @use_color = true
+      @host = nil
+      @messages = []
+      @hosts = Hash.new { |h,k| h[k] = {:error => false, :messages => []} }
+      @lock = Mutex.new
+      @silenced = false
+    end
+    
+    # This method is not threadsafe. So call it before spawning threads.
+    def silence(&block)
+      begin
+        was_silenced = @silenced
+        @silenced = true
+        yield if block_given?
+      ensure
+        @silenced = was_silenced
+      end
+    end
+    
+    def with_host(host, &block)
+      return unless block_given?
+      
+      begin
+        set_current_host(host)
+        info("host is #{host.address}")
+        yield
+      ensure
+        set_current_host(nil)
+      end
+      
+      nil
+    end
+
+    # logs a command
+    # options[:command] - override command line to be logged. useful for masking passwords.
+    def log_command(status, options = {})
+      command = options[:command] || status.command
+      
+      log_command_line(command, status.success?)
+      log_command_status(status)
+      
+      overall_error! if current_host && !status.success?
+    end
+    
+    def log_command_line(command, success = true)
+      if command.is_a?(Net::SSH::Connection::Session::ExecStatus)
+        success = command.success?
+        command = command.command
+      end
+      if success
+        append green("$", true), " ", green(sanitize(command))
+      else
+        append red("$", true), " ", red(sanitize(command))
+      end
+    end
+    
+    def log_command_status(result, success = true)
+      if result.is_a?(Net::SSH::Connection::Session::ExecStatus)
+        success = result.success?
+        result  = (success ? result.stdout : result.stderr).last_line.strip
+      end
+      unless result.empty?
+        if success
+          append green("-", true), " ", green(sanitize(result))
+        else
+          append red("!", true), " ", red(sanitize(result))
+        end
+      end
+    end
+    
+    def overall_success!
+      raise "Cannot log host success when no host is set" unless current_host
+      @hosts[current_host.to_s][:error] = false
+    end
+    
+    def overall_error!
+      raise "Cannot log host error when no host is set" unless current_host
+      @hosts[current_host.to_s][:error] = true
+    end
+    
+    def error(message)
+      append red("[ERROR]", true), ' ', red(sanitize(message))
+      overall_error! if current_host
+    end
+    
+    def success(message)
+      append green("[OK]", true), ' ', sanitize(message)
+    end
+    
+    def notice(message)
+      append cyan("[NOTICE]", true), ' ', sanitize(message)
+    end
+    
+    def warn(message)
+      append yellow("[WARNING]", true), ' ', sanitize(message)
+    end
+    
+    def info(message)
+      append '[INFO]', ' ', sanitize(message)
+    end
+    
+    def debug(message)
+      append blue('[DEBUG]', true), ' ', sanitize(message)
+    end
+    
+    def collector?
+      true
+    end
+    
+    def result
+      out = ""
+      
+      @hosts.keys.sort.each do |host_name|
+        host_status = @hosts[host_name]
+        next if host_status[:messages].empty?
+        out << magenta('* ' + host_name, true) << " " << (host_status[:error] ? red('[ERROR]', true) : green('[OK]', true)) << "\n"
+        host_status[:messages].each { |m| out << m.indent(2) << "\n" }
+        out << "\n"
+      end
+      
+      unless @messages.empty?
+        out << magenta("* Global Messages", true) << "\n"
+        @messages.each { |m| out << m.indent(2) << "\n" }
+        out << "\n"
+      end
+      
+      out
+    end
+    
+    def append(*chunks)
+      unless @silenced
+        @lock.synchronize do
+          active_log << chunks.join('').strip
+        end
+      end
+    end
+    
+  private
+  
+    def sanitize(message)
+      message.to_s
+    end
+  
+    def active_log
+      current_host ? @hosts[current_host.to_s][:messages] : @messages
+    end
+    
+    def current_host
+      Thread.current[THREAD_HOST_KEY]
+    end
+    
+    def set_current_host(host)
+      raise "Cannot set host; host already set" if host && current_host
+      Thread.current[THREAD_HOST_KEY] = host
+    end
+  
+  end
+
+end; end

data/lib/makitzo/logging/colorize.rb

@@ -0,0 +1,38 @@
+module Makitzo; module Logging
+  module Colorize
+    def bold(text)
+      if use_color?
+        "\033[1m#{text}\033[0m"
+      else
+        text
+      end
+    end
+    
+    def colorize(text, ansi, bold = false)
+      if use_color?
+        code = "\033["
+        code << "1;" if bold
+        code << "#{ansi}m"
+        code + text + "\033[0m"
+      else
+        text
+      end
+    end
+    
+    { :black      => 30,
+      :red        => 31,
+      :green      => 32,
+      :yellow     => 33,
+      :blue       => 34,
+      :magenta    => 35,
+      :cyan       => 36,
+      :white      => 37
+    }.each do |color, ansi|
+      class_eval <<-CODE
+        def #{color}(text, bold = false)
+          colorize(text, #{ansi}, bold)
+        end
+      CODE
+    end
+  end
+end; end

data/lib/makitzo/memoized_proc.rb

@@ -0,0 +1,15 @@
+module Makitzo
+  class MemoizedProc
+    def initialize(&block)
+      @proc = block
+    end
+    
+    def call(*args)
+      if defined?(@value)
+        @value
+      else
+        @value = @proc.call(*args)
+      end
+    end
+  end
+end

data/lib/makitzo/migrations/commands.rb

@@ -0,0 +1,11 @@
+module Makitzo; module Migrations
+  module Commands
+    def upload_migration_file(name)
+      target = remote_migration_file(name)
+      scp_upload(local_migration_file(name), target)
+      target
+    end
+    
+    bangify :upload_migration_file
+  end
+end; end

data/lib/makitzo/migrations/generator.rb

@@ -0,0 +1,24 @@
+module Makitzo; module Migrations
+  class Generator
+    include ApplicationAware
+    include Paths
+    
+    def initialize(app)
+      @app = app
+    end
+    
+    def create_migration(name)
+      @migration_name = name
+      @migration_timestamp = Time.now.to_i
+      @migration_directory = File.join(local_migration_path, "#{@migration_timestamp}_#{@migration_name}")
+      @migration_class_name = @migration_name.camelize
+      
+      template = ERB.new(File.read(File.join(Makitzo::ROOT, 'templates', 'migration.erb')))
+      
+      FileUtils.mkdir_p(@migration_directory)
+      
+      migration_source = template.result(binding)
+      File.open(File.join(@migration_directory, 'migration.rb'), 'w') { |f| f.write(migration_source) }
+    end
+  end
+end; end

data/lib/makitzo/migrations/migration.rb

@@ -0,0 +1,69 @@
+module Makitzo; module Migrations
+  class Migration < Makitzo::SSH::Context
+    class << self
+      def timestamp;      @timestamp;       end
+      def timestamp=(ts); @timestamp = ts;  end
+      
+      def directory;      @directory;       end
+      def directory=(d);  @directory = d;   end
+      
+      def roles(*roles)
+        @roles ||= []
+        @roles.concat([roles].flatten) unless roles.empty?
+        @roles
+      end
+      
+      def hosts(*hosts)
+        @hosts ||= []
+        @hosts.concat([hosts].flatten) unless hosts.empty?
+        @hosts
+      end
+      
+      def query
+        unless @query
+          @query = World::Query.new
+          roles.each { |r| @query.roles << r }
+          hosts.each { |h| @query.hosts << h }
+        end
+        @query
+      end
+      
+      alias_method :role, :roles
+      alias_method :host, :hosts
+      
+      # Returns an array of methods which are required by migrations.
+      # Used to prevent helpers from defining conflicting methods.
+      def protected_context_methods
+        %w(up down local_directory local_migration_file remote_directory remote_migration_file)
+      end
+    end
+    
+    def local_directory
+      self.class.directory
+    end
+    
+    def local_migration_file(file)
+      File.join(local_directory, file)
+    end
+    
+    def remote_directory
+      File.join(host.migration_history_dir, self.class.timestamp.to_s)
+    end
+    
+    def remote_migration_file(file)
+      File.join(remote_directory, file)
+    end
+    
+    def up
+      raise UnsupportedMigrationError, "up direction is not defined!"
+    end
+    
+    def down
+      raise UnsupportedMigrationError, "down direction is not defined!"
+    end
+    
+    def to_i
+      self.class.timestamp
+    end
+  end
+end; end

data/lib/makitzo/migrations/migrator.rb

@@ -0,0 +1,87 @@
+module Makitzo; module Migrations
+  class Migrator
+    include ApplicationAware
+    include SSH::Multi
+    include Paths
+    
+    def initialize(app)
+      @app = app
+    end
+    
+    def migrate(target_hosts)
+      all_migrations = migrations
+      return if all_migrations.empty?
+      
+      # start with a query matching all hosts and reduce to set of hosts
+      # affected by existing migrations
+      migration_hosts_query = World::Query.all
+      all_migrations.each { |m| migration_hosts_query.merge!(m.query) }
+      
+      # get list of hosts and intersect with hosts specified on command-line
+      migration_hosts = migration_hosts_query.exec(config)
+      migration_hosts &= target_hosts
+      
+      # finally, remove any hosts with no pending migrations
+      applied_migrations = store.applied_migrations_for_all_hosts
+      migration_hosts.delete_if { |host|
+        all_migrations.all? { |m|
+          (applied_migrations[host.to_s] || []).include?(m.timestamp) || !m.query.includes?(host)
+        }
+      }
+      
+      multi_ssh(migration_hosts) do |host, conn, error|
+        logger.with_host(host) do
+          if error
+            # log connection error
+          else
+            begin
+              overseer_session = ssh_context_class.new(host, conn)
+              overseer_session.makitzo_install_check!
+          
+              # then select the appropriate migrations, create session classes and run
+              all_migrations.each do |migration_klass|
+                next if (applied_migrations[host.to_s] || []).include?(migration_klass.timestamp)
+                next unless migration_klass.query.includes?(host)
+                migration = migration_klass.new(host, conn)
+                migration.exec!("mkdir -p #{migration.x(migration.remote_directory)}")
+                migration.up
+                host.mark_migration_as_applied(migration)
+                logger.success "Migration #{migration_klass.timestamp} applied"
+              end
+            rescue SSH::CommandFailed => e
+              logger.error "Migration failed"
+            end
+          end
+        end
+      end
+    end
+    
+  private
+  
+    def migrations
+      unless @migrations
+        @migrations = Dir["#{local_migration_path}/*"].map do |candidate|
+          next unless File.directory?(candidate)
+          timestamp, *rest = File.basename(candidate).split('_')
+          class_name = rest.join('_').camelize
+          begin
+            require File.join(candidate, 'migration.rb')
+            klass = class_name.constantize
+          rescue LoadError => e
+            raise MigrationNotFound, "migration file not found: #{candidate}"
+          rescue NameError => e
+            raise MigrationNotFound, "expected #{candidate} to define #{class_name}"
+          end
+          klass.timestamp = timestamp.to_i
+          klass.directory = File.expand_path(candidate)
+          klass.send(:include, config.helpers)
+          klass
+        end
+        @migrations.compact!
+        @migrations.sort { |m1,m2| m1.timestamp <=> m2.timestamp }
+      end
+      @migrations
+    end
+    
+  end
+end; end

data/lib/makitzo/migrations/paths.rb

@@ -0,0 +1,7 @@
+module Makitzo; module Migrations
+  module Paths
+    def local_migration_path
+      File.join(app.root_directory, 'migrations')
+    end
+  end
+end; end

data/lib/makitzo/monkeys/array.rb

@@ -0,0 +1,10 @@
+class Array
+  def in_groups_of(group_size)
+    out = []
+    each_with_index do |ele, i|
+      out << [] if i % group_size == 0
+      out.last << ele
+    end
+    out
+  end
+end

data/lib/makitzo/monkeys/bangify.rb

@@ -0,0 +1,15 @@
+class Module
+  def bangify(*methods)
+    exception_class = 'RuntimeError'
+    exception_class = methods.pop if (methods.last.is_a?(Class) || methods.last =~ /^[A-Z]/)
+    methods.each do |method|
+      class_eval <<-CODE
+        def #{method}!(*args, &block)
+          result = #{method}(*args, &block)
+          raise #{exception_class} unless result
+          result
+        end
+      CODE
+    end
+  end
+end

data/lib/makitzo/monkeys/net-ssh.rb

@@ -0,0 +1,73 @@
+class Net::SSH::Connection::Session
+  class ExecStatus
+    attr_accessor :command, :stdout, :stderr, :exit_code, :exit_signal
+    
+    def initialize
+      @command, @stdout, @stderr, @exit_code, @exit_signal = "", "", "", 0, 0, nil
+    end
+    
+    def success?
+      @exit_code == 0
+    end
+    
+    def error?
+      !success?
+    end
+    
+    def to_i
+      @exit_code
+    end
+    
+    def to_s
+      @stdout
+    end
+    
+    def inspect
+      "<ExecStatus command=#{@command.inspect} stdout=#{@stdout.inspect} status=#{@exit_code}>"
+    end
+    
+    def hash
+      "#{command}\n#{exit_code}\n#{stdout}".hash
+    end
+    
+    def eql?(other)
+      other.is_a?(ExecStatus) && (command == other.command && exit_code == other.exit_code && stdout == other.stdout)
+    end
+  end
+  
+  # Adapted from:
+  # http://stackoverflow.com/questions/3386233/how-to-get-exit-status-with-rubys-netssh-library
+  #
+  # FIXME: we're currently opening a channel per command which strikes me
+  # as inefficient and prevents, for example, working directory from
+  # persisting across requests.
+  def exec2!(command, options = {})
+    options = {:log => true}.update(options)
+    
+    status = ExecStatus.new
+    status.command = command
+    
+    ch = open_channel do |channel|
+      channel.exec(command) do |ch, success|
+        raise "could not execute command: #{command.inspect}" unless success
+        
+        channel.on_data { |ch, data| status.stdout << data }
+        channel.on_extended_data { |ch, type, data| status.stderr << data }
+        channel.on_request("exit-status") { |ch,data| status.exit_code = data.read_long }
+        channel.on_request("exit-signal") { |ch,data| status.exit_signal = data.read_long }
+      end
+    end
+    
+    self.loop
+    
+    if options[:log] && self[:logger]
+      if options[:log].is_a?(String)
+        self[:logger].log_command(status, :command => options[:log])
+      else
+        self[:logger].log_command(status)
+      end
+    end
+    
+    status
+  end
+end

data/lib/makitzo/monkeys/string.rb

@@ -0,0 +1,15 @@
+class String
+  def indent(spaces)
+    gsub(/^/, " " * spaces)
+  end
+  
+  # returns last non-empty line of string, or the empty string if none exists
+  def last_line
+    lines = split("\n")
+    while line = lines.pop
+      line.strip!
+      return line unless line.length == 0
+    end
+    return ''
+  end
+end

data/lib/makitzo/multiplexed_reader.rb

@@ -0,0 +1,26 @@
+module Makitzo
+  # relays IO from a single source to multiple threads
+  # each thread will see the same input.
+  # necessarily has to store all data as set of reader threads is not known.
+  # possible solution: associate reader with ThreadGroup so we can access
+  # list of threads. can then keep track of threads that have read least/most
+  # data and discard anything that's no longer needed.
+  class MultiplexedReader
+    def initialize(io)
+      @io, @mutex, @state, @lines = io, Mutex.new, Hash.new { |h,k| h[k] = 0 }, []
+    end
+    
+    def gets
+      @mutex.synchronize do
+        lines_read_by_thread = @state[Thread.current]
+        if lines_read_by_thread >= @lines.count
+          @lines << @io.gets
+        end
+        @state[Thread.current] = lines_read_by_thread + 1
+        
+        line = @lines[lines_read_by_thread]
+        line ? line.dup : line
+      end
+    end
+  end
+end

data/lib/makitzo/settings.rb

@@ -0,0 +1,30 @@
+module Makitzo
+  # mixin providing classes with a settings hash
+  module Settings
+    def settings
+      @settings ||= {}
+    end
+
+    def [](key)
+      read(key)
+    end
+  
+    def []=(key, value)
+      set(key, value)
+    end
+    
+    def read(key, default = nil)
+      val = settings[key.to_sym]
+      val = val.call if val.respond_to?(:call)
+      val.nil? ? default : val
+    end
+
+    def set(key, value = nil, &block)
+      settings[key.to_sym] = block_given? ? block : value
+    end
+  
+    def memo(key, &block)
+      set(key, MemoizedProc.new(&block))
+    end
+  end
+end