makitzo 0.1.2

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+gem 'activesupport', '~> 3.0.2'
+gem 'net-ssh', '~> 2.1.0'
+gem 'net-scp', '~> 1.0.4'
+gem 'highline', '~> 1.6.1'
+
+group :development do
+  gem "bundler"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,27 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.0.9)
+    git (1.2.5)
+    highline (1.6.2)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.1.4)
+    rake (0.9.2)
+    rcov (0.9.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 3.0.2)
+  bundler
+  highline (~> 1.6.1)
+  jeweler (~> 1.6.4)
+  net-scp (~> 1.0.4)
+  net-ssh (~> 2.1.0)
+  rcov

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Jason Frame
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown

@@ -0,0 +1,22 @@
+makitzo
+=======
+
+© 2011 Jason Frame [ [jason@onehackoranother.com](mailto:jason@onehackoranother.com) / [@jaz303](http://twitter.com/jaz303) ]  
+Released under the MIT License.
+
+makitzo is a remote-host Swiss army knife and has been battle-tested in an environment with ~150 nodes.
+
+Feature list:
+
+* Ruby configuration DSL
+* host migrations
+* key/value store for centralised storage of per-host config data, with pluggable backends
+* "helper" system for defining remote commands in Ruby code
+* run any command on a subset of defined hosts via SSH
+* compare output of commands across multiple hosts
+    
+Installation:
+
+    $ gem install makitzo
+    
+More documentation coming later!

data/RESOURCES

@@ -0,0 +1,2 @@
+http://code.google.com/p/rye/
+http://www.jedi.be/blog/2009/11/17/shell-scripting-dsl-in-ruby/

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "makitzo"
+  gem.homepage = "http://github.com/jaz303/makitzo"
+  gem.license = "MIT"
+  gem.summary = %Q{the swiss army knife of remote host manipulation}
+  gem.email = "jason@onehackoranother.com"
+  gem.authors = ["Jason Frame"]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "makitzo #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.2

data/bin/makitzo

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'makitzo'
+
+Makitzo::CLI.new.run(ARGV)

data/lib/makitzo/application.rb

@@ -0,0 +1,151 @@
+module Makitzo
+  class Application
+    include SSH::Multi
+    
+    attr_reader :config
+    attr_reader :logger
+    attr_accessor :query
+    attr_accessor :root_directory
+    
+    def initialize
+      @config = Config.new(self)
+      @logger = Logging::Collector.new
+      @root_directory = '.'
+    end
+    
+    def target_hosts
+      query = @query || World::Query.all
+      query.exec(config)
+    end
+    
+    def valid_commands
+      %w(install uninstall exec migrate create_migration list compare shell stream sudo)
+    end
+    
+    def invoke(command, *args)
+      raise ArgumentError, "unknown command: #{command}" unless valid_commands.include?(command)
+      success = false
+    
+      old_wd = Dir.getwd
+      Dir.chdir(@root_directory)
+      success = send(command.to_sym, *args)
+    
+      result = @logger.result
+      puts @logger.result unless result.length == 0
+      
+      success
+    ensure
+      Dir.chdir(old_wd)
+    end
+    
+    def install;      exec(:makitzo_install);   end
+    def uninstall;    exec(:makitzo_uninstall); end
+    
+    # Execute an aribtrary helper method on all target systems
+    def exec(*command)
+      config.store.open do
+        multi_session(target_hosts) { |session, host| session.send(*command) }
+      end
+    end
+    
+    # Migrate all target systems
+    def migrate
+      config.store.open do
+        migrator = Migrations::Migrator.new(self)
+        migrator.migrate(target_hosts)
+      end
+    end
+    
+    def create_migration(name)
+      generator = Migrations::Generator.new(self)
+      generator.create_migration(name)
+    end
+    
+    # List the hosts which would be affected, taking into account
+    # any query parameters.
+    def list
+      target_hosts.map { |h| h.name }.sort.each { |h| puts h }
+    end
+    
+    # Run a shell command on hosts
+    def shell(*command)
+      multi_session(target_hosts) { |session, host| session.exec(command.join(' ')) }
+    end
+    
+    # read from IO and send commands to target hosts
+    # this is intended to be used non-interactively (e.g. with a pipe)
+    # TODO: other system operations should be rewritten to use the "shell" service as
+    # it remembers state such as working directory etc
+    # TODO: how to we specify which shell to open?!
+    # TODO: this should probably be extracted
+    def stream(io)
+      reader = MultiplexedReader.new(io)
+      multi_ssh(target_hosts) do |host, conn, error|
+        logger.with_host(host) do
+          if error
+            logger.error("could not connect to host: #{error.message} (#{error.class})")
+          else
+            conn.open_channel do |ch|
+              ch.send_channel_request("shell") do |ch, success|
+                if success
+                  logger.info "shell opened"
+                  
+                  ch.on_data { |ch, data| }
+                  ch.on_close { logger.info "shell closed" }
+                  
+                  while line = reader.gets
+                    line.strip!
+                    logger.log_command_line(line)
+                    line += "\n"
+                    ch.send_data(line)
+                  end
+                  ch.send_data("exit\n")
+                else
+                  logger.error "shell could not be opened"
+                end
+              end
+            end
+            
+            conn.loop
+          end
+        end
+      end
+    end
+    
+    # Run a shell command on hosts using sudo
+    def sudo(*command)
+      multi_session(target_hosts) { |session, host| session.sudo { session.exec(command.join(' ')) } }
+    end
+    
+    # Run a shell command on hosts and compare output
+    def compare(*command)
+      command = command.join(' ')
+      sessions = nil
+      results, mutex = Hash.new { |h,k| h[k] = [] }, Mutex.new
+      
+      logger.silence do
+        sessions = multi_session(target_hosts) do |session, host|
+          result = session.exec(command)
+          mutex.synchronize { results[result] << host }
+        end
+      end
+      
+      logger.log_command_line(command)
+      
+      sessions.each do |s|
+        if s.connection_error
+          logger.error "connection to #{s.host.name} failed: #{s.connection_error.class} (#{s.connection_error.message})"
+        end
+      end
+      
+      logger.info("#{results.length} unique response(s)")
+      
+      results.each do |result, hosts|
+        hosts.each do |h|
+          logger.info(h.name + ":")
+        end
+        logger.log_command_status(result)
+      end
+    end
+  end
+end

data/lib/makitzo/application_aware.rb

@@ -0,0 +1,19 @@
+module Makitzo
+  module ApplicationAware
+    def app
+      @app
+    end
+    
+    def config
+      @app.config
+    end
+    
+    def logger
+      @app.logger
+    end
+    
+    def store
+      config.store
+    end
+  end
+end

data/lib/makitzo/cli.rb

@@ -0,0 +1,71 @@
+module Makitzo
+  class CLI
+    def run(args)
+      
+      # TODO: locate this automatically or allow it from a configuration file
+      worldfile = 'Worldfile'
+      root_directory = File.dirname(worldfile)
+      
+      app = Application.new
+      app.root_directory = root_directory
+      
+      query = World::Query.new
+      config = app.config
+      
+      trace = false
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: makitzo [options] command"
+        
+        opts.on("--trace", "Show backtrace on error") do
+          trace = true
+        end
+        
+        opts.on("--role [ROLE]", "Restrict command to role (may appear multiple times)") do |r|
+          query.roles << r
+        end
+        
+        opts.on("--host [HOST]", "Restrict command to host (may appear multiple times)") do |h|
+          query.hosts << h
+        end
+        
+        opts.on("-c", "--concurrency [NUM]", "Maximum number of connections to open in parallel") do |c|
+          config.concurrency = c.to_i
+        end
+        
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end
+      
+      opts.parse!(args)
+      
+      app.query = query
+      
+      config.instance_eval(File.read(worldfile))
+      
+      if $stdin.isatty
+        command = ARGV
+      else
+        command = ['stream', $stdin]
+      end
+      
+      if command.empty?
+        $stderr.puts "Error: no command specified"
+        $stderr.puts "  Valid commands: #{app.valid_commands.join(', ')}"
+        exit 1
+      end
+      
+      result = app.invoke(*command)
+    
+    rescue => e
+      $stderr.puts "#{e.message} (#{e.class})"
+      if trace
+        $stderr.puts e.backtrace.join("\n")
+      else
+        $stderr.puts "(run with --trace for detailed error information)"
+      end
+      exit 1
+    end
+  end
+end

data/lib/makitzo/config.rb

@@ -0,0 +1,148 @@
+module Makitzo
+  class Config
+    include ApplicationAware
+    include Settings
+    
+    def initialize(app)
+      @app = app
+      @options_stack = []
+      @terminal = HighLine.new
+      @store = nil
+      @concurrency = nil
+      
+      @helpers = Module.new do
+        def self.method_added(method_name)
+          if SSH::Context.protected_context_methods.include?(method_name.to_s)
+            raise "The method name '#{method_name}' is used internally by SSH sessions. Please rename your helper."
+          end
+        end
+      end
+      
+      @mutex = Mutex.new
+      initialize_roles
+      initialize_hosts
+    end
+    
+    #
+    #
+    
+    def concurrency=(concurrency)
+      @concurrency = concurrency
+    end
+    
+    def concurrency
+      @concurrency
+    end
+    
+    #
+    # Store
+    
+    def store=(store)
+      @store = store
+    end
+    
+    def store
+      raise Store::MissingStoreError if @store.nil?
+      @store
+    end
+    
+    #
+    # Helpers
+    
+    def helpers(&block)
+      @helpers.class_eval(&block) if block_given?
+      @helpers
+    end
+    
+    #
+    #
+    
+    def memoize(&block)
+      MemoizedProc.new(&block)
+    end
+    
+    def synchronize(&block)
+      @mutex.synchronize(&block)
+    end
+    
+    #
+    # Prompting
+    
+    extend Forwardable
+    def_delegators :@terminal, :agree, :ask, :choose, :say
+    
+    def password_prompt(prompt = 'Enter password: ')
+      ask(prompt) { |q| q.echo = false }
+    end
+    
+    #
+    # Options
+    
+    def with_options(options)
+      begin
+        @options_stack.push(options)
+        yield if block_given?
+      ensure
+        @options_stack.pop
+      end
+    end
+    
+    MERGER = lambda do |k,o,n|
+      if o.is_a?(Array)
+        n.is_a?(Array) ? (o + n) : (o.dup << n)
+      else
+        n
+      end
+    end
+    
+    def merged_options(extra_options = {})
+      opts = @options_stack.inject({}) { |m,hsh| m.update(hsh, &MERGER) }
+      opts.update(extra_options, &MERGER)
+    end
+    
+    #
+    # Hosts & Roles
+      
+    { 'role' => '::Makitzo::World::Role',
+      'host' => '::Makitzo::World::Host' }.each do |entity, klass|
+      class_eval <<-CODE
+        public
+          def #{entity}(name, options = {})
+            thing = #{klass}.new(@app, name, merged_options(options))
+            raise "Duplicate #{entity} name '\#{name}'" if @#{entity}s.include?(thing)
+            @#{entity}s << thing
+            @#{entity}_index[thing.name.to_s] = thing
+            yield thing if block_given?
+            thing
+          end
+          
+          def #{entity}_for_name(name)
+            @#{entity}_index[name.to_s]
+          end
+          
+          def #{entity}_for_name!(name)
+            #{entity}_for_name(name) or raise "Unknown #{entity} '#{name}'!"
+          end
+          
+          def #{entity}s
+            @#{entity}s.to_a
+          end
+          
+        private
+          def initialize_#{entity}s
+            @#{entity}s = Set.new
+            @#{entity}_index = {}
+          end
+      CODE
+    end
+    
+    def resolve_role(thing)
+      if thing.is_a?(::Makitzo::World::Role)
+        thing
+      else
+        role_for_name!(thing.to_s)
+      end
+    end
+
+  end
+end

data/lib/makitzo/file_system.rb

@@ -0,0 +1,24 @@
+module Makitzo
+  # classes including this module must define a #root method that returns
+  # the path of the makitzo control directory (e.g. /home/foo/makitzo)
+  module FileSystem
+    INSTALL_FILE            = 'INSTALL'
+    HISTORY_DIR             = 'migrations-history'
+    
+    def self.install_file(root)
+      File.join(root, INSTALL_FILE)
+    end
+    
+    def self.migration_history_dir(root)
+      File.join(root, HISTORY_DIR)
+    end
+    
+    def install_file
+      ::Makitzo::FileSystem.install_file(root)
+    end
+    
+    def migration_history_dir
+      ::Makitzo::FileSystem.migration_history_dir(root)
+    end
+  end
+end

data/lib/makitzo/logging/blackhole.rb

@@ -0,0 +1,16 @@
+module Makitzo; module Logging
+  class Blackhole
+    def with_host(host, &block); yield; end
+    def log_command(status); end
+    def overall_success!; end
+    def overall_error!; end
+    def error(msg); end
+    def success(msg); end
+    def notice(msg); end
+    def warn(msg); end
+    def info(msg); end
+    def debug(msg); end
+    def collector?; false; end
+    def result; ""; end
+  end
+end; end