makitzo 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. data/Gemfile +12 -0
  2. data/Gemfile.lock +27 -0
  3. data/LICENSE.txt +20 -0
  4. data/README.mdown +22 -0
  5. data/RESOURCES +2 -0
  6. data/Rakefile +51 -0
  7. data/VERSION +1 -0
  8. data/bin/makitzo +6 -0
  9. data/lib/makitzo/application.rb +151 -0
  10. data/lib/makitzo/application_aware.rb +19 -0
  11. data/lib/makitzo/cli.rb +71 -0
  12. data/lib/makitzo/config.rb +148 -0
  13. data/lib/makitzo/file_system.rb +24 -0
  14. data/lib/makitzo/logging/blackhole.rb +16 -0
  15. data/lib/makitzo/logging/collector.rb +170 -0
  16. data/lib/makitzo/logging/colorize.rb +38 -0
  17. data/lib/makitzo/memoized_proc.rb +15 -0
  18. data/lib/makitzo/migrations/commands.rb +11 -0
  19. data/lib/makitzo/migrations/generator.rb +24 -0
  20. data/lib/makitzo/migrations/migration.rb +69 -0
  21. data/lib/makitzo/migrations/migrator.rb +87 -0
  22. data/lib/makitzo/migrations/paths.rb +7 -0
  23. data/lib/makitzo/monkeys/array.rb +10 -0
  24. data/lib/makitzo/monkeys/bangify.rb +15 -0
  25. data/lib/makitzo/monkeys/net-ssh.rb +73 -0
  26. data/lib/makitzo/monkeys/string.rb +15 -0
  27. data/lib/makitzo/multiplexed_reader.rb +26 -0
  28. data/lib/makitzo/settings.rb +30 -0
  29. data/lib/makitzo/ssh/commands/apple.rb +104 -0
  30. data/lib/makitzo/ssh/commands/file_system.rb +59 -0
  31. data/lib/makitzo/ssh/commands/file_transfer.rb +9 -0
  32. data/lib/makitzo/ssh/commands/http.rb +51 -0
  33. data/lib/makitzo/ssh/commands/makitzo.rb +46 -0
  34. data/lib/makitzo/ssh/commands/ruby.rb +18 -0
  35. data/lib/makitzo/ssh/commands/unix.rb +7 -0
  36. data/lib/makitzo/ssh/context.rb +91 -0
  37. data/lib/makitzo/ssh/multi.rb +79 -0
  38. data/lib/makitzo/store/mysql.rb +176 -0
  39. data/lib/makitzo/store/skeleton.rb +46 -0
  40. data/lib/makitzo/world/host.rb +84 -0
  41. data/lib/makitzo/world/named_entity.rb +41 -0
  42. data/lib/makitzo/world/query.rb +54 -0
  43. data/lib/makitzo/world/role.rb +4 -0
  44. data/lib/makitzo.rb +90 -0
  45. data/makitzo.gemspec +106 -0
  46. data/templates/migration.erb +9 -0
  47. data/test/helper.rb +17 -0
  48. data/test/test_makitzo.rb +7 -0
  49. metadata +222 -0
data/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'activesupport', '~> 3.0.2'
4
+ gem 'net-ssh', '~> 2.1.0'
5
+ gem 'net-scp', '~> 1.0.4'
6
+ gem 'highline', '~> 1.6.1'
7
+
8
+ group :development do
9
+ gem "bundler"
10
+ gem "jeweler", "~> 1.6.4"
11
+ gem "rcov", ">= 0"
12
+ end
data/Gemfile.lock ADDED
@@ -0,0 +1,27 @@
1
+ GEM
2
+ remote: http://rubygems.org/
3
+ specs:
4
+ activesupport (3.0.9)
5
+ git (1.2.5)
6
+ highline (1.6.2)
7
+ jeweler (1.6.4)
8
+ bundler (~> 1.0)
9
+ git (>= 1.2.5)
10
+ rake
11
+ net-scp (1.0.4)
12
+ net-ssh (>= 1.99.1)
13
+ net-ssh (2.1.4)
14
+ rake (0.9.2)
15
+ rcov (0.9.9)
16
+
17
+ PLATFORMS
18
+ ruby
19
+
20
+ DEPENDENCIES
21
+ activesupport (~> 3.0.2)
22
+ bundler
23
+ highline (~> 1.6.1)
24
+ jeweler (~> 1.6.4)
25
+ net-scp (~> 1.0.4)
26
+ net-ssh (~> 2.1.0)
27
+ rcov
data/LICENSE.txt ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2011 Jason Frame
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.mdown ADDED
@@ -0,0 +1,22 @@
1
+ makitzo
2
+ =======
3
+
4
+ © 2011 Jason Frame [ [jason@onehackoranother.com](mailto:jason@onehackoranother.com) / [@jaz303](http://twitter.com/jaz303) ]
5
+ Released under the MIT License.
6
+
7
+ makitzo is a remote-host Swiss army knife and has been battle-tested in an environment with ~150 nodes.
8
+
9
+ Feature list:
10
+
11
+ * Ruby configuration DSL
12
+ * host migrations
13
+ * key/value store for centralised storage of per-host config data, with pluggable backends
14
+ * "helper" system for defining remote commands in Ruby code
15
+ * run any command on a subset of defined hosts via SSH
16
+ * compare output of commands across multiple hosts
17
+
18
+ Installation:
19
+
20
+ $ gem install makitzo
21
+
22
+ More documentation coming later!
data/RESOURCES ADDED
@@ -0,0 +1,2 @@
1
+ http://code.google.com/p/rye/
2
+ http://www.jedi.be/blog/2009/11/17/shell-scripting-dsl-in-ruby/
data/Rakefile ADDED
@@ -0,0 +1,51 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+ begin
6
+ Bundler.setup(:default, :development)
7
+ rescue Bundler::BundlerError => e
8
+ $stderr.puts e.message
9
+ $stderr.puts "Run `bundle install` to install missing gems"
10
+ exit e.status_code
11
+ end
12
+ require 'rake'
13
+
14
+ require 'jeweler'
15
+ Jeweler::Tasks.new do |gem|
16
+ # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
17
+ gem.name = "makitzo"
18
+ gem.homepage = "http://github.com/jaz303/makitzo"
19
+ gem.license = "MIT"
20
+ gem.summary = %Q{the swiss army knife of remote host manipulation}
21
+ gem.email = "jason@onehackoranother.com"
22
+ gem.authors = ["Jason Frame"]
23
+ end
24
+ Jeweler::RubygemsDotOrgTasks.new
25
+
26
+ require 'rake/testtask'
27
+ Rake::TestTask.new(:test) do |test|
28
+ test.libs << 'lib' << 'test'
29
+ test.pattern = 'test/**/test_*.rb'
30
+ test.verbose = true
31
+ end
32
+
33
+ require 'rcov/rcovtask'
34
+ Rcov::RcovTask.new do |test|
35
+ test.libs << 'test'
36
+ test.pattern = 'test/**/test_*.rb'
37
+ test.verbose = true
38
+ test.rcov_opts << '--exclude "gems/*"'
39
+ end
40
+
41
+ task :default => :test
42
+
43
+ require 'rake/rdoctask'
44
+ Rake::RDocTask.new do |rdoc|
45
+ version = File.exist?('VERSION') ? File.read('VERSION') : ""
46
+
47
+ rdoc.rdoc_dir = 'rdoc'
48
+ rdoc.title = "makitzo #{version}"
49
+ rdoc.rdoc_files.include('README*')
50
+ rdoc.rdoc_files.include('lib/**/*.rb')
51
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.2
data/bin/makitzo ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ require 'makitzo'
5
+
6
+ Makitzo::CLI.new.run(ARGV)
@@ -0,0 +1,151 @@
1
+ module Makitzo
2
+ class Application
3
+ include SSH::Multi
4
+
5
+ attr_reader :config
6
+ attr_reader :logger
7
+ attr_accessor :query
8
+ attr_accessor :root_directory
9
+
10
+ def initialize
11
+ @config = Config.new(self)
12
+ @logger = Logging::Collector.new
13
+ @root_directory = '.'
14
+ end
15
+
16
+ def target_hosts
17
+ query = @query || World::Query.all
18
+ query.exec(config)
19
+ end
20
+
21
+ def valid_commands
22
+ %w(install uninstall exec migrate create_migration list compare shell stream sudo)
23
+ end
24
+
25
+ def invoke(command, *args)
26
+ raise ArgumentError, "unknown command: #{command}" unless valid_commands.include?(command)
27
+ success = false
28
+
29
+ old_wd = Dir.getwd
30
+ Dir.chdir(@root_directory)
31
+ success = send(command.to_sym, *args)
32
+
33
+ result = @logger.result
34
+ puts @logger.result unless result.length == 0
35
+
36
+ success
37
+ ensure
38
+ Dir.chdir(old_wd)
39
+ end
40
+
41
+ def install; exec(:makitzo_install); end
42
+ def uninstall; exec(:makitzo_uninstall); end
43
+
44
+ # Execute an aribtrary helper method on all target systems
45
+ def exec(*command)
46
+ config.store.open do
47
+ multi_session(target_hosts) { |session, host| session.send(*command) }
48
+ end
49
+ end
50
+
51
+ # Migrate all target systems
52
+ def migrate
53
+ config.store.open do
54
+ migrator = Migrations::Migrator.new(self)
55
+ migrator.migrate(target_hosts)
56
+ end
57
+ end
58
+
59
+ def create_migration(name)
60
+ generator = Migrations::Generator.new(self)
61
+ generator.create_migration(name)
62
+ end
63
+
64
+ # List the hosts which would be affected, taking into account
65
+ # any query parameters.
66
+ def list
67
+ target_hosts.map { |h| h.name }.sort.each { |h| puts h }
68
+ end
69
+
70
+ # Run a shell command on hosts
71
+ def shell(*command)
72
+ multi_session(target_hosts) { |session, host| session.exec(command.join(' ')) }
73
+ end
74
+
75
+ # read from IO and send commands to target hosts
76
+ # this is intended to be used non-interactively (e.g. with a pipe)
77
+ # TODO: other system operations should be rewritten to use the "shell" service as
78
+ # it remembers state such as working directory etc
79
+ # TODO: how to we specify which shell to open?!
80
+ # TODO: this should probably be extracted
81
+ def stream(io)
82
+ reader = MultiplexedReader.new(io)
83
+ multi_ssh(target_hosts) do |host, conn, error|
84
+ logger.with_host(host) do
85
+ if error
86
+ logger.error("could not connect to host: #{error.message} (#{error.class})")
87
+ else
88
+ conn.open_channel do |ch|
89
+ ch.send_channel_request("shell") do |ch, success|
90
+ if success
91
+ logger.info "shell opened"
92
+
93
+ ch.on_data { |ch, data| }
94
+ ch.on_close { logger.info "shell closed" }
95
+
96
+ while line = reader.gets
97
+ line.strip!
98
+ logger.log_command_line(line)
99
+ line += "\n"
100
+ ch.send_data(line)
101
+ end
102
+ ch.send_data("exit\n")
103
+ else
104
+ logger.error "shell could not be opened"
105
+ end
106
+ end
107
+ end
108
+
109
+ conn.loop
110
+ end
111
+ end
112
+ end
113
+ end
114
+
115
+ # Run a shell command on hosts using sudo
116
+ def sudo(*command)
117
+ multi_session(target_hosts) { |session, host| session.sudo { session.exec(command.join(' ')) } }
118
+ end
119
+
120
+ # Run a shell command on hosts and compare output
121
+ def compare(*command)
122
+ command = command.join(' ')
123
+ sessions = nil
124
+ results, mutex = Hash.new { |h,k| h[k] = [] }, Mutex.new
125
+
126
+ logger.silence do
127
+ sessions = multi_session(target_hosts) do |session, host|
128
+ result = session.exec(command)
129
+ mutex.synchronize { results[result] << host }
130
+ end
131
+ end
132
+
133
+ logger.log_command_line(command)
134
+
135
+ sessions.each do |s|
136
+ if s.connection_error
137
+ logger.error "connection to #{s.host.name} failed: #{s.connection_error.class} (#{s.connection_error.message})"
138
+ end
139
+ end
140
+
141
+ logger.info("#{results.length} unique response(s)")
142
+
143
+ results.each do |result, hosts|
144
+ hosts.each do |h|
145
+ logger.info(h.name + ":")
146
+ end
147
+ logger.log_command_status(result)
148
+ end
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,19 @@
1
+ module Makitzo
2
+ module ApplicationAware
3
+ def app
4
+ @app
5
+ end
6
+
7
+ def config
8
+ @app.config
9
+ end
10
+
11
+ def logger
12
+ @app.logger
13
+ end
14
+
15
+ def store
16
+ config.store
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,71 @@
1
+ module Makitzo
2
+ class CLI
3
+ def run(args)
4
+
5
+ # TODO: locate this automatically or allow it from a configuration file
6
+ worldfile = 'Worldfile'
7
+ root_directory = File.dirname(worldfile)
8
+
9
+ app = Application.new
10
+ app.root_directory = root_directory
11
+
12
+ query = World::Query.new
13
+ config = app.config
14
+
15
+ trace = false
16
+ opts = OptionParser.new do |opts|
17
+ opts.banner = "Usage: makitzo [options] command"
18
+
19
+ opts.on("--trace", "Show backtrace on error") do
20
+ trace = true
21
+ end
22
+
23
+ opts.on("--role [ROLE]", "Restrict command to role (may appear multiple times)") do |r|
24
+ query.roles << r
25
+ end
26
+
27
+ opts.on("--host [HOST]", "Restrict command to host (may appear multiple times)") do |h|
28
+ query.hosts << h
29
+ end
30
+
31
+ opts.on("-c", "--concurrency [NUM]", "Maximum number of connections to open in parallel") do |c|
32
+ config.concurrency = c.to_i
33
+ end
34
+
35
+ opts.on_tail("-h", "--help", "Show this message") do
36
+ puts opts
37
+ exit
38
+ end
39
+ end
40
+
41
+ opts.parse!(args)
42
+
43
+ app.query = query
44
+
45
+ config.instance_eval(File.read(worldfile))
46
+
47
+ if $stdin.isatty
48
+ command = ARGV
49
+ else
50
+ command = ['stream', $stdin]
51
+ end
52
+
53
+ if command.empty?
54
+ $stderr.puts "Error: no command specified"
55
+ $stderr.puts " Valid commands: #{app.valid_commands.join(', ')}"
56
+ exit 1
57
+ end
58
+
59
+ result = app.invoke(*command)
60
+
61
+ rescue => e
62
+ $stderr.puts "#{e.message} (#{e.class})"
63
+ if trace
64
+ $stderr.puts e.backtrace.join("\n")
65
+ else
66
+ $stderr.puts "(run with --trace for detailed error information)"
67
+ end
68
+ exit 1
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,148 @@
1
+ module Makitzo
2
+ class Config
3
+ include ApplicationAware
4
+ include Settings
5
+
6
+ def initialize(app)
7
+ @app = app
8
+ @options_stack = []
9
+ @terminal = HighLine.new
10
+ @store = nil
11
+ @concurrency = nil
12
+
13
+ @helpers = Module.new do
14
+ def self.method_added(method_name)
15
+ if SSH::Context.protected_context_methods.include?(method_name.to_s)
16
+ raise "The method name '#{method_name}' is used internally by SSH sessions. Please rename your helper."
17
+ end
18
+ end
19
+ end
20
+
21
+ @mutex = Mutex.new
22
+ initialize_roles
23
+ initialize_hosts
24
+ end
25
+
26
+ #
27
+ #
28
+
29
+ def concurrency=(concurrency)
30
+ @concurrency = concurrency
31
+ end
32
+
33
+ def concurrency
34
+ @concurrency
35
+ end
36
+
37
+ #
38
+ # Store
39
+
40
+ def store=(store)
41
+ @store = store
42
+ end
43
+
44
+ def store
45
+ raise Store::MissingStoreError if @store.nil?
46
+ @store
47
+ end
48
+
49
+ #
50
+ # Helpers
51
+
52
+ def helpers(&block)
53
+ @helpers.class_eval(&block) if block_given?
54
+ @helpers
55
+ end
56
+
57
+ #
58
+ #
59
+
60
+ def memoize(&block)
61
+ MemoizedProc.new(&block)
62
+ end
63
+
64
+ def synchronize(&block)
65
+ @mutex.synchronize(&block)
66
+ end
67
+
68
+ #
69
+ # Prompting
70
+
71
+ extend Forwardable
72
+ def_delegators :@terminal, :agree, :ask, :choose, :say
73
+
74
+ def password_prompt(prompt = 'Enter password: ')
75
+ ask(prompt) { |q| q.echo = false }
76
+ end
77
+
78
+ #
79
+ # Options
80
+
81
+ def with_options(options)
82
+ begin
83
+ @options_stack.push(options)
84
+ yield if block_given?
85
+ ensure
86
+ @options_stack.pop
87
+ end
88
+ end
89
+
90
+ MERGER = lambda do |k,o,n|
91
+ if o.is_a?(Array)
92
+ n.is_a?(Array) ? (o + n) : (o.dup << n)
93
+ else
94
+ n
95
+ end
96
+ end
97
+
98
+ def merged_options(extra_options = {})
99
+ opts = @options_stack.inject({}) { |m,hsh| m.update(hsh, &MERGER) }
100
+ opts.update(extra_options, &MERGER)
101
+ end
102
+
103
+ #
104
+ # Hosts & Roles
105
+
106
+ { 'role' => '::Makitzo::World::Role',
107
+ 'host' => '::Makitzo::World::Host' }.each do |entity, klass|
108
+ class_eval <<-CODE
109
+ public
110
+ def #{entity}(name, options = {})
111
+ thing = #{klass}.new(@app, name, merged_options(options))
112
+ raise "Duplicate #{entity} name '\#{name}'" if @#{entity}s.include?(thing)
113
+ @#{entity}s << thing
114
+ @#{entity}_index[thing.name.to_s] = thing
115
+ yield thing if block_given?
116
+ thing
117
+ end
118
+
119
+ def #{entity}_for_name(name)
120
+ @#{entity}_index[name.to_s]
121
+ end
122
+
123
+ def #{entity}_for_name!(name)
124
+ #{entity}_for_name(name) or raise "Unknown #{entity} '#{name}'!"
125
+ end
126
+
127
+ def #{entity}s
128
+ @#{entity}s.to_a
129
+ end
130
+
131
+ private
132
+ def initialize_#{entity}s
133
+ @#{entity}s = Set.new
134
+ @#{entity}_index = {}
135
+ end
136
+ CODE
137
+ end
138
+
139
+ def resolve_role(thing)
140
+ if thing.is_a?(::Makitzo::World::Role)
141
+ thing
142
+ else
143
+ role_for_name!(thing.to_s)
144
+ end
145
+ end
146
+
147
+ end
148
+ end
@@ -0,0 +1,24 @@
1
+ module Makitzo
2
+ # classes including this module must define a #root method that returns
3
+ # the path of the makitzo control directory (e.g. /home/foo/makitzo)
4
+ module FileSystem
5
+ INSTALL_FILE = 'INSTALL'
6
+ HISTORY_DIR = 'migrations-history'
7
+
8
+ def self.install_file(root)
9
+ File.join(root, INSTALL_FILE)
10
+ end
11
+
12
+ def self.migration_history_dir(root)
13
+ File.join(root, HISTORY_DIR)
14
+ end
15
+
16
+ def install_file
17
+ ::Makitzo::FileSystem.install_file(root)
18
+ end
19
+
20
+ def migration_history_dir
21
+ ::Makitzo::FileSystem.migration_history_dir(root)
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,16 @@
1
+ module Makitzo; module Logging
2
+ class Blackhole
3
+ def with_host(host, &block); yield; end
4
+ def log_command(status); end
5
+ def overall_success!; end
6
+ def overall_error!; end
7
+ def error(msg); end
8
+ def success(msg); end
9
+ def notice(msg); end
10
+ def warn(msg); end
11
+ def info(msg); end
12
+ def debug(msg); end
13
+ def collector?; false; end
14
+ def result; ""; end
15
+ end
16
+ end; end