mofa 0.0.1

data/lib/mofa/cookbook.rb

@@ -0,0 +1,87 @@
+require 'pathname'
+require 'thor/base'
+require 'thor/actions'
+
+class Cookbook
+  include Thor::Base
+  include Thor::Actions
+  include FileUtils
+
+  attr_accessor :name
+  attr_accessor :version
+  attr_accessor :type
+  attr_accessor :pkg_uri
+  attr_accessor :source_uri
+  attr_accessor :cookbooks_url
+  attr_accessor :token
+
+  def self.create(cookbook_name_or_path='.', token=nil)
+    cb = nil
+    begin
+      case
+        when cookbook_name_or_path.match(/:/)
+          fail "Did not find released Cookbook #{cookbook_name_or_path}!" unless ReleasedCookbook.exists?(cookbook_name_or_path)
+          fail "Did not find Version #{cookbook_version} of released Cookbook #{cookbook_name_or_path}!" unless ReleasedCookbook.exists?(cookbook_name_or_path, cookbook_version)
+
+          cb = ReleasedCookbook.new(cookbook_name_or_path)
+
+        else
+          cb = SourceCookbook.new(cookbook_name_or_path)
+      end
+    rescue RuntimeError => e
+      error e.message
+      raise "Cookbook not found/detected!"
+    end
+    cb.token = token
+    cb.autodetect_type
+
+    cb
+
+  end
+
+  def autodetect_type
+    env_indicator = Mofa::Config.config['cookbook_type_indicator']['env']
+    wrapper_indicator = Mofa::Config.config['cookbook_type_indicator']['wrapper']
+    base_indicator = Mofa::Config.config['cookbook_type_indicator']['base']
+
+    say "Autodetecting Cookbook Architectural Type... "
+    case
+      when @name.match(env_indicator)
+        @type = 'env'
+      when @name.match(wrapper_indicator)
+        @type = 'wrapper'
+      when @name.match(base_indicator)
+        @type = 'base'
+      else
+        @type = 'application'
+    end
+    say "#{type.capitalize} Cookbook"
+  end
+
+  def say(message = "", color = nil, force_new_line = (message.to_s !~ /( |\t)$/))
+    color ||= :green
+    super
+  end
+
+  def ok(detail=nil)
+    text = detail ? "OK, #{detail}." : "OK."
+    say text, :green
+  end
+
+  def error(detail)
+    say detail, :red
+  end
+
+  # Enforce silent system calls, unless the --verbose option is passed.
+  # One may either pass -v, --verbose or --[v|verbose]=[true|t|yes|y|1].
+  #
+  def run(cmd, *args)
+    args = args.empty? ? {} : args.pop
+    verbose = (Mofa::CLI::option_debug) ? true : false
+    #verbose = !!(options[:verbose] && options[:verbose].to_s.match(/(verbose|true|t|yes|y|1)$/i))
+    exit_code = super(cmd, args.merge(:verbose => verbose))
+    fail "Failed to run #{cmd.inspect}!" unless exit_code == true
+  end
+
+
+end

data/lib/mofa/hostlist.rb

@@ -0,0 +1,61 @@
+require 'rest-client'
+require 'net/ping'
+
+class Hostlist
+  attr_accessor :list
+  attr_accessor :filter
+  attr_accessor :service_host
+  attr_accessor :service_url
+  attr_accessor :filter
+  attr_accessor :api_key
+
+  def self.create(filter = nil)
+    hl = Hostlist.new
+    filter ||= Mofa::Config.config['service_hostlist_default_filter']
+    hl.filter = filter
+    hl.service_host = Mofa::Config.config['service_hostlist_url'].gsub(/^http:\/\//, '').gsub(/\/.*$/, '').gsub(/:.*$/, '')
+    hl.service_url = Mofa::Config.config['service_hostlist_url']
+    hl.api_key = Mofa::Config.config['service_hostlist_api_key']
+    hl
+  end
+
+  def retrieve
+    fail "Hostlist Service not reachable! (cannot ping #{service_host})" unless up?
+    response = RestClient.get(@service_url, { :params => {:key => api_key}})
+    hosts_list_json = JSON.parse response.body
+    @list = hosts_list_json['data'].collect { |i| i['cname'] }
+
+    apply_filter
+    sort_by_domainname
+  end
+
+
+  def up?
+    p = Net::Ping::TCP.new(@service_host, 'http')
+    p.ping?
+  end
+
+  def apply_filter
+    # building matcher
+    regex = @filter.gsub(/\*/, '__ASTERISK__')
+    regex = Regexp.escape(regex).gsub(/__ASTERISK__/, '.*')
+    regex = '^' + regex + '$'
+
+    puts "regex=#{regex}"
+
+    @list.select! {|hostname| hostname.match(regex) }
+  end
+
+  def sort_by_domainname
+    sortable = {}
+    @list.each do |hostname|
+      sortable.store(hostname.split(/\./).reverse.join('.'), hostname)
+    end
+    @list = sortable.keys.sort.collect { |s| sortable[s] }
+  end
+
+  def filter_by_runlist_map(runlist_map)
+    @list.select! { |hostname| runlist_map.mp.key?(hostname)}
+  end
+
+end

data/lib/mofa/mofa_cmd.rb

@@ -0,0 +1,246 @@
+require 'net/ssh'
+require 'net/sftp'
+
+class MofaCmd
+  attr_accessor :token
+  attr_accessor :cookbook
+  attr_accessor :hostlist
+  attr_accessor :runlist_map
+
+  def self.generate_token
+    Digest::SHA1.hexdigest([Time.now, rand].join)[0..10]
+  end
+
+  def self.create(cookbook, hostlist, runlist_map, token)
+    mofa_cmd = MofaCmd.new
+    mofa_cmd.token = token
+    mofa_cmd.cookbook = cookbook
+    mofa_cmd.hostlist = hostlist
+    mofa_cmd.runlist_map = runlist_map
+    mofa_cmd
+  end
+
+  def prepare
+    cookbook.prepare
+    fail "Hostlist Service not reachable! (cannot ping #{hostlist.service_url})" unless hostlist.up?
+  end
+
+  def execute
+    cookbook.execute
+    hostlist.retrieve
+    runlist_map.generate
+
+    puts "Runlist Map: #{runlist_map.mp.inspect}"
+    puts "Hostlist before runlist filtering: #{hostlist.list.inspect}"
+
+    hostlist.filter_by_runlist_map(runlist_map)
+
+    puts "Hostlist after runlist filtering: #{hostlist.list.inspect}"
+
+    exit_code = run_chef_solo_on_hosts
+
+    exit_code
+  end
+
+  def cleanup
+    cookbook.cleanup
+  end
+
+  # FIXME
+  # This Code is Copy'n'Pasted from the old mofa tooling. Only to make the MVP work in time!!
+  # This needs to be refactored ASAP.
+
+  def run_chef_solo_on_hosts
+    time = Time.new
+    puts 'Chef-Solo Run started at ' + time.strftime('%Y-%m-%d %H:%M:%S')
+    puts "Will use ssh_user #{Mofa::Config.config['ssh_user']} and ssh_key_file #{Mofa::Config.config['ssh_keyfile']}"
+    at_least_one_chef_solo_run_failed = false
+    chef_solo_runs = {}
+    host_index = 0
+    hostlist.list.each do |hostname|
+      host_index = host_index + 1
+      puts
+      puts "----------------------------------------------------------------------"
+      puts "Chef-Solo on Host #{hostname} (#{host_index}/#{hostlist.list.length.to_s})"
+      puts "----------------------------------------------------------------------"
+      chef_solo_runs.store(hostname, {})
+
+      # do only one for faster dev-cycle...
+      #next unless hostname.match(/^dash/)
+
+      puts "Pinging host #{hostname}..."
+      exit_status = system("ping -q -c 1 #{hostname} >/dev/null 2>&1")
+      unless exit_status then
+        puts "  --> Host #{hostname} is unavailable!"
+        chef_solo_runs[hostname].store('status', 'UNAVAIL')
+        chef_solo_runs[hostname].store('status_msg', "Host #{hostname} unreachable.")
+      else
+        puts "  --> Host #{hostname} is available."
+        prerequesits_met = true
+        # Create a temp working dir on the target host
+        solo_dir = '/var/tmp/' + time.strftime('%Y-%m-%d_%H%M%S')
+        Net::SSH.start(hostname, Mofa::Config.config['ssh_user'], :keys => [Mofa::Config.config['ssh_keyfile']], :verbose => :error) do |ssh|
+          puts "Remotely creating solo_dir \"#{solo_dir}\" on host #{hostname}"
+          # remotely create the temp folder
+          out = ssh_exec!(ssh, "[ -d #{solo_dir} ] || mkdir #{solo_dir}")
+          puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+
+          # remotely create a data_bags folder structure on the target host
+          if File.directory?("#{cookbook.source_dir}/data_bags")
+            Dir.entries("#{cookbook.source_dir}/data_bags").select{|f| !f.match(/^\.\.?$/)}.each do |data_bag|
+              puts "Remotely creating data_bags dir \"#{solo_dir}/data_bags/#{data_bag}\""
+              out = ssh_exec!(ssh, "[ -d #{solo_dir}/data_bags/#{data_bag} ] || mkdir -p #{solo_dir}/data_bags/#{data_bag}")
+              puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+            end
+          end
+        end
+      end
+
+      # skip the rest if prerequesits are not met
+      next unless prerequesits_met
+
+
+      Net::SFTP.start(hostname, Mofa::Config.config['ssh_user'], :keys => [Mofa::Config.config['ssh_keyfile']], :verbose => :error) do |sftp|
+
+        # remotely creating solo.rb
+        puts "Remotely creating \"#{solo_dir}/solo.rb\""
+        sftp.file.open("#{solo_dir}/solo.rb", "w") do |file|
+          solo_rb = <<-"EOF"
+        cookbook_path [ "#{solo_dir}/cookbooks" ]
+        data_bag_path "#{solo_dir}/data_bags"
+        log_level :info
+        log_location "#{solo_dir}/log"
+        verify_api_cert true
+          EOF
+
+          file.write(solo_rb)
+        end
+
+        # remotely creating node.json
+        puts "Remotely creating \"#{solo_dir}/node.json\""
+        node_json = {}
+        node_json.store('run_list', runlist_map.mp[hostname])
+
+        sftp.file.open("#{solo_dir}/node.json", "w") do |file|
+          file.write(JSON.pretty_generate(node_json))
+        end
+
+        # remotely create data_bag items
+        if File.directory?("#{cookbook.source_dir}/data_bags")
+          Dir.entries("#{cookbook.source_dir}/data_bags").select{|f| !f.match(/^\.\.?$/)}.each do |data_bag|
+            Dir.entries("#{cookbook.source_dir}/data_bags/#{data_bag}").select{|f| f.match(/\.json$/)}.each do |data_bag_item|
+              puts "Uploading data_bag_item #{data_bag_item}... "
+              sftp.upload!("#{cookbook.source_dir}/data_bags/#{data_bag}/#{data_bag_item}", "#{solo_dir}/data_bags/#{data_bag}/#{data_bag_item}")
+              puts "OK."
+            end
+          end
+        end
+
+        if cookbook.instance_of?(SourceCookbook)
+          puts "Cookbook is a SourceCookbook! Uploading Snapshot Package #{cookbook.pkg_name}... "
+          sftp.upload!("#{cookbook.pkg_dir}/#{cookbook.pkg_name}", "#{solo_dir}/#{cookbook.pkg_name}")
+          puts "OK."
+        end
+
+        # Do it -> Execute the chef-solo run!
+        Net::SSH.start(hostname, Mofa::Config::config['ssh_user'], :keys => [Mofa::Config::config['ssh_keyfile']], :verbose => :error) do |ssh|
+
+          if cookbook.instance_of?(SourceCookbook)
+            puts "Remotely unpacking Snapshot Package #{cookbook.pkg_name}... "
+            out = ssh_exec!(ssh, "cd #{solo_dir}; tar xvfz #{cookbook.pkg_name}")
+            if out[0] != 0
+              puts "ERROR (#{out[0]}): #{out[2]}"
+              puts out[1]
+            else
+              puts "OK."
+            end
+          end
+
+          puts "Remotely running chef-solo -c #{solo_dir}/solo.rb -j #{solo_dir}/node.json"
+          out = ssh_exec!(ssh, "sudo chef-solo -c #{solo_dir}/solo.rb -j #{solo_dir}/node.json")
+          if out[0] != 0
+            puts "ERROR (#{out[0]}): #{out[2]}"
+            out = ssh_exec!(ssh, "sudo cat #{solo_dir}/log")
+            puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+            puts out[1]
+            chef_solo_runs[hostname].store('status', 'FAIL')
+            chef_solo_runs[hostname].store('status_msg', out[1])
+          else
+            unless Mofa::CLI::option_debug
+              out = ssh_exec!(ssh, "sudo grep 'Chef Run' #{solo_dir}/log")
+              puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+              puts "Done."
+            else
+              out = ssh_exec!(ssh, "sudo cat #{solo_dir}/log")
+              puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+              puts out[1]
+            end
+            chef_solo_runs[hostname].store('status', 'SUCCESS')
+            chef_solo_runs[hostname].store('status_msg', '')
+          end
+          out = ssh_exec!(ssh, "sudo chown -R #{Mofa::Config.config['ssh_user']}.#{Mofa::Config.config['ssh_user']} #{solo_dir}")
+          puts "ERROR (#{out[0]}): #{out[2]}" if out[0] != 0
+        end
+      end
+      at_least_one_chef_solo_run_failed = true if chef_solo_runs[hostname]['status'] == 'FAIL'
+    end
+
+    # ------- print out report
+    puts
+    puts "----------------------------------------------------------------------"
+    puts "Chef-Solo Run REPORT"
+    puts "----------------------------------------------------------------------"
+    puts "Chef-Solo has been run on #{chef_solo_runs.keys.length.to_s} hosts."
+
+    chef_solo_runs.each do |hostname, content|
+      status_msg = ''
+      status_msg = "(#{content['status_msg']})" if content['status'] == 'FAIL'
+      puts "#{content['status']}: #{hostname} #{status_msg}"
+    end
+
+    exit_code = 0
+    if at_least_one_chef_solo_run_failed
+      exit_code = 1
+    end
+
+    puts "Exiting with exit code #{exit_code}."
+    exit_code
+
+  end
+
+  def ssh_exec!(ssh, command)
+    stdout_data = ""
+    stderr_data = ""
+    exit_code = nil
+    exit_signal = nil
+    ssh.open_channel do |channel|
+      channel.exec(command) do |ch, success|
+        unless success
+          abort "FAILED: couldn't execute command (ssh.channel.exec)"
+        end
+        channel.on_data do |ch, data|
+          stdout_data+=data
+        end
+
+        channel.on_extended_data do |ch, type, data|
+          stderr_data+=data
+        end
+
+        channel.on_request("exit-status") do |ch, data|
+          exit_code = data.read_long
+        end
+
+        channel.on_request("exit-signal") do |ch, data|
+          exit_signal = data.read_long
+        end
+      end
+    end
+    ssh.loop
+    [exit_code, stdout_data, stderr_data, exit_signal]
+  end
+
+
+  private
+
+end
+

data/lib/mofa/released_cookbook.rb

@@ -0,0 +1,47 @@
+class ReleasedCookbook < Cookbook
+  attr_accessor :pkg_dir
+  attr_accessor :pkg_name
+
+  def initialize(cookbook_name_or_path)
+    super()
+    # TODO: this needs proper vaidation!
+    @name = cookbook_name_or_path.split(/:/).first
+    @version = cookbook_name_or_path.split(/:/).last
+  end
+
+  # ------------- Interface Methods
+
+  def prepare
+    @pkg_name = "#{name}-#{version}.tar.gz"
+    @pkg_dir = "#{Mofa::Config.config['tmp_dir']}/.mofa/#{token}"
+  end
+
+  def execute
+    # TODO: Download & unpack released cookbook
+    # Important for guessing role runlists (when cookbook is an env-cookbook)
+  end
+
+  def cleanup
+    say "Removing folder #{pkg_dir}...#{nl}"
+    run "rm -r #{pkg_dir}"
+    ok
+  end
+
+  # ------------- /Interface Methods
+
+  def cleanup!
+    unless (Dir.entries("#{Mofa::Config.config['tmp_dir']}/.mofa") - %w{ . .. }).empty?
+      say "Removing content of folder #{Mofa::Config.config['tmp_dir']}/.mofa"
+      run "rm -r #{Mofa::Config.config['tmp_dir']}/.mofa/*"
+    else
+      say "Folder #{Mofa::Config.config['tmp_dir']}/.mofa is (already) clean."
+    end
+  end
+
+  private
+
+  def nl
+    return (Mofa::CLI::option_verbose) ? '' : ' '
+  end
+
+end

data/lib/mofa/runlist_map.rb

@@ -0,0 +1,54 @@
+class RunlistMap
+  attr_accessor :mp
+  attr_accessor :cookbook
+  attr_accessor :hostlist
+  attr_accessor :token
+  attr_accessor :option_runlist
+  attr_accessor :default_runlist
+
+  def self.create(cookbook, hostlist, token, option_runlist = nil)
+    rl = RunlistMap.new
+    rl.cookbook = cookbook
+    rl.hostlist = hostlist
+    rl.token = token
+    rl.default_runlist = (!option_runlist.nil?) ? option_runlist : nil
+    rl
+  end
+
+  def initialize
+    @mp = {}
+  end
+
+  def generate
+    @default_runlist ||= "recipe[#{cookbook.name}::default]"
+
+    case cookbook.type
+      when 'env'
+        guess_runlists_by_hostnames
+      else
+        set_default_runlist_for_every_host
+    end
+  end
+
+  def guess_runlists_by_hostnames
+    # recipes/jkmaster.rb --> runlist[<env_cookbook_name>::jkmaster] for all hosts with shortname jkmaster
+    # recipes/jkslave.rb --> runlist[<env_cookbook_name>::jkslave] for all hosts with shortname jkslave[0-9]
+    # and so on
+    hostlist.list.each do |hostname|
+      cookbook.recipes.each do |recipe|
+        recipe_regex = "^#{recipe}[0-9]*\."
+        if hostname.match(recipe_regex)
+          @mp.store(hostname, "recipe[#{cookbook.name}::#{recipe}]")
+        end
+      end
+    end
+
+  end
+
+  def set_default_runlist_for_every_host
+    hostlist.list.each do |hostname|
+      @mp.store(hostname, @default_runlist)
+    end
+  end
+
+end