fog-prune 0.1.0

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,9 @@
+# FogPrune
+
+Prune node related data with the help of fog.
+
+Prune data using fog for help.
+
+## Info
+
+* Repository: https://github.com/heavywater/fog-prune

data/bin/fog-prune

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'fog-prune'
+
+FogPrune::Options.new.configure(ARGV)
+FogPrune.new.prune!

data/fog-prune.gemspec

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'fog-prune/version'
+Gem::Specification.new do |s|
+  s.name = 'fog-prune'
+  s.version = FogPrune::VERSION.version
+  s.summary = 'Chef pruning with fog integration'
+  s.author = 'Chris Roberts'
+  s.email = 'chrisroberts.code@gmail.com'
+  s.homepage = 'http://github.com/heavywater/fog-prune'
+  s.description = 'Chef pruner'
+  s.require_path = 'lib'
+  s.bindir = 'bin'
+  s.executables << 'fog-prune'
+  s.add_dependency 'chef'
+  s.add_dependency 'fog', '~> 1.20'
+  s.files = Dir['**/*']
+end

data/lib/fog-prune/config.rb

@@ -0,0 +1,27 @@
+require 'chef/mash'
+require 'mixlib/config'
+
+class FogPrune
+
+  class Config
+    extend Mixlib::Config
+
+    fog Mash.new
+    chef_converge_every 3600
+    prune []
+    nodes nil
+    debug false
+
+    class << self
+      def load_config_file!
+        raise 'No config file path defined!' unless self[:config]
+        raise 'No configuration file found at defined path!' unless File.exists?(self[:config])
+        Mash.new(Chef::JSONCompat.from_json(File.read(self[:config]))).each do |k,v|
+          self.send(k, v)
+        end
+        self
+      end
+    end
+  end
+
+end

data/lib/fog-prune/options.rb

@@ -0,0 +1,122 @@
+require 'mixlib/cli'
+require 'fog-prune/config'
+
+class FogPrune
+
+  class Options
+    include Mixlib::CLI
+
+    option(:config,
+      :short => '-c PATH',
+      :long => '--config-file PATH',
+      :description => 'JSON configuration file'
+    )
+    option(:fog_credentials,
+      :short => '-f KEY=VALUE[,KEY=VALUE,...]',
+      :long => '--fog-credentials KEY=VALUE[,KEY=VALUE,...]',
+      :description => 'Fog credentials',
+      :proc => lambda {|val|
+        pairs = val.split(',').map{|v| v.split('=').map(&:strip) }
+        provider = pairs.detect do |ary|
+          ary.first == 'provider'
+        end.last
+        raise 'Credentials must include `provider`' unless provider
+        FogPrune::Config[:fog][provider] ||= []
+        FogPrune::Config[:fog][provider].push(Mash[*pairs.flatten]).uniq!
+      }
+    )
+    option(:prune,
+      :short => '-a sensu,chef',
+      :long => '--apply-to sensu,chef',
+      :description => 'Apply pruning to these items',
+      :proc => lambda {|v| v.split(',')}
+    )
+    option(:chef_converge_every,
+      :short => '-r SECS',
+      :long => '--converge-recur SECS',
+      :description => 'Number of seconds between convergences',
+      :default => 3600
+    )
+    option(:sensu_host,
+      :short => '-h HOST',
+      :long => '--sensu-host HOST',
+      :description => 'Hostname of sensu API'
+    )
+    option(:sensu_port,
+      :short => '-p PORT',
+      :long => '--sensu-port PORT',
+      :description => 'Port of sensu API',
+      :default => 4567,
+      :proc => lambda {|v| v.to_i }
+    )
+    option(:sensu_username,
+      :short => '-u USERNAME',
+      :long => '--sensu-username USERNAME',
+      :description => 'Sensu API username'
+    )
+    option(:sensu_password,
+      :short => '-P PASSWORD',
+      :long => '--sensu-password PASSWORD',
+      :description => 'Sensu API password'
+    )
+    option(:nodes,
+      :short => '-n NODE[,NODE...]',
+      :long => '--nodes NODE[,NODE...]',
+      :description => 'List of nodes to prune',
+      :proc => lambda {|val|
+        val.split(',').map(&:strip)
+      }
+    )
+    option(:filter,
+      :short => '-f "Solr filter"',
+      :long => '--filter "Solr filter"',
+      :description => 'Add filter to node search',
+      :proc => lambda {|val|
+        FogPrune::Config[:filter] = [FogPrune::Config[:filter], val].compact.join(' AND ')
+      }
+    )
+    option(:debug,
+      :short => '-d',
+      :long => '--debug',
+      :description => 'Turn on debug output'
+    )
+    option(:print_only,
+      :short => '-o',
+      :long => '--[no-]print-only',
+      :boolean => true,
+      :default => false,
+      :description => 'Print action and exit'
+    )
+    option(:stale_nodes,
+      :short => '-S',
+      :long => '--[no-]stale-nodes',
+      :boolean => true,
+      :default => false,
+      :description => 'Remove stale nodes'
+    )
+    option(:stale_node_timeout,
+      :long => '--stale-node-timeout TIMEOUT',
+      :default => 3600,
+      :description => 'Delete stale nodes that exceed given timeout'
+    )
+    option(:tag_stale_nodes,
+      :short => '-T',
+      :long => '--[no-]tag-stale-nodes',
+      :default => true,
+      :description => 'Tag nodes with no ohai time set (used for stale node removal)'
+    )
+
+    def configure(args)
+      begin
+        require 'chef/knife'
+        Chef::Knife.new.configure_chef
+      rescue => e
+        $stderr.puts 'WARN: Failed to load defaults via knife configuration.'
+      end
+      parse_options(args)
+      Config.merge!(config)
+      Config[:debug] = true if Config[:print_only]
+      Config.load_config_file! if Config[:config]
+    end
+  end
+end

data/lib/fog-prune/prune.rb

@@ -0,0 +1,189 @@
+require 'fog'
+require 'chef'
+require 'chef/knife'
+require 'chef/knife/core/ui'
+
+require 'fog-prune/config'
+require 'fog-prune/options'
+
+class FogPrune
+
+  QUERY_ROWS = 1000
+  PROVIDER_ALIASES = Mash.new(
+    :ec2 => 'aws'
+  )
+
+  attr_reader :ui
+
+  def initialize
+    Chef::Knife.new.configure_chef
+    @compute = Mash.new
+    @ui = Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
+  end
+
+  def compute(provider)
+    p_key = PROVIDER_ALIASES[provider] || provider
+    unless(@compute[p_key])
+      raise "No Fog credentials provided for #{provider}!" unless Config[:fog][p_key]
+      @compute[p_key] = []
+      @compute[p_key] = Config[:fog][p_key].map do |args|
+        Fog::Compute.new(
+          format_fog_hash(args.merge(:provider => p_key))
+        )
+      end
+    end
+    @compute[p_key]
+  end
+
+  def format_fog_hash(hash)
+    new_hash = {}
+    hash.each do |k,v|
+      new_hash[k.to_sym] = v
+    end
+    new_hash
+  end
+
+  def sensu?
+    Config[:prune].include?('sensu')
+  end
+
+
+  def chef?
+    Config[:prune].include?('chef')
+  end
+
+  def prune!
+    tag_stale_nodes if Config[:tag_stale_nodes]
+    ui.info "Starting node pruning..."
+    ui.warn "Pruning from: #{Config[:prune].join(', ')}"
+    nodes_to_prune = discover_prunable_nodes
+    debug "Initial nodes discovered: #{nodes_to_prune.map(&:name).sort.join(', ')}"
+    debug "Initial node count: #{nodes_to_prune.size}"
+    nodes_to_prune = filter_prunables_via_fog(nodes_to_prune)
+    ui.warn "Nodes to prune: #{nodes_to_prune.size}"
+    debug "#{nodes_to_prune.map(&:name).sort.join(', ')}"
+    unless(Config[:print_only])
+      ui.confirm('Destroy these nodes')
+      nodes_to_prune.each do |node|
+        prune_node(node)
+      end
+    end
+  end
+
+  def discover_prunable_nodes
+    if(Config[:nodes])
+      query = Array(Config[:nodes]).flatten.map do |name|
+        "name:#{name}"
+      end.join(' OR ')
+    else
+      max_ohai_time = Time.now.to_f - Config[:chef_converge_every].to_f
+      query = ["ohai_time:[0.0 TO #{max_ohai_time}]"]
+      if(Config[:stale_nodes])
+        max_prune_time = Time.now.to_f - Config[:stale_node_timeout].to_f
+        query << ["(prune_tag_time:[0.0 TO #{max_prune_time}] NOT ohai_time:[* TO *])"]
+        query = ["(#{query.join(' OR ')})"]
+      end
+    end
+    collect_search('AND ' + query.join(' AND '))
+  end
+
+  def collect_search(query)
+    new_query = ['name:*', Config[:filter]].compact.join(' AND ')
+    new_query << ' ' << query
+    debug "Running search with query: #{new_query.inspect}"
+    result = []
+    idx = 0
+    while((new_items = Chef::Search::Query.new.search(:node, new_query, 'X_CHEF_id_CHEF_X asc', QUERY_ROWS * idx, QUERY_ROWS).first).size == QUERY_ROWS)
+      debug "Fetched #{new_items.size} new items from server"
+      result += new_items
+      idx += 1
+    end
+    debug "Adding last fetched items (#{new_items.size})"
+    result += new_items
+    result
+  end
+
+  def tag_stale_nodes
+    query = '-ohai_time:[* TO *] -prune_tag_time:[* TO *]'
+    nodes_to_tag = collect_search(query)
+    ui.info "Tagging nodes with no ohai_time set. (#{nodes_to_tag.size} nodes)"
+    debug "Nodes to be tagged: #{nodes_to_tag.map(&:name).sort.join(', ')}"
+    unless(Config[:print_only])
+      nodes_to_tag.each do |node|
+        node.set[:prune_tag_time] = Time.now.to_f
+        node.save
+      end
+    else
+      ui.warn 'Skipping node tagging due to print only restriction'
+    end
+  end
+
+  def prune_node(node)
+    prune_sensu(node) if sensu?
+    prune_chef(node) if chef?
+  end
+
+  def prune_sensu(node)
+    debug "Pruning node from sensu server: #{node.name}"
+    url = "http://#{args[:sensu][:host]}:#{args[:sensu][:port]}" <<
+      "/clients/#{node.name}"
+    begin
+      timeout(30) do
+        RestClient::Resource.new(
+          api_url,
+          :user => args[:sensu][:username],
+          :password => args[:sensu][:password]
+        ).delete
+      end
+    rescue => e
+      ui.error "Failed to remove #{node.name} - Unexpected error: #{e}"
+    end
+  end
+
+  def prune_chef(node)
+    debug "Pruning node from chef server: #{node.name}"
+    node.destroy
+    Chef::ApiClient.load(node.name).destroy
+  end
+
+  def filter_prunables_via_fog(nodes_to_prune)
+    nodes_to_prune.map do |node|
+      if(node[:cloud] && node.cloud.provider)
+        if(respond_to?(check_method = "#{node.cloud.provider}_check"))
+          send(check_method, node) ? node : nil
+        end
+      elsif(node[:prune_tag_time] && node[:ohai_time].nil?)
+        node
+      end
+    end.compact
+  end
+
+  ## Checks
+
+  def ec2_check(node)
+    aws_node = ec2_nodes.detect{|n| n.id == node.ec2.instance_id}
+    unless(aws_node)
+      debug "#{node.name} returned nil from aws"
+    else
+      debug "#{node.name} state on aws: #{aws_node.state}"
+    end
+    aws_node.nil? || aws_node.state == 'terminated'
+  end
+
+  def ec2_nodes
+    unless(@ec2_nodes)
+      @ec2_nodes = compute(:aws).map do |compute_con|
+        compute_con.servers.all
+      end.flatten
+    end
+    @ec2_nodes
+  end
+
+  def rackspace_check(node)
+    raise 'Not implemented'
+  end
+
+  def debug(msg)
+    ui.info "#{ui.color('[DEBUG]', :magenta)} #{msg}" if Config[:debug]
+  end
+end

data/lib/fog-prune/version.rb

@@ -0,0 +1,6 @@
+class FogPrune
+  class Version < Gem::Version
+  end
+
+  VERSION = Version.new('0.1.0')
+end

data/lib/fog-prune.rb

@@ -0,0 +1,7 @@
+require 'fog-prune/version'
+require 'fog-prune/prune'
+
+class FogPrune
+  autoload :Config, 'fog-prune/config'
+  autoload :Options, 'fog-prune/options'
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: fog-prune
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.20'
+description: Chef pruner
+email: chrisroberts.code@gmail.com
+executables:
+- fog-prune
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fog-prune.rb
+- lib/fog-prune/version.rb
+- lib/fog-prune/options.rb
+- lib/fog-prune/config.rb
+- lib/fog-prune/prune.rb
+- Gemfile
+- README.md
+- bin/fog-prune
+- fog-prune.gemspec
+- CHANGELOG.md
+homepage: http://github.com/heavywater/fog-prune
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Chef pruning with fog integration
+test_files: []
+has_rdoc: