sqspoller 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/sqspoller +5 -0
- data/lib/sqspoller/cli.rb +23 -0
- data/lib/sqspoller/config.rb +27 -0
- data/lib/sqspoller/console_logger.rb +16 -0
- data/lib/sqspoller/handler.rb +15 -0
- data/lib/sqspoller/handlers/debug.rb +9 -0
- data/lib/sqspoller/handlers/puppetsdb.rb +80 -0
- data/lib/sqspoller/util.rb +18 -0
- data/lib/sqspoller/version.rb +3 -0
- data/lib/sqspoller.rb +50 -0
- metadata +94 -0
data/bin/sqspoller
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'daemons'
|
3
|
+
require 'sqspoller'
|
4
|
+
|
5
|
+
module SQSPoller
|
6
|
+
class CLI
|
7
|
+
class << self
|
8
|
+
def run
|
9
|
+
Daemons.run_proc('sqspoller',
|
10
|
+
options = {:dir_mode => :system,
|
11
|
+
:backtrace => true,
|
12
|
+
:monitor => true,
|
13
|
+
:log_output => true,
|
14
|
+
}
|
15
|
+
) do
|
16
|
+
sqspoller = SQSPoller.new
|
17
|
+
sqspoller.poll_queues { |queue_name, msg| sqspoller.process(queue_name, msg) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module SQSPoller
|
2
|
+
class Config
|
3
|
+
class << self
|
4
|
+
def load(file_list)
|
5
|
+
@config = {}
|
6
|
+
file_list.each do |cfg_file|
|
7
|
+
cfg_file = File.expand_path(cfg_file)
|
8
|
+
@config.merge!(YAML.load(File.read(cfg_file))) if File.exist?(cfg_file)
|
9
|
+
end
|
10
|
+
@config
|
11
|
+
end
|
12
|
+
|
13
|
+
def load_handlers
|
14
|
+
@config['sqspoller']['queues'].keys.each do |queue|
|
15
|
+
(SQSPoller.warn("Queue '#{queue}' is not configured with a handler");next) if @config['sqspoller']['queues'][queue]['handlers'].nil?
|
16
|
+
handlers = @config['sqspoller']['queues'][queue]['handlers'].keys
|
17
|
+
handlers.each { |handler| require "sqspoller/handlers/#{handler.downcase}" }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def [](key)
|
22
|
+
@config[key]
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module SQSPoller
|
2
|
+
module Console_logger
|
3
|
+
class << self
|
4
|
+
def info(msg)
|
5
|
+
STDOUT.puts("INFO: %s: %s" % [Util.get_timestamp_utc.to_s, msg])
|
6
|
+
end
|
7
|
+
def warn(msg)
|
8
|
+
STDERR.puts("WARN: %s: %s" % [Util.get_timestamp_utc.to_s, msg])
|
9
|
+
end
|
10
|
+
|
11
|
+
def debug(msg)
|
12
|
+
STDERR.puts("DEBUG: %s: %s" % [Util.get_timestamp_utc.to_s, msg])
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module SQSPoller
|
2
|
+
module Handler
|
3
|
+
class << self
|
4
|
+
def process(queue_name, msg)
|
5
|
+
if Config['sqspoller']['queues'][queue_name]['handlers']
|
6
|
+
Config['sqspoller']['queues'][queue_name]['handlers'].keys.each do |handler_name|
|
7
|
+
handler = Handler.const_get(handler_name.capitalize).new
|
8
|
+
handler_config = Config['sqspoller']['queues'][queue_name]['handlers'][handler_name]
|
9
|
+
handler.process(queue_name, msg, handler_config)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# Configuration
|
2
|
+
#
|
3
|
+
#sqspoller:
|
4
|
+
# queues:
|
5
|
+
# mytestqueue:
|
6
|
+
# poll_interval: 15
|
7
|
+
# handlers:
|
8
|
+
# puppetsdb:
|
9
|
+
# domain: the_simpledb_domain
|
10
|
+
# template_map:
|
11
|
+
# auto_scaling_group_name01: reference_node01
|
12
|
+
# auto_scaling_group_name02: reference_node02
|
13
|
+
# myweb_autoscale_group: www_template
|
14
|
+
#
|
15
|
+
# If this autoscaling group has a LAUNCH event, the configuration entry of
|
16
|
+
# 'reference_node' will be used as the template. The reference node's
|
17
|
+
# configuration will be used as the basis for the new instance that is created.
|
18
|
+
#
|
19
|
+
# For instance, in the above configuration, if 'auto_scaling_group_name01' autoscales and adds an instance, puppetsdb will
|
20
|
+
# take the configuration of 'reference_node01' and copy the yaml configuration
|
21
|
+
# for the new instance 'i-abcdef' if the new instance created was 'i-abcdef'.
|
22
|
+
# This means you must use the instance-id as your 'certname' in puppet. This
|
23
|
+
# cannot be avoided for AutoScaling cannot provide OS-level information. It is
|
24
|
+
# entirely possible to make this handler more complex to retrieve the
|
25
|
+
# 'certname' that makes sense for your environment.
|
26
|
+
|
27
|
+
require 'puppetsdb'
|
28
|
+
require 'json'
|
29
|
+
|
30
|
+
module SQSPoller
|
31
|
+
module Handler
|
32
|
+
class Puppetsdb
|
33
|
+
def process(queue_name, msg, conf)
|
34
|
+
@queue_name = queue_name
|
35
|
+
domain = conf['domain']
|
36
|
+
template_map = conf['template_map']
|
37
|
+
begin
|
38
|
+
msg_data = msg.as_sns_message.to_h
|
39
|
+
sns_msg_data = JSON.load(msg_data[:body])
|
40
|
+
rescue
|
41
|
+
SQSPoller.warn("#{@queue_name}: Message is not a properly formatted SNS message")
|
42
|
+
return
|
43
|
+
end
|
44
|
+
msg_id = msg_data.message_id
|
45
|
+
instance_id = sns_msg_data['EC2InstanceId']
|
46
|
+
event_type = sns_msg_data['Event']
|
47
|
+
autoscale_name = sns_msg_data['AutoScalingGroupName']
|
48
|
+
reference_node = template_map[autoscale_name]
|
49
|
+
|
50
|
+
SQSPoller.debug("#{@queue_name}: Found message '#{msg_id}'")
|
51
|
+
SQSPoller.debug("#{@queue_name}: Processing event '#{event_type}' for autoscaling group '#{autoscale_name}' and instance-id #{instance_id}")
|
52
|
+
(SQSPoller.warn("#{@queue_name}: Unregistered autoscaling group: #{autoscale_name}"); return) if reference_node.nil?
|
53
|
+
|
54
|
+
@puppet_enc = PuppetSDB::PuppetSDB.new(:domain => domain)
|
55
|
+
case event_type
|
56
|
+
when "autoscaling:EC2_INSTANCE_LAUNCH"
|
57
|
+
add_node_from_ref(reference_node, instance_id, autoscale_name)
|
58
|
+
when "autoscaling:EC2_INSTANCE_TERMINATE"
|
59
|
+
del_node(instance_id, autoscale_name)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def add_node_from_ref(ref_node, dest_node, as_name)
|
64
|
+
yaml_data = @puppet_enc.get_node_yaml(ref_node)
|
65
|
+
if yaml_data.empty?
|
66
|
+
SQSPoller.warn("#{@queue_name}: Reference node configuration '#{ref_node}' not found for autoscaling group '#{as_name}'")
|
67
|
+
else
|
68
|
+
SQSPoller.info("#{@queue_name}: Writing node configuration for #{dest_node} due to event from autoscaling group #{as_name}")
|
69
|
+
@puppet_enc.write_node_data(dest_node, yaml_data)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def del_node(node, as_name)
|
74
|
+
SQSPoller.info("#{@queue_name}: Deleting node configuration for #{node} due to event from autoscaling group #{as_name}")
|
75
|
+
@puppet_enc.delete_item(node)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module SQSPoller
|
2
|
+
module Util
|
3
|
+
module_function
|
4
|
+
|
5
|
+
def config_files
|
6
|
+
cfg_file_paths = ['/etc/sqspoller/config.yml']
|
7
|
+
if ENV['HOME']
|
8
|
+
cfg_file_paths << '~/.sqspoller/config.yml'
|
9
|
+
end
|
10
|
+
cfg_file_paths
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_timestamp_utc
|
14
|
+
Time.now.utc
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
data/lib/sqspoller.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'aws'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module SQSPoller
|
6
|
+
class SQSPoller
|
7
|
+
autoload :Config, "sqspoller/config"
|
8
|
+
autoload :Util, "sqspoller/util"
|
9
|
+
autoload :Handler, "sqspoller/handler"
|
10
|
+
autoload :Console_logger, "sqspoller/console_logger"
|
11
|
+
|
12
|
+
attr_accessor :sqs, :queue, :conf
|
13
|
+
|
14
|
+
@logger = Console_logger
|
15
|
+
class << self
|
16
|
+
def info(msg); @logger.info(msg); end
|
17
|
+
def warn(msg); @logger.warn(msg); end
|
18
|
+
def debug(msg); @logger.debug(msg); end
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(options={})
|
22
|
+
options[:config] ||= Util.config_files
|
23
|
+
@config = Config.load(options[:config])
|
24
|
+
Config.load_handlers
|
25
|
+
@queues ||= @config['sqspoller']['queues'].keys
|
26
|
+
AWS.config(@config['aws'])
|
27
|
+
@sqs = AWS::SQS.new
|
28
|
+
end
|
29
|
+
|
30
|
+
def poll_queues
|
31
|
+
threads = []
|
32
|
+
|
33
|
+
for queue_short_name in @queues
|
34
|
+
poll_config = Config['sqspoller']['queues'][queue_short_name]
|
35
|
+
SQSPoller.info("Initiated polling of queue '#{queue_short_name}'")
|
36
|
+
queue_url = @sqs.queues.named(queue_short_name)
|
37
|
+
threads << Thread.new(queue_short_name) { |queue_name|
|
38
|
+
queue_url.poll(poll_config) { |msg| yield(queue_name, msg) }
|
39
|
+
}
|
40
|
+
end
|
41
|
+
|
42
|
+
threads.each { |aThread| aThread.join }
|
43
|
+
end
|
44
|
+
|
45
|
+
def process(queue_name, msg)
|
46
|
+
Handler.process(queue_name, msg)
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
metadata
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sqspoller
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
version: 0.0.1
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Brian Wong
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2012-11-23 00:00:00 -05:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: aws-sdk
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
version: "0"
|
30
|
+
type: :runtime
|
31
|
+
version_requirements: *id001
|
32
|
+
- !ruby/object:Gem::Dependency
|
33
|
+
name: daemons
|
34
|
+
prerelease: false
|
35
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
segments:
|
40
|
+
- 0
|
41
|
+
version: "0"
|
42
|
+
type: :runtime
|
43
|
+
version_requirements: *id002
|
44
|
+
description: An Amazon SQS poller that can be configured to perform actions upon receiving SQS messages
|
45
|
+
email: bwong114@gmail.com
|
46
|
+
executables:
|
47
|
+
- sqspoller
|
48
|
+
extensions: []
|
49
|
+
|
50
|
+
extra_rdoc_files: []
|
51
|
+
|
52
|
+
files:
|
53
|
+
- lib/sqspoller.rb
|
54
|
+
- lib/sqspoller/cli.rb
|
55
|
+
- lib/sqspoller/config.rb
|
56
|
+
- lib/sqspoller/console_logger.rb
|
57
|
+
- lib/sqspoller/handler.rb
|
58
|
+
- lib/sqspoller/util.rb
|
59
|
+
- lib/sqspoller/version.rb
|
60
|
+
- lib/sqspoller/handlers/debug.rb
|
61
|
+
- lib/sqspoller/handlers/puppetsdb.rb
|
62
|
+
- bin/sqspoller
|
63
|
+
has_rdoc: true
|
64
|
+
homepage: https://github.com/bwong114/sqspoller
|
65
|
+
licenses: []
|
66
|
+
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
|
70
|
+
require_paths:
|
71
|
+
- lib
|
72
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
segments:
|
77
|
+
- 0
|
78
|
+
version: "0"
|
79
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
segments:
|
84
|
+
- 0
|
85
|
+
version: "0"
|
86
|
+
requirements: []
|
87
|
+
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 1.3.6
|
90
|
+
signing_key:
|
91
|
+
specification_version: 3
|
92
|
+
summary: An Amazon SQS poller with execution handlers
|
93
|
+
test_files: []
|
94
|
+
|