qurd 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.ruby-version +1 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +22 -0
- data/README.md +367 -0
- data/Rakefile +9 -0
- data/bin/qurd +10 -0
- data/lib/hash.rb +6 -0
- data/lib/qurd.rb +49 -0
- data/lib/qurd/action.rb +92 -0
- data/lib/qurd/action/chef.rb +128 -0
- data/lib/qurd/action/dummy.rb +27 -0
- data/lib/qurd/action/route53.rb +168 -0
- data/lib/qurd/configuration.rb +182 -0
- data/lib/qurd/listener.rb +207 -0
- data/lib/qurd/message.rb +231 -0
- data/lib/qurd/mixins.rb +8 -0
- data/lib/qurd/mixins/aws_clients.rb +37 -0
- data/lib/qurd/mixins/configuration.rb +29 -0
- data/lib/qurd/mixins/configuration_helpers.rb +79 -0
- data/lib/qurd/processor.rb +97 -0
- data/lib/qurd/version.rb +5 -0
- data/lib/string.rb +12 -0
- data/qurd.gemspec +32 -0
- data/test/action_test.rb +115 -0
- data/test/chef_test.rb +206 -0
- data/test/configuration_test.rb +333 -0
- data/test/dummy_action_test.rb +51 -0
- data/test/inputs/foo.pem +27 -0
- data/test/inputs/knife.rb +9 -0
- data/test/inputs/qurd.yml +32 -0
- data/test/inputs/qurd_chef.yml +35 -0
- data/test/inputs/qurd_chef_route53.yml +43 -0
- data/test/inputs/qurd_route53.yml +39 -0
- data/test/inputs/qurd_route53_wrong.yml +37 -0
- data/test/inputs/validator.pem +27 -0
- data/test/listener_test.rb +135 -0
- data/test/message_test.rb +187 -0
- data/test/mixin_aws_clients_test.rb +28 -0
- data/test/mixin_configuration_test.rb +36 -0
- data/test/processor_test.rb +41 -0
- data/test/responses/aws/ec2-describe-instances-0.xml +2 -0
- data/test/responses/aws/ec2-describe-instances-1.xml +127 -0
- data/test/responses/aws/error-response.xml +1 -0
- data/test/responses/aws/route53-change-resource-record-sets.xml +2 -0
- data/test/responses/aws/route53-list-hosted-zones-by-name-0.xml +3 -0
- data/test/responses/aws/route53-list-hosted-zones-by-name-1.xml +4 -0
- data/test/responses/aws/route53-list-hosted-zones-by-name-n.xml +5 -0
- data/test/responses/aws/route53-list-resource-record-sets-0.xml +2 -0
- data/test/responses/aws/route53-list-resource-record-sets-1.xml +4 -0
- data/test/responses/aws/route53-list-resource-record-sets-n.xml +6 -0
- data/test/responses/aws/sqs-list-queues-0.xml +1 -0
- data/test/responses/aws/sqs-list-queues-n.xml +4 -0
- data/test/responses/aws/sqs-receive-message-1-launch.xml +6 -0
- data/test/responses/aws/sqs-receive-message-1-launch_error.xml +6 -0
- data/test/responses/aws/sqs-receive-message-1-other.xml +12 -0
- data/test/responses/aws/sqs-receive-message-1-terminate.xml +6 -0
- data/test/responses/aws/sqs-receive-message-1-terminate_error.xml +6 -0
- data/test/responses/aws/sqs-receive-message-1-test.xml +12 -0
- data/test/responses/aws/sqs-set-queue-attributes.xml +1 -0
- data/test/responses/aws/sts-assume-role.xml +17 -0
- data/test/responses/chef/search-client-name-0.json +6 -0
- data/test/responses/chef/search-client-name-1.json +7 -0
- data/test/responses/chef/search-client-name-n.json +8 -0
- data/test/responses/chef/search-node-instance-0.json +5 -0
- data/test/responses/chef/search-node-instance-1.json +784 -0
- data/test/responses/chef/search-node-instance-n.json +1565 -0
- data/test/responses/ec2/latest-meta-data-iam-security-credentials-client.txt +9 -0
- data/test/responses/ec2/latest-meta-data-iam-security-credentials.txt +1 -0
- data/test/route53_test.rb +231 -0
- data/test/support/web_mock_stubs.rb +109 -0
- data/test/test_helper.rb +10 -0
- metadata +307 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: cb531c8fce5f427730dc00640ab95c9efc9766bf
|
4
|
+
data.tar.gz: 7eaee5c8c805f610cd2e0962cb7785347f33b476
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 20500894f93834ad793f274ca2865183c6313547fd700bfea25c96852741f562e56d1b289ba655f79f0be95153d9aaf989f21b73150f55213ca235e82da1383b
|
7
|
+
data.tar.gz: cd3332095e49af044260a9aa026081aad942f93c1b8d2c46bbd565a5e311bc04374c9786485bcac043cb5f1dddea70e16b12bed82181353d1d53fb55a86e2ac3
|
data/.gitignore
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.1.2
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Philip Champon
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,367 @@
|
|
1
|
+
# QURD - QUeue Resource Daemon
|
2
|
+
|
3
|
+
The Queue Resource Daemon is an extensible SQS monitoring service, which can be
|
4
|
+
configured to react to or ignore AutoScaling messages. Qurd can be configured to
|
5
|
+
monitor multiple accounts, any number of queues, and any type of auto scaling
|
6
|
+
event.
|
7
|
+
|
8
|
+
When the daemon starts up, it finds the queues it's meant to monitor and sets
|
9
|
+
the `visibility_timeout` and `waittimeseconds` for each queue. Qurd uses long
|
10
|
+
polling to monitor the queues, by default.
|
11
|
+
|
12
|
+
This daemon makes extensive use of threads, you should really consider running
|
13
|
+
this with Ruby version 2.0 or higher.
|
14
|
+
|
15
|
+
## Plugin architecture
|
16
|
+
|
17
|
+
It is possible to provide your own actions, aside from Chef, Route53, and Dummy.
|
18
|
+
Dummy is provided as a simple example, but, in a nutshell, inherit from
|
19
|
+
Qurd::Action and override the actions you respond to.
|
20
|
+
|
21
|
+
Your action class can configure itself, by overriding the class method
|
22
|
+
`configure`. Instances must override the action methods launch, launch_error,
|
23
|
+
terminate, terminate_error, and test. Action instances have two attributes,
|
24
|
+
`message` and `context`. Message is a `Qurd::Message` instance. Context is a
|
25
|
+
`Cabin::Context`, used for logging. Callbacks, to interact with the action
|
26
|
+
before and after the instance are executed, can be overridden.
|
27
|
+
|
28
|
+
The mixins for AwsClients and Configuration are also available at the class and
|
29
|
+
instance level.
|
30
|
+
|
31
|
+
```ruby
|
32
|
+
# This contrived example creates a file in s3 when an instance launches
|
33
|
+
# It can be triggered by adding the class name to the list of actions in the
|
34
|
+
# configuration file, ie
|
35
|
+
# bucket: example-bucket
|
36
|
+
# actions:
|
37
|
+
# launch:
|
38
|
+
# - "Foo"
|
39
|
+
class Foo < Qurd::Action
|
40
|
+
def self.configure(_action)
|
41
|
+
qurd_configuration.bucket || qurd_logger!("Missing bucket")
|
42
|
+
end
|
43
|
+
|
44
|
+
def run_before
|
45
|
+
aws_retryable do
|
46
|
+
aws_client(:S3).delete_object(
|
47
|
+
bucket: qurd_configuration.bucket,
|
48
|
+
key: message.instance_id
|
49
|
+
)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def launch
|
54
|
+
aws_retryable do
|
55
|
+
aws_client(:S3).put_object(
|
56
|
+
bucket: qurd_configuration.bucket,
|
57
|
+
key: message.instance_id,
|
58
|
+
body: message.instance.private_ip_address
|
59
|
+
)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def run_after
|
64
|
+
aws_retryable do
|
65
|
+
o = aws_client(:S3).get_object(
|
66
|
+
bucket: qurd_configuration.bucket,
|
67
|
+
key: message.instance_id
|
68
|
+
)
|
69
|
+
qurd_logger.debug("Found #{o.body}")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
## AWS IAM Policy
|
77
|
+
|
78
|
+
QURD requires, at a minimum, SQS privileges and EC2 privileges, ie
|
79
|
+
|
80
|
+
```json
|
81
|
+
{
|
82
|
+
"Version": "2012-10-17",
|
83
|
+
"Statement": [
|
84
|
+
{
|
85
|
+
"Sid": "qurd_sqs",
|
86
|
+
"Effect": "Allow",
|
87
|
+
"Action": [
|
88
|
+
"sqs:DeleteMessage",
|
89
|
+
"sqs:ListQueues",
|
90
|
+
"sqs:ReceiveMessage",
|
91
|
+
"sqs:SetQueueAttributes"
|
92
|
+
],
|
93
|
+
"Resource": [
|
94
|
+
"*"
|
95
|
+
]
|
96
|
+
},
|
97
|
+
{
|
98
|
+
"Sid": "qurd_ec2",
|
99
|
+
"Effect": "Allow",
|
100
|
+
"Action": [
|
101
|
+
"ec2:DescribeInstances"
|
102
|
+
],
|
103
|
+
"Resource": [
|
104
|
+
"*"
|
105
|
+
]
|
106
|
+
}
|
107
|
+
]
|
108
|
+
}
|
109
|
+
```
|
110
|
+
|
111
|
+
If you are using the route53 action, you will also need
|
112
|
+
|
113
|
+
```json
|
114
|
+
{
|
115
|
+
"Version": "2012-10-17",
|
116
|
+
"Statement": [
|
117
|
+
{
|
118
|
+
"Sid": "Stmt1428424119000",
|
119
|
+
"Effect": "Allow",
|
120
|
+
"Action": [
|
121
|
+
"route53:ChangeResourceRecordSets",
|
122
|
+
"route53:DeleteHostedZone",
|
123
|
+
"route53:GetHostedZone",
|
124
|
+
"route53:ListHostedZones",
|
125
|
+
"route53:ListResourceRecordSets"
|
126
|
+
],
|
127
|
+
"Resource": [
|
128
|
+
"*"
|
129
|
+
]
|
130
|
+
}
|
131
|
+
]
|
132
|
+
}
|
133
|
+
```
|
134
|
+
|
135
|
+
## Configuration
|
136
|
+
|
137
|
+
To configure the daemon, edit the YAML configuration file. The default path is
|
138
|
+
`/etc/qurd/config.yml`. An alternate path can be specified on the command line.
|
139
|
+
|
140
|
+
<table>
|
141
|
+
<tr>
|
142
|
+
<th>Option</th>
|
143
|
+
<th>Description</th>
|
144
|
+
<th>Type</th>
|
145
|
+
<th>Default</th>
|
146
|
+
</tr>
|
147
|
+
|
148
|
+
<tr>
|
149
|
+
<td><tt><a href="#aws_credentials">aws_credentials</a></tt></td>
|
150
|
+
<td>AWS credentials</td>
|
151
|
+
<td><tt>[Array<Hash>]</tt></td>
|
152
|
+
<td> </td>
|
153
|
+
</tr>
|
154
|
+
|
155
|
+
<tr>
|
156
|
+
<td><tt><a href="#auto_scaling_queues">auto_scaling_queues</a></tt></td>
|
157
|
+
<td>Describe the queues to be monitored</td>
|
158
|
+
<td><tt>[Hash<Hash>]</tt></td>
|
159
|
+
<td> </td>
|
160
|
+
</tr>
|
161
|
+
|
162
|
+
<tr>
|
163
|
+
<td><tt><a href="#actions">actions</a></tt></td>
|
164
|
+
<td>describe actions to take, for any autoscaling event</td>
|
165
|
+
<td><tt>[Hash<Array>]</tt></td>
|
166
|
+
<td> </td>
|
167
|
+
</tr>
|
168
|
+
|
169
|
+
<tr>
|
170
|
+
<td><tt>daemonize</tt></td>
|
171
|
+
<td>Force qurd to log to a file, even if <tt>log_file</tt> is not defined.</td>
|
172
|
+
<td><tt>Boolean</tt></td>
|
173
|
+
<td><tt>false</tt></td>
|
174
|
+
</tr>
|
175
|
+
|
176
|
+
<tr>
|
177
|
+
<td><tt>dry_run</tt></td>
|
178
|
+
<td>Log what qurd would have done</td>
|
179
|
+
<td><tt>Boolean</tt></td>
|
180
|
+
<td><tt>false</tt></td>
|
181
|
+
</tr>
|
182
|
+
|
183
|
+
<tr>
|
184
|
+
<td><tt>listen_timeout</tt></td>
|
185
|
+
<td>Defines the timeout, in seconds, for a thread to process a message</td>
|
186
|
+
<td><tt>Float</tt></td>
|
187
|
+
<td><tt>visibility_timeout</tt></td>
|
188
|
+
</tr>
|
189
|
+
|
190
|
+
<tr>
|
191
|
+
<td><tt>log_file</tt></td>
|
192
|
+
<td>The path to qurd's log file</td>
|
193
|
+
<td><tt>String</tt></td>
|
194
|
+
<td><tt>/var/log/qurd/qurd.log</tt></td>
|
195
|
+
</tr>
|
196
|
+
|
197
|
+
<tr>
|
198
|
+
<td><tt>log_level</tt></td>
|
199
|
+
<td>The log level to catch</td>
|
200
|
+
<td><tt>String</tt></td>
|
201
|
+
<td><tt>info</tt></td>
|
202
|
+
</tr>
|
203
|
+
|
204
|
+
<tr>
|
205
|
+
<td><tt>pid_file</tt></td>
|
206
|
+
<td>The path of qurd's pid file</td>
|
207
|
+
<td><tt>String</tt></td>
|
208
|
+
<td><tt>/var/run/qurd/qurd.pid</tt></td>
|
209
|
+
</tr>
|
210
|
+
|
211
|
+
<tr>
|
212
|
+
<td><tt>save_failures</tt></td>
|
213
|
+
<td>Save messages if any action fails</td>
|
214
|
+
<td><tt>Boolean</tt></td>
|
215
|
+
<td><tt>true</tt></td>
|
216
|
+
</tr>
|
217
|
+
|
218
|
+
<tr>
|
219
|
+
<td><tt>sqs_set_attributes_timeout</tt></td>
|
220
|
+
<td>Defines the timeout, in seconds, for a thread setting SQS attributes</td>
|
221
|
+
<td><tt>Float</tt></td>
|
222
|
+
<td><tt>10</tt></td>
|
223
|
+
</tr>
|
224
|
+
|
225
|
+
<tr>
|
226
|
+
<td><tt>visibility_timeout</tt></td>
|
227
|
+
<td>Set the SQS <a href="http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/AboutVT.html">visibility timeout</a></td>
|
228
|
+
<td><tt>Integer</tt></td>
|
229
|
+
<td><tt>300</tt></td>
|
230
|
+
</tr>
|
231
|
+
|
232
|
+
<tr>
|
233
|
+
<td><tt>wait_time</tt></td>
|
234
|
+
<td>Set the SQS <a href="http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html">wait time seconds</a></td>
|
235
|
+
<td><tt>Integer</tt></td>
|
236
|
+
<td><tt>20</tt></td>
|
237
|
+
</tr>
|
238
|
+
|
239
|
+
</table>
|
240
|
+
|
241
|
+
### aws_credentials
|
242
|
+
<a name="aws_credentials">
|
243
|
+
|
244
|
+
Qurd supports [AssumeRoleCredentials][1], [Credentials][2],
|
245
|
+
[InstanceProfileCredentials][3], and [SharedCredentials][4]. Each credential
|
246
|
+
must be named and must have a type defined. The options key allows the caller to
|
247
|
+
define keys and values, mirroring the options for each of the credential types.
|
248
|
+
|
249
|
+
If no aws_credentials are defined in the configuration file, the key `default`
|
250
|
+
is created and it will attempt to use `Aws::InstanceProfileCredentials`. Each
|
251
|
+
auto_scaling_queues will have its credentials automatically set to `default`.
|
252
|
+
|
253
|
+
```yaml
|
254
|
+
aws_credentials:
|
255
|
+
- name: prod
|
256
|
+
type: assume_role_credentials
|
257
|
+
options:
|
258
|
+
role_arn: "arn:aws:iam::1:user/bob@example.com"
|
259
|
+
role_session_name: foo
|
260
|
+
- name: staging
|
261
|
+
type: credentials
|
262
|
+
options:
|
263
|
+
access_key_id: abc123
|
264
|
+
secret_access_key: 123abc
|
265
|
+
- name: dev
|
266
|
+
type: instance_profile_credentials
|
267
|
+
- name: test
|
268
|
+
type: shared_credentials
|
269
|
+
options:
|
270
|
+
profile_name: default
|
271
|
+
```
|
272
|
+
|
273
|
+
### auto_scaling_queues
|
274
|
+
<a name="auto_scaling_queues">
|
275
|
+
|
276
|
+
A hash of hashes, which describe the queues to be monitored. The outer key is
|
277
|
+
the name of the group of queues, ie production, staging, etc. The inner keys
|
278
|
+
`credentials`, `region`, and `queues` are required. Credentials should refer to the
|
279
|
+
`name` of an `aws_credential`. The `region` is the region of the queues. The
|
280
|
+
`queues` key is an array of queue names and regular expressions. Regular
|
281
|
+
expressions are strings, which begin and end with forward slash. Regular
|
282
|
+
expressions can also have [modifiers][5] applied to them.
|
283
|
+
|
284
|
+
The optional keys `wait_time` and `visibility_timeout` override the global
|
285
|
+
options of the same name.
|
286
|
+
|
287
|
+
The `credentials` key will be overridden, and set to `default`, if no
|
288
|
+
`aws_credentials` are defined.
|
289
|
+
|
290
|
+
```yaml
|
291
|
+
auto_scaling_queues:
|
292
|
+
dev:
|
293
|
+
credentials: dev
|
294
|
+
region: us-east-1
|
295
|
+
queues: "/scalingnotificationsqueue/i"
|
296
|
+
staging:
|
297
|
+
credentials: staging
|
298
|
+
region: us-west-2
|
299
|
+
visibility_timeout: 100
|
300
|
+
wait_time: 20
|
301
|
+
queues:
|
302
|
+
- FooQueue
|
303
|
+
- BarQueue
|
304
|
+
- "/ScalingNotificationsQueue/"
|
305
|
+
```
|
306
|
+
|
307
|
+
### actions
|
308
|
+
<a name="actions">
|
309
|
+
|
310
|
+
A hash of arrays, describing actions to take, for any autoscaling event. To test
|
311
|
+
the various options, you could configure the dummy action for each event.
|
312
|
+
|
313
|
+
```yaml
|
314
|
+
actions:
|
315
|
+
launch:
|
316
|
+
- "Qurd::Action::Dummy"
|
317
|
+
launch_error:
|
318
|
+
- "Qurd::Action::Dummy"
|
319
|
+
terminate:
|
320
|
+
- "Qurd::Action::Dummy"
|
321
|
+
terminate_error:
|
322
|
+
- "Qurd::Action::Dummy"
|
323
|
+
test:
|
324
|
+
- "Qurd::Action::Dummy"
|
325
|
+
```
|
326
|
+
|
327
|
+
## Installation
|
328
|
+
|
329
|
+
Add this line to your application's Gemfile:
|
330
|
+
|
331
|
+
```ruby
|
332
|
+
gem 'qurd'
|
333
|
+
```
|
334
|
+
|
335
|
+
And then execute:
|
336
|
+
|
337
|
+
$ bundle
|
338
|
+
|
339
|
+
Or install it yourself as:
|
340
|
+
|
341
|
+
$ gem install qurd
|
342
|
+
|
343
|
+
## Usage
|
344
|
+
|
345
|
+
`qurd [/PATH/TO/CONFIG.yml]`
|
346
|
+
|
347
|
+
## Tests
|
348
|
+
|
349
|
+
`bundle exec rake`
|
350
|
+
|
351
|
+
WebMock stubs can be found in test/support/web_mock_stubs.rb, responses are in
|
352
|
+
test/responses.
|
353
|
+
|
354
|
+
## Contributing
|
355
|
+
|
356
|
+
1. Fork it ( https://github.com/Adaptly/qurd/fork )
|
357
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
358
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
359
|
+
1. Write some tests!
|
360
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
361
|
+
5. Create a new Pull Request
|
362
|
+
|
363
|
+
[1]: http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html
|
364
|
+
[2]: http://docs.aws.amazon.com/sdkforruby/api/Aws/Credentials.html
|
365
|
+
[3]: http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html
|
366
|
+
[4]: http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html
|
367
|
+
[5]: http://ruby-doc.org/core-2.1.1/Regexp.html#class-Regexp-label-Options
|
data/Rakefile
ADDED
data/bin/qurd
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$: << File.dirname(__FILE__) + '/../lib'
|
3
|
+
require 'qurd'
|
4
|
+
if ARGV[0] && !File.exist?(ARGV[0])
|
5
|
+
STDERR.puts "File does not exist: '#{ARGV[0]}'"
|
6
|
+
STDERR.puts "Usage: qurd [/path/to/config.yml]"
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
Qurd.start(ARGV[0])
|
10
|
+
# vim:ft=ruby:
|