sqs-grep 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/sqs-grep +62 -0
- data/lib/sqs-grep/client.rb +25 -0
- data/lib/sqs-grep/config.rb +34 -0
- data/lib/sqs-grep/runner.rb +92 -0
- data/lib/sqs-grep/version.rb +3 -0
- data/lib/sqs-grep.rb +4 -0
- metadata +69 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f5cd92b73616690e04c935d2cf9b0a0e5ee59c4e
|
4
|
+
data.tar.gz: 9c50f8b486da9993030941678111c5872aa66949
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b1b2cb9529c844bd238af52fc8e09e73971cbb4bf5f629594dbcdb977bbfe40975aadee2750c0c27c80b62cd59043c659dd8a5e70e4f16714c017d2041c699d7
|
7
|
+
data.tar.gz: a9c63c479a1484182b966ae27f9499905e43c993e112eb0b65f71df689e4ca82976d2420ae73c0fbc25a2653ddc7037198f805d2c0dfb90c65f1492764810e11
|
data/bin/sqs-grep
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
require 'sqs-grep'
|
6
|
+
|
7
|
+
config = SqsGrep::Config.new
|
8
|
+
|
9
|
+
opts_parser = OptionParser.new do |opts|
|
10
|
+
opts.banner = "Usage: sqs-grep [OPTIONS] PATTERN QUEUE"
|
11
|
+
opts.separator ""
|
12
|
+
opts.on("-d", "--delete", "Consume (delete) matched messages") do
|
13
|
+
config.delete_matched = true
|
14
|
+
end
|
15
|
+
opts.on("-v", "--invert-match", "Invert the matching logic") do
|
16
|
+
config.invert_match = true
|
17
|
+
end
|
18
|
+
opts.on("-m", "--max-count N", "Stop after N matches") do |n|
|
19
|
+
config.max_count = n.to_i
|
20
|
+
end
|
21
|
+
opts.on("-t", "--visibility-timeout N", "Set the visibility timeout (default: #{config.visibility_timeout})") do |n|
|
22
|
+
config.visibility_timeout = n.to_i
|
23
|
+
end
|
24
|
+
opts.on("-w", "--wait-time-seconds N", "Give up after this many seconds of no new messages (default: #{config.wait_time_seconds})") do |n|
|
25
|
+
config.wait_time_seconds = n.to_i
|
26
|
+
end
|
27
|
+
opts.on("-j", "--json", "Show output in json format") do
|
28
|
+
config.json_format = true
|
29
|
+
end
|
30
|
+
opts.separator ""
|
31
|
+
opts.separator <<-EOF
|
32
|
+
Examples. (Note the use of the pattern "^" to match all messages).
|
33
|
+
|
34
|
+
# Show all messages, delete nothing:
|
35
|
+
sqs-grep ^ name-of-queue
|
36
|
+
|
37
|
+
# Show only messages matching the given pattern, without deleting them:
|
38
|
+
sqs-grep "[Ee]xception.*at line" name-of-queue
|
39
|
+
|
40
|
+
# Show and consume (delete) all messages (somewhat like sqs-receive):
|
41
|
+
sqs-grep --delete ^ name-of-queue
|
42
|
+
|
43
|
+
# Show and consume (delete) only messages matching matching some pattern:
|
44
|
+
sqs-grep --delete 1caf7fdb-e47a-46af-b8e3-4d955883a396 name-of-queue
|
45
|
+
|
46
|
+
EOF
|
47
|
+
end
|
48
|
+
|
49
|
+
opts_parser.parse!
|
50
|
+
|
51
|
+
if ARGV.count != 2
|
52
|
+
$stderr.puts opts_parser.help
|
53
|
+
exit 2
|
54
|
+
end
|
55
|
+
|
56
|
+
(config.pattern, config.queue_name) = ARGV
|
57
|
+
config.validate
|
58
|
+
|
59
|
+
rc = SqsGrep::Runner.new(config).run
|
60
|
+
exit rc
|
61
|
+
|
62
|
+
# eof sqs-grep
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module SqsGrep
|
2
|
+
|
3
|
+
class Client
|
4
|
+
|
5
|
+
def self.configure
|
6
|
+
Aws.config.merge! core_v2_options
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.core_v2_options
|
10
|
+
{
|
11
|
+
http_proxy: get_proxy,
|
12
|
+
user_agent_suffix: "sqs-grep #{VERSION}",
|
13
|
+
# http_wire_trace: true,
|
14
|
+
}
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.get_proxy
|
18
|
+
e = ENV['https_proxy']
|
19
|
+
e = "https://#{e}" if e && !e.empty? && !e.start_with?('http')
|
20
|
+
return e
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module SqsGrep
|
2
|
+
|
3
|
+
class Config
|
4
|
+
# Stronger builder pattern would be nice
|
5
|
+
attr_accessor :client_options, :sqs_client,
|
6
|
+
:pattern, :queue_name,
|
7
|
+
:visibility_timeout,
|
8
|
+
:wait_time_seconds,
|
9
|
+
:max_count,
|
10
|
+
:delete_matched,
|
11
|
+
:invert_match,
|
12
|
+
:json_format
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
@client_options = {}
|
16
|
+
@visibility_timeout = 30
|
17
|
+
@wait_time_seconds = 10
|
18
|
+
@max_count = nil
|
19
|
+
@delete_matched = false
|
20
|
+
@invert_match = false
|
21
|
+
@json_format = false
|
22
|
+
end
|
23
|
+
|
24
|
+
def validate
|
25
|
+
if !@pattern
|
26
|
+
raise "Missing pattern"
|
27
|
+
end
|
28
|
+
if !@queue_name
|
29
|
+
raise "Missing queue name"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
require 'json'
|
3
|
+
require 'set'
|
4
|
+
|
5
|
+
module SqsGrep
|
6
|
+
|
7
|
+
class Runner
|
8
|
+
|
9
|
+
def initialize(config)
|
10
|
+
@config = config
|
11
|
+
|
12
|
+
@config.sqs_client ||= begin
|
13
|
+
effective_options = SqsGrep::Client.core_v2_options.merge(config.client_options)
|
14
|
+
Aws::SQS::Client.new(effective_options)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Sets $stdout.sync
|
19
|
+
def run
|
20
|
+
queue_name = @config.queue_name
|
21
|
+
queue_url = @config.sqs_client.list_queues(queue_name_prefix: queue_name).queue_urls.find {|url| File.basename(url) == queue_name}
|
22
|
+
queue_url or raise "Can't find queue named #{queue_name.inspect}"
|
23
|
+
|
24
|
+
$stdout.sync = true
|
25
|
+
|
26
|
+
seen_message_ids = Set.new
|
27
|
+
num_matched = 0
|
28
|
+
|
29
|
+
loop do
|
30
|
+
r = @config.sqs_client.receive_message(
|
31
|
+
queue_url: queue_url,
|
32
|
+
attribute_names: %w[ All ],
|
33
|
+
max_number_of_messages: 10,
|
34
|
+
visibility_timeout: @config.visibility_timeout,
|
35
|
+
wait_time_seconds: @config.wait_time_seconds,
|
36
|
+
)
|
37
|
+
|
38
|
+
break if r.messages.empty?
|
39
|
+
|
40
|
+
r.messages.each do |m|
|
41
|
+
# puts "%s\t%s\t%s" % [ queue_name, m.message_id, m.receipt_handle ]
|
42
|
+
|
43
|
+
if seen_message_ids.include? m.message_id
|
44
|
+
$stderr.puts "Already seen message #{m.message_id} - bailing out in case we're looping"
|
45
|
+
return 0
|
46
|
+
end
|
47
|
+
seen_message_ids << m.message_id
|
48
|
+
|
49
|
+
matches = (m.body.match(@config.pattern) != nil)
|
50
|
+
|
51
|
+
if matches ^ @config.invert_match
|
52
|
+
json_data = {
|
53
|
+
queue_url: queue_url,
|
54
|
+
queue_name: queue_name,
|
55
|
+
message_id: m.message_id,
|
56
|
+
attributes: m.attributes.to_h,
|
57
|
+
body: m.body,
|
58
|
+
}
|
59
|
+
if !@config.json_format
|
60
|
+
puts "%s\t%s" % [ queue_name, m.message_id ]
|
61
|
+
puts m.attributes.inspect
|
62
|
+
puts m.body
|
63
|
+
puts ""
|
64
|
+
end
|
65
|
+
|
66
|
+
if @config.delete_matched
|
67
|
+
delete_res = @config.sqs_client.delete_message(
|
68
|
+
queue_url: queue_url,
|
69
|
+
receipt_handle: m.receipt_handle,
|
70
|
+
)
|
71
|
+
if !@config.json_format
|
72
|
+
p delete_res
|
73
|
+
puts ""
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
if @config.json_format
|
78
|
+
puts JSON.pretty_generate(json_data)
|
79
|
+
end
|
80
|
+
|
81
|
+
num_matched = num_matched + 1
|
82
|
+
return 0 if @config.max_count and num_matched >= @config.max_count
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
return 0
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
data/lib/sqs-grep.rb
ADDED
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sqs-grep
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Rachel Evans
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-07-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.0'
|
27
|
+
description: "\n sqs-grep iterates through each message on the given SQS queue,
|
28
|
+
testing its\n body against the given regular expression, displaying matching
|
29
|
+
messages to\n standard output.\n\n Options are available to control match
|
30
|
+
inversion, json output, deletion of\n matching messages, match limits, and timeouts.\n\n
|
31
|
+
\ Respects $https_proxy.\n "
|
32
|
+
email: sqs-grep-git@rve.org.uk
|
33
|
+
executables:
|
34
|
+
- sqs-grep
|
35
|
+
extensions: []
|
36
|
+
extra_rdoc_files: []
|
37
|
+
files:
|
38
|
+
- bin/sqs-grep
|
39
|
+
- lib/sqs-grep.rb
|
40
|
+
- lib/sqs-grep/client.rb
|
41
|
+
- lib/sqs-grep/config.rb
|
42
|
+
- lib/sqs-grep/runner.rb
|
43
|
+
- lib/sqs-grep/version.rb
|
44
|
+
homepage: https://github.com/rvedotrc/sqs-grep
|
45
|
+
licenses:
|
46
|
+
- Apache-2.0
|
47
|
+
metadata: {}
|
48
|
+
post_install_message:
|
49
|
+
rdoc_options: []
|
50
|
+
require_paths:
|
51
|
+
- lib
|
52
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '0'
|
57
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
requirements: []
|
63
|
+
rubyforge_project:
|
64
|
+
rubygems_version: 2.5.1
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: Find messages on an SQS queue by regular expression, and optionally delete
|
68
|
+
them
|
69
|
+
test_files: []
|