sqs-grep 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f5cd92b73616690e04c935d2cf9b0a0e5ee59c4e
+  data.tar.gz: 9c50f8b486da9993030941678111c5872aa66949
+SHA512:
+  metadata.gz: b1b2cb9529c844bd238af52fc8e09e73971cbb4bf5f629594dbcdb977bbfe40975aadee2750c0c27c80b62cd59043c659dd8a5e70e4f16714c017d2041c699d7
+  data.tar.gz: a9c63c479a1484182b966ae27f9499905e43c993e112eb0b65f71df689e4ca82976d2420ae73c0fbc25a2653ddc7037198f805d2c0dfb90c65f1492764810e11

data/bin/sqs-grep

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+require 'sqs-grep'
+
+config = SqsGrep::Config.new
+
+opts_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: sqs-grep [OPTIONS] PATTERN QUEUE"
+  opts.separator ""
+  opts.on("-d", "--delete", "Consume (delete) matched messages") do
+    config.delete_matched = true
+  end
+  opts.on("-v", "--invert-match", "Invert the matching logic") do
+    config.invert_match = true
+  end
+  opts.on("-m", "--max-count N", "Stop after N matches") do |n|
+    config.max_count = n.to_i
+  end
+  opts.on("-t", "--visibility-timeout N", "Set the visibility timeout (default: #{config.visibility_timeout})") do |n|
+    config.visibility_timeout = n.to_i
+  end
+  opts.on("-w", "--wait-time-seconds N", "Give up after this many seconds of no new messages (default: #{config.wait_time_seconds})") do |n|
+    config.wait_time_seconds = n.to_i
+  end
+  opts.on("-j", "--json", "Show output in json format") do
+    config.json_format = true
+  end
+  opts.separator ""
+  opts.separator <<-EOF
+Examples. (Note the use of the pattern "^" to match all messages).
+
+  # Show all messages, delete nothing:
+  sqs-grep ^ name-of-queue
+
+  # Show only messages matching the given pattern, without deleting them:
+  sqs-grep "[Ee]xception.*at line" name-of-queue
+
+  # Show and consume (delete) all messages (somewhat like sqs-receive):
+  sqs-grep --delete ^ name-of-queue
+
+  # Show and consume (delete) only messages matching matching some pattern:
+  sqs-grep --delete 1caf7fdb-e47a-46af-b8e3-4d955883a396 name-of-queue
+
+  EOF
+end
+
+opts_parser.parse!
+
+if ARGV.count != 2
+  $stderr.puts opts_parser.help
+  exit 2
+end
+
+(config.pattern, config.queue_name) = ARGV
+config.validate
+
+rc = SqsGrep::Runner.new(config).run
+exit rc
+
+# eof sqs-grep

data/lib/sqs-grep/client.rb

@@ -0,0 +1,25 @@
+module SqsGrep
+
+  class Client
+
+    def self.configure
+      Aws.config.merge! core_v2_options
+    end
+
+    def self.core_v2_options
+      {
+	http_proxy: get_proxy,
+	user_agent_suffix: "sqs-grep #{VERSION}",
+	# http_wire_trace: true,
+      }
+    end
+
+    def self.get_proxy
+      e = ENV['https_proxy']
+      e = "https://#{e}" if e && !e.empty? && !e.start_with?('http')
+      return e
+    end
+
+  end
+  
+end

data/lib/sqs-grep/config.rb

@@ -0,0 +1,34 @@
+module SqsGrep
+
+  class Config
+    # Stronger builder pattern would be nice
+    attr_accessor :client_options, :sqs_client,
+      :pattern, :queue_name,
+      :visibility_timeout,
+      :wait_time_seconds,
+      :max_count,
+      :delete_matched,
+      :invert_match,
+      :json_format
+
+    def initialize
+      @client_options = {}
+      @visibility_timeout = 30
+      @wait_time_seconds = 10
+      @max_count = nil
+      @delete_matched = false
+      @invert_match = false
+      @json_format = false
+    end
+
+    def validate
+      if !@pattern
+        raise "Missing pattern"
+      end
+      if !@queue_name
+        raise "Missing queue name"
+      end
+    end
+  end
+
+end

data/lib/sqs-grep/runner.rb

@@ -0,0 +1,92 @@
+require 'aws-sdk'
+require 'json'
+require 'set'
+
+module SqsGrep
+
+  class Runner
+
+    def initialize(config)
+      @config = config
+
+      @config.sqs_client ||= begin
+                               effective_options = SqsGrep::Client.core_v2_options.merge(config.client_options)
+                               Aws::SQS::Client.new(effective_options)
+                             end
+    end
+
+    # Sets $stdout.sync
+    def run
+      queue_name = @config.queue_name
+      queue_url = @config.sqs_client.list_queues(queue_name_prefix: queue_name).queue_urls.find {|url| File.basename(url) == queue_name}
+      queue_url or raise "Can't find queue named #{queue_name.inspect}"
+
+      $stdout.sync = true
+
+      seen_message_ids = Set.new
+      num_matched = 0
+
+      loop do
+        r = @config.sqs_client.receive_message(
+          queue_url: queue_url,
+          attribute_names: %w[ All ],
+          max_number_of_messages: 10,
+          visibility_timeout: @config.visibility_timeout,
+          wait_time_seconds: @config.wait_time_seconds,
+        )
+
+        break if r.messages.empty?
+
+        r.messages.each do |m|
+          # puts "%s\t%s\t%s" % [ queue_name, m.message_id, m.receipt_handle ]
+
+          if seen_message_ids.include? m.message_id
+            $stderr.puts "Already seen message #{m.message_id} - bailing out in case we're looping"
+            return 0
+          end
+          seen_message_ids << m.message_id
+
+          matches = (m.body.match(@config.pattern) != nil)
+
+          if matches ^ @config.invert_match
+            json_data = {
+              queue_url: queue_url,
+              queue_name: queue_name,
+              message_id: m.message_id,
+              attributes: m.attributes.to_h,
+              body: m.body,
+            }
+            if !@config.json_format
+              puts "%s\t%s" % [ queue_name, m.message_id ]
+              puts m.attributes.inspect
+              puts m.body
+              puts ""
+            end
+
+            if @config.delete_matched
+              delete_res = @config.sqs_client.delete_message(
+                queue_url: queue_url,
+                receipt_handle: m.receipt_handle,
+              )
+              if !@config.json_format
+                p delete_res
+                puts ""
+              end
+            end
+
+            if @config.json_format
+              puts JSON.pretty_generate(json_data)
+            end
+
+            num_matched = num_matched + 1
+            return 0 if @config.max_count and num_matched >= @config.max_count
+          end
+        end
+      end
+
+      return 0
+    end
+
+  end
+
+end

data/lib/sqs-grep/version.rb

@@ -0,0 +1,3 @@
+module SqsGrep
+  VERSION = '0.1.0'
+end

data/lib/sqs-grep.rb

@@ -0,0 +1,4 @@
+require_relative './sqs-grep/client.rb'
+require_relative './sqs-grep/config.rb'
+require_relative './sqs-grep/runner.rb'
+require_relative './sqs-grep/version.rb'

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: sqs-grep
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Rachel Evans
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: "\n    sqs-grep iterates through each message on the given SQS queue,
+  testing its\n    body against the given regular expression, displaying matching
+  messages to\n    standard output.\n\n    Options are available to control match
+  inversion, json output, deletion of\n    matching messages, match limits, and timeouts.\n\n
+  \   Respects $https_proxy.\n  "
+email: sqs-grep-git@rve.org.uk
+executables:
+- sqs-grep
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/sqs-grep
+- lib/sqs-grep.rb
+- lib/sqs-grep/client.rb
+- lib/sqs-grep/config.rb
+- lib/sqs-grep/runner.rb
+- lib/sqs-grep/version.rb
+homepage: https://github.com/rvedotrc/sqs-grep
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Find messages on an SQS queue by regular expression, and optionally delete
+  them
+test_files: []