pheme 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46433ad750c6060d267dc7124847580d107274c9
-  data.tar.gz: 11cf1262b1da8a5e590eef8ef023a88734f1874f
+  metadata.gz: 25cd1b421feda98de4c2a6e14505c6f6c73f86e8
+  data.tar.gz: a90b834b7f9daf20e4cc371c9d7c36ffeb71a703
 SHA512:
-  metadata.gz: b309d7c66b7ed47901c4748476e5ac428513dfb89b531c64f1936f0f48e1dfc555acc3fbc586372c25592015d9998f6e6bfd6c236a21d8504bc26e11afa8b919
-  data.tar.gz: f9c7870ba06f05d0d01dbc35f09e6d392cb5d4ebd272d0ef7fc7ef938516d52862517875322aafe03390441d02829148160f7ddc8315b7136f5924cfcfe742c9
+  metadata.gz: 8d2101348a395491f6df3dbc74e14ab7e2701f21566f876d0a0f9fe256128a592b75280f18094f24ee97f90a420d5c0a7d9b1fc0c1ff0c2d294c9ee6d69d7bf6
+  data.tar.gz: 4f4e17a4660e095ca04115d600a257406f5e91f56695dc93bccf3d59ab85df2071b6dcc0d0607a7adc466f83f150d2f19e3ba8cfd55857f952f79a0fc6cb283e

data/Gemfile

@@ -1,3 +1,7 @@
 source "https://rubygems.org"
 
+group :development, :test do
+  gem 'bundler-audit', require: false
+end
+
 gemspec

data/circle.yml

@@ -1,9 +1,13 @@
+machine:
+  ruby:
+    version: 2.2.2
 dependencies:
   override:
-    - 'rvm-exec 2.1.0 bundle install'
+    - 'rvm-exec 2.2.2 bundle install'
     - 'rvm-exec 2.3.0 bundle install'
-
+  post:
+    - bundle exec bundle-audit update && bundle exec bundle-audit check
 test:
   override:
-    - 'rvm-exec 2.1.0 bundle exec rspec'
+    - 'rvm-exec 2.2.2 bundle exec rspec'
     - 'rvm-exec 2.3.0 bundle exec rspec'

data/lib/pheme.rb

@@ -2,10 +2,12 @@ require 'active_support/all'
 require 'recursive-open-struct'
 require 'aws-sdk'
 require 'securerandom'
+require 'smarter_csv'
 
 require 'pheme/version'
 require 'pheme/configuration'
 require 'pheme/logger'
+require 'pheme/rollbar'
 require 'pheme/topic_publisher'
 require 'pheme/message_handler'
 require 'pheme/queue_poller'

data/lib/pheme/configuration.rb

@@ -16,7 +16,7 @@ module Pheme
   end
 
   class Configuration
-    ATTRIBUTES = [:sns_client, :sqs_client, :logger]
+    ATTRIBUTES = [:sns_client, :sqs_client, :logger, :rollbar]
     attr_accessor *ATTRIBUTES
 
     def initialize

data/lib/pheme/queue_poller.rb

@@ -1,29 +1,39 @@
 module Pheme
   class QueuePoller
-    attr_accessor :queue_url, :queue_poller, :connection_pool_block, :poller_configuration
+    attr_accessor :queue_url, :queue_poller, :connection_pool_block, :format, :max_messages, :poller_configuration
 
-    def initialize(queue_url:, connection_pool_block: false, poller_configuration: {})
+    def initialize(queue_url:, connection_pool_block: false, max_messages: nil, format: :json, poller_configuration: {})
       raise ArgumentError, "must specify non-nil queue_url" unless queue_url.present?
       @queue_url = queue_url
       @queue_poller = Aws::SQS::QueuePoller.new(queue_url)
       @connection_pool_block = connection_pool_block
+      @format = format
+      @max_messages = max_messages
       @poller_configuration = {
         wait_time_seconds: 10, # amount of time a long polling receive call can wait for a mesage before receiving a empty response (which will trigger another polling request)
         idle_timeout: 20, # disconnects poller after 20 seconds of idle time
         skip_delete: true, # manually delete messages
       }.merge(poller_configuration || {})
+
+      if max_messages
+        queue_poller.before_request do |stats|
+          throw :stop_polling if stats.received_message_count >= max_messages
+        end
+      end
     end
 
     def poll
       Pheme.log(:info, "Long-polling for messages on #{queue_url}")
       with_optional_connection_pool_block do
         queue_poller.poll(poller_configuration) do |message|
+          data = parse_message(message)
           begin
-            handle(parse_message(message))
+            handle(data)
             queue_poller.delete_message(message)
           rescue => e
             Pheme.log(:error, "Exception: #{e.inspect}")
             Pheme.log(:error, e.backtrace.join("\n"))
+            Pheme.rollbar(e, "#{self.class} failed to process message", data)
           end
         end
       end
@@ -33,8 +43,24 @@ module Pheme
     def parse_message(message)
       Pheme.log(:info, "Received JSON payload: #{message.body}")
       body = JSON.parse(message.body)
-      parsed_body = JSON.parse(body['Message'])
-      RecursiveOpenStruct.new(parsed_body, recurse_over_arrays: true)
+      case format
+      when :csv
+        parse_csv(body['Message'])
+      when :json
+        parse_json(body['Message'])
+      else
+        raise ArgumentError.new("Unknown format #{format}. Valid formats: :csv, :json.")
+      end
+    end
+
+    def parse_csv(message_contents)
+      parsed_body = SmarterCSV.process(StringIO.new(message_contents))
+      parsed_body.map{ |item| RecursiveOpenStruct.new(item, recurse_over_arrays: true) }
+    end
+
+    def parse_json(message_contents)
+      parsed_body = JSON.parse(message_contents)
+      RecursiveOpenStruct.new({wrapper: parsed_body}, recurse_over_arrays: true).wrapper
     end
 
     def handle(message)

data/lib/pheme/rollbar.rb

@@ -0,0 +1,6 @@
+module Pheme
+  def self.rollbar(exception, message, data={})
+    return  if configuration.rollbar.nil?
+    configuration.rollbar.error(exception, message, data)
+  end
+end

data/lib/pheme/version.rb

@@ -1,3 +1,3 @@
 module Pheme
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/pheme.gemspec

@@ -21,8 +21,9 @@ Gem::Specification.new do |gem|
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "aws-sdk", "~> 2"
-  gem.add_dependency "activesupport", "~> 4"
+  gem.add_dependency "activesupport", ">= 4"
   gem.add_dependency "recursive-open-struct", "~> 1"
+  gem.add_dependency "smarter_csv", "~> 1"
 
   gem.add_development_dependency "rspec", "~> 3.4"
   gem.add_development_dependency "rspec_junit_formatter", "~> 0.2"

data/spec/queue_poller_spec.rb

@@ -3,6 +3,8 @@ describe Pheme::QueuePoller do
   let(:poller) do
     poller = double
     allow(poller).to receive(:poll).with(kind_of(Hash))
+    allow(poller).to receive(:parse_message)
+    allow(poller).to receive(:before_request)
     poller
   end
   before(:each) do
@@ -22,6 +24,64 @@ describe Pheme::QueuePoller do
         expect { ExampleQueuePoller.new(queue_url: nil) }.to raise_error(ArgumentError)
       end
     end
+
+    context "when initialized with max_messages" do
+      it "should set max_messages" do
+        expect(ExampleQueuePoller.new(queue_url: "queue_url", max_messages: 5).max_messages).to eq(5)
+      end
+    end
+  end
+
+  describe "#parse_message" do
+    context "with JSON message" do
+      subject { ExampleQueuePoller.new(queue_url: queue_url) }
+
+      let!(:message) { OpenStruct.new({
+        body: '{"Message":"{\"test\":\"test\"}"}'
+      })}
+
+      it 'should parse the message correctly' do
+        expect(subject.parse_message(message).test).to eq("test")
+      end
+    end
+
+    context "with CSV message" do
+      subject { ExampleQueuePoller.new(queue_url: queue_url, format: :csv) }
+
+      let!(:message) { OpenStruct.new({
+        body:'{"Message":"test,test2\nvalue,value2\nvalue3,value4"}'
+      })}
+
+      it 'should parse the message correctly' do
+        expect(subject.parse_message(message)).to be_a(Array)
+        expect(subject.parse_message(message).count).to eq(2)
+      end
+    end
+
+    context "with unknown message format" do
+      subject { ExampleQueuePoller.new(queue_url: queue_url, format: :invalid_format) }
+
+      let!(:message) { OpenStruct.new({
+        body:'{"Message":"test,test2\nvalue,value2\nvalue3,value4"}'
+      })}
+
+      it 'should raise an error' do
+        expect{ subject.parse_message(message) }.to raise_error
+      end
+    end
+
+    context "with array JSON message" do
+      subject { ExampleQueuePoller.new(queue_url: queue_url).parse_message(message) }
+      let!(:message) { OpenStruct.new({
+        body: '{"Message":"[[{\"test\":\"test\"}]]"}'
+      })}
+      it 'should parse the message correctly' do
+        expect(subject.first.first.test).to eq("test")
+        expect(subject).to be_a Array
+        expect(subject.first).to be_a Array
+        expect(subject.first.first).to be_a RecursiveOpenStruct
+      end
+    end
   end
 
   describe "#poll" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pheme
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Peter Graham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-02 00:00:00.000000000 Z
+date: 2016-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -28,14 +28,14 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
 - !ruby/object:Gem::Dependency
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +112,7 @@ files:
 - lib/pheme/logger.rb
 - lib/pheme/message_handler.rb
 - lib/pheme/queue_poller.rb
+- lib/pheme/rollbar.rb
 - lib/pheme/topic_publisher.rb
 - lib/pheme/version.rb
 - pheme.gemspec
@@ -144,4 +159,3 @@ test_files:
 - spec/support/example_queue_poller.rb
 - spec/topic_publisher_spec.rb
 - spec/version_spec.rb
-has_rdoc: