pheme 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b5feb32f5663e1118ccd4307e0552cb85e513a10
+  data.tar.gz: 80cad78d39a25a965129eedb7411cbc96ccffc6b
+SHA512:
+  metadata.gz: 3348f6ea4828665392ce7770e577ba35fb4d07d7d5409c38a1dde176b9fc194a2e3315c9a9eef4f1445ef9c0875acd3ee62cb778233eb5758300ebbfcaeff581
+  data.tar.gz: 2d608692cb8dba727888309160fba22f84a05d2121ed00251186f0fa980ff4380aa696812cfa2fa7c411d1e1749bffe64a8a3b988f061aa513d8bf78827ee1a4

data/.gitignore

@@ -0,0 +1,36 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--format documentation

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Wealthsimple
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# pheme [![Circle CI](https://circleci.com/gh/wealthsimple/pheme.svg?style=svg)](https://circleci.com/gh/wealthsimple/pheme)
+
+Ruby SNS publisher + SQS poller & message handler
+
+## installation & config
+
+```ruby
+# Gemfile
+gem 'pheme'
+```
+
+```ruby
+# Initializer
+aws_config = {
+  credentials: Aws::Credentials.new('YOUR_ACCESS_KEY_ID', 'YOUR_SECRET_ACCESS_KEY'),
+  region: 'us-east-1',
+}
+AWS_SNS_CLIENT = Aws::SNS::Client.new(aws_config)
+AWS_SQS_CLIENT = Aws::SQS::Client.new(aws_config)
+
+Pheme.configure do |config|
+  config.sqs_client = AWS_SQS_CLIENT
+  config.sns_client = AWS_SNS_CLIENT
+  config.logger = Logger.new(STDOUT) # Optionally replace with your app logger, e.g. `Rails.logger`
+end
+```
+
+# usage
+
+See https://github.com/wealthsimple/pheme/tree/master/spec/support for example implementations of each class.
+
+TODO: write better usage instructions.

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  ruby:
+    version: 2.3.0

data/lib/pheme.rb

@@ -0,0 +1,11 @@
+require 'active_support/all'
+require 'recursive-open-struct'
+require 'aws-sdk'
+require 'securerandom'
+
+require 'pheme/version'
+require 'pheme/configuration'
+require 'pheme/logger'
+require 'pheme/topic_publisher'
+require 'pheme/message_handler'
+require 'pheme/queue_poller'

data/lib/pheme/configuration.rb

@@ -0,0 +1,34 @@
+module Pheme
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+
+  def self.reset_configuration!
+    @configuration = Configuration.new
+  end
+
+  class Configuration
+    ATTRIBUTES = [:sns_client, :sqs_client, :logger]
+    attr_accessor *ATTRIBUTES
+
+    def initialize
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def validate!
+      ATTRIBUTES.each do |attribute|
+        raise "Invalid or missing configuration for #{attribute}"  unless send(attribute).present?
+      end
+      raise "sns_client must be a Aws::SNS::Client"  unless sns_client.is_a?(Aws::SNS::Client)
+      raise "sns_client must be a Aws::SQS::Client"  unless sqs_client.is_a?(Aws::SQS::Client)
+    end
+  end
+end

data/lib/pheme/logger.rb

@@ -0,0 +1,7 @@
+module Pheme
+  def self.log(method, text)
+    @logger ||= ActiveSupport::TaggedLogging.new(configuration.logger)
+    @tag ||= "pheme_#{SecureRandom.uuid}"
+    @logger.tagged(@tag) { @logger.send(method, text) }
+  end
+end

data/lib/pheme/message_handler.rb

@@ -0,0 +1,13 @@
+module Pheme
+  class MessageHandler
+    attr_reader :message
+
+    def initialize(message:)
+      @message = message
+    end
+
+    def handle
+      raise NotImplementedError
+    end
+  end
+end

data/lib/pheme/queue_poller.rb

@@ -0,0 +1,54 @@
+module Pheme
+  class QueuePoller
+    attr_accessor :queue_url, :queue_poller, :connection_pool_block, :poller_configuration
+
+    def initialize(queue_url:, connection_pool_block: false, poller_configuration: {})
+      @queue_url = queue_url
+      @queue_poller = Aws::SQS::QueuePoller.new(queue_url)
+      @connection_pool_block = connection_pool_block
+      @poller_configuration = poller_configuration.merge({
+        wait_time_seconds: 10, # amount of time a long polling receive call can wait for a mesage before receiving a empty response (which will trigger another polling request)
+        idle_timeout: 20, # disconnects poller after 20 seconds of idle time
+        visibility_timeout: 30, # length of time in seconds that this message will not be visible to other receiving components
+        skip_delete: true, # manually delete messages
+      })
+    end
+
+    def poll
+      Pheme.log(:info, "Long-polling for messages on #{queue_url}")
+      with_optional_connection_pool_block do
+        queue_poller.poll(poller_configuration) do |message|
+          begin
+            handle(parse_message(message))
+            queue_poller.delete_message(message)
+          rescue => e
+            Pheme.log(:error, "Exception: #{e.inspect}")
+            Pheme.log(:error, e.backtrace.join("\n"))
+          end
+        end
+      end
+      Pheme.log(:info, "Finished long-polling after #{@poller_configuration[:idle_timeout]} seconds.")
+    end
+
+    def parse_message(message)
+      Pheme.log(:info, "Received JSON payload: #{message.body}")
+      body = JSON.parse(message.body)
+      parsed_body = JSON.parse(body['Message'])
+      RecursiveOpenStruct.new(parsed_body, recurse_over_arrays: true)
+    end
+
+    def handle(message)
+      raise NotImplementedError
+    end
+
+  private
+
+    def with_optional_connection_pool_block(&blk)
+      if connection_pool_block
+        ActiveRecord::Base.connection_pool.with_connection { blk.call }
+      else
+        blk.call
+      end
+    end
+  end
+end

data/lib/pheme/topic_publisher.rb

@@ -0,0 +1,18 @@
+module Pheme
+  class TopicPublisher
+    attr_accessor :topic_arn
+
+    def initialize(topic_arn:)
+      @topic_arn = topic_arn
+    end
+
+    def publish_events
+      raise NotImplementedError
+    end
+
+    def publish(message)
+      Pheme.log(:info, "Publishing to #{topic_arn}: #{message}")
+      Pheme.configuration.sns_client.publish(topic_arn: topic_arn, message: message.to_json)
+    end
+  end
+end

data/lib/pheme/version.rb

@@ -0,0 +1,3 @@
+module Pheme
+  VERSION = "0.0.1"
+end

data/pheme.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "pheme/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "pheme"
+  gem.version       = Pheme::VERSION
+  gem.authors       = ["Peter Graham"]
+  gem.email         = ["peter@wealthsimple.com"]
+  gem.description   = %q{Ruby AWS SNS publisher + SQS poller & message handler}
+  gem.summary       = %q{Ruby SNS publisher + SQS poller & message handler}
+  gem.homepage      = "https://github.com/wealthsimple/pheme"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.licenses      = ["MIT"]
+  gem.required_ruby_version = ">= 2.0.0"
+
+  gem.add_dependency "aws-sdk", "~> 2"
+  gem.add_dependency "activesupport", "~> 4"
+  gem.add_dependency "recursive-open-struct", "~> 1"
+
+  gem.add_development_dependency "rspec", "~> 3.4"
+  gem.add_development_dependency "rspec_junit_formatter", "~> 0.2"
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,22 @@
+describe Pheme do
+  describe ".configure" do
+    let(:sns_client) { double }
+    let(:sqs_client) { double }
+    let(:custom_logger) { double }
+    it "sets global configuration" do
+      expect(described_class.configuration.sns_client).to be_nil
+      expect(described_class.configuration.sqs_client).to be_nil
+      expect(described_class.configuration.logger).to be_a(Logger)
+
+      described_class.configure do |config|
+        config.sns_client = sns_client
+        config.sqs_client = sqs_client
+        config.logger = custom_logger
+      end
+
+      expect(described_class.configuration.sns_client).to eq(sns_client)
+      expect(described_class.configuration.sqs_client).to eq(sqs_client)
+      expect(described_class.configuration.logger).to eq(custom_logger)
+    end
+  end
+end

data/spec/message_handler_spec.rb

@@ -0,0 +1,12 @@
+describe Pheme::MessageHandler do
+  before(:each) { use_default_configuration! }
+  let(:message) { RecursiveOpenStruct.new(status: "complete") }
+  subject { ExampleMessageHandler.new(message: message) }
+
+  describe "#handle" do
+    it "handles the message correctly" do
+      expect(Pheme).to receive(:log).with(:info, "Done")
+      subject.handle
+    end
+  end
+end

data/spec/queue_poller_spec.rb

@@ -0,0 +1,111 @@
+describe Pheme::QueuePoller do
+  let(:queue_url) { "https://sqs.us-east-1.amazonaws.com/whatever" }
+  let(:poller) do
+    poller = double
+    allow(poller).to receive(:poll).with(kind_of(Hash))
+    poller
+  end
+  before(:each) do
+    use_default_configuration!
+    allow(Aws::SQS::QueuePoller).to receive(:new) { poller }
+  end
+
+  describe "#poll" do
+    before(:each) do
+      module ActiveRecord
+        class Base
+          def self.connection_pool
+          end
+        end
+      end
+    end
+
+    context "with connection pool block" do
+      let(:mock_connection_pool) { double }
+
+      before(:each) do
+        allow(ActiveRecord::Base).to receive(:connection_pool) { mock_connection_pool }
+        allow(mock_connection_pool).to receive(:with_connection).and_yield
+      end
+
+      subject { ExampleQueuePoller.new(queue_url: queue_url, connection_pool_block: true) }
+
+      it "uses the connection pool block" do
+        expect(mock_connection_pool).to receive(:with_connection)
+        subject.poll
+      end
+    end
+
+    context "without connection pool block" do
+      subject { ExampleQueuePoller.new(queue_url: queue_url) }
+
+      it "does not call ActiveRecord" do
+        expect(ActiveRecord::Base).not_to receive(:connection_pool)
+        subject.poll
+      end
+    end
+
+    context "when a valid message is yielded" do
+      let(:message_body) do
+        {
+          id: "id-123",
+          status: "complete",
+        }
+      end
+      let(:message) do
+        message = double
+        allow(message).to receive(:body) do
+          {Message: message_body.to_json,}.to_json
+        end
+        message
+      end
+      before(:each) do
+        allow(poller).to receive(:poll).and_yield(message)
+      end
+
+      subject { ExampleQueuePoller.new(queue_url: queue_url) }
+
+      it "handles the message" do
+        expect(ExampleMessageHandler).to receive(:new).with(message: RecursiveOpenStruct.new(message_body))
+        subject.poll
+      end
+
+      it "deletes the message from the queue" do
+        expect(poller).to receive(:delete_message).with(message)
+        subject.poll
+      end
+    end
+
+    context "when an invalid message is yielded" do
+      let(:message_body) do
+        {
+          id: "id-123",
+          status: "unknown-abc",
+        }
+      end
+      let(:message) do
+        message = double
+        allow(message).to receive(:body) do
+          {Message: message_body.to_json}.to_json
+        end
+        message
+      end
+      before(:each) do
+        allow(poller).to receive(:poll).and_yield(message)
+        allow(Pheme).to receive(:log)
+      end
+
+      subject { ExampleQueuePoller.new(queue_url: queue_url) }
+
+      it "logs the error" do
+        subject.poll
+        expect(Pheme).to have_received(:log).with(:error, "Exception: #<ArgumentError: Unknown message status: unknown-abc>")
+      end
+
+      it "does not delete the message from the queue" do
+        expect(poller).not_to receive(:delete_message)
+        subject.poll
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+require 'rspec'
+
+require './lib/pheme'
+
+Dir["./spec/support/**/*.rb"].each { |f| require f }
+
+RSpec.configure do |config|
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  config.before(:each) do
+    Pheme.reset_configuration!
+  end
+end
+
+def use_default_configuration!
+  Pheme.configure do |config|
+    config.sqs_client = double
+    config.sns_client = double
+    config.logger = Logger.new(nil)
+  end
+end

data/spec/support/example_message_handler.rb

@@ -0,0 +1,10 @@
+class ExampleMessageHandler < Pheme::MessageHandler
+  def handle
+    case message.status
+    when "complete"
+      Pheme.log(:info, "Done")
+    when "rejected"
+      Pheme.log(:error, "Oops")
+    end
+  end
+end

data/spec/support/example_publisher.rb

@@ -0,0 +1,10 @@
+class ExamplePublisher < Pheme::TopicPublisher
+  def publish_events
+    2.times do |n|
+      publish({
+        id: "id-#{n}",
+        status: "complete",
+      })
+    end
+  end
+end

data/spec/support/example_queue_poller.rb

@@ -0,0 +1,10 @@
+class ExampleQueuePoller < Pheme::QueuePoller
+  def handle(message)
+    case message.status
+    when 'complete', 'rejected'
+      ExampleMessageHandler.new(message: message).handle
+    else
+      raise ArgumentError, "Unknown message status: #{message.status}"
+    end
+  end
+end

data/spec/topic_publisher_spec.rb

@@ -0,0 +1,18 @@
+describe Pheme::TopicPublisher do
+  before(:each) { use_default_configuration! }
+  subject { ExamplePublisher.new(topic_arn: "arn:aws:sns:whatever") }
+
+  describe "#publish_events" do
+    it "publishes the correct events" do
+      expect(Pheme.configuration.sns_client).to receive(:publish).with({
+        topic_arn: "arn:aws:sns:whatever",
+        message: {id: "id-0", status: "complete"}.to_json,
+      })
+      expect(Pheme.configuration.sns_client).to receive(:publish).with({
+        topic_arn: "arn:aws:sns:whatever",
+        message: {id: "id-1", status: "complete"}.to_json,
+      })
+      subject.publish_events
+    end
+  end
+end

data/spec/version_spec.rb

@@ -0,0 +1,5 @@
+describe Pheme do
+  it "has a version" do
+    expect(described_class::VERSION).to be_a(String)
+  end
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: pheme
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Peter Graham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: recursive-open-struct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: Ruby AWS SNS publisher + SQS poller & message handler
+email:
+- peter@wealthsimple.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE
+- README.md
+- circle.yml
+- lib/pheme.rb
+- lib/pheme/configuration.rb
+- lib/pheme/logger.rb
+- lib/pheme/message_handler.rb
+- lib/pheme/queue_poller.rb
+- lib/pheme/topic_publisher.rb
+- lib/pheme/version.rb
+- pheme.gemspec
+- spec/configuration_spec.rb
+- spec/message_handler_spec.rb
+- spec/queue_poller_spec.rb
+- spec/spec_helper.rb
+- spec/support/example_message_handler.rb
+- spec/support/example_publisher.rb
+- spec/support/example_queue_poller.rb
+- spec/topic_publisher_spec.rb
+- spec/version_spec.rb
+homepage: https://github.com/wealthsimple/pheme
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Ruby SNS publisher + SQS poller & message handler
+test_files:
+- spec/configuration_spec.rb
+- spec/message_handler_spec.rb
+- spec/queue_poller_spec.rb
+- spec/spec_helper.rb
+- spec/support/example_message_handler.rb
+- spec/support/example_publisher.rb
+- spec/support/example_queue_poller.rb
+- spec/topic_publisher_spec.rb
+- spec/version_spec.rb