pheme 3.3.3 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a6423f85631076b6455c45862fd88cd8fa834b8f4efa38ad9ff288d331f147c
-  data.tar.gz: 60a56ff0d795f713e59652dd5ee84607eea611b6792cb77435fbedc0803a003b
+  metadata.gz: 0a03bd21407e73fb60ef72e1802d45683fe8e5fb20017f5144d2a6fd5a34b59e
+  data.tar.gz: fd9e6b59b514c56700ded3dc91a654cea4fc0fc132c32cfd9496deea30688ab0
 SHA512:
-  metadata.gz: 476dc46442e717cde068c8566070234c8cca502c10cd9e2687e910e0e48a8eb557b17b12be1fb7012999b4524133369910db23627162cfe54d9475db1c10741e
-  data.tar.gz: 00abafdcf85d301da6d7d8464e926c5f9e3ac569d5bf339c59cd5163f0309510cc37eccb8ef7d8386f1ca9f31b52e3007449020aa8152ca04af3b332ab9b1d7d
+  metadata.gz: e69e7edc5eacbe3d6e81d2e478c7963acf878a44f970f7a8ebd0b1cfa91f2077132bf2f137e6fe13c82168170ac9e6258b6c3b78c7e7cb5caea5a9a8c9874345
+  data.tar.gz: 62ceeeec4b07913d09cd22d8e28b57e6fe19e72d62e2fa658e099ccd8ffea54753f1db71f1249e6d03366dc16c443ea0d2ff31601324f81d06b0f0e628fd925c

data/.circleci/config.yml

@@ -2,17 +2,17 @@ version: 2
 
 references:
   defaults:
-    2_4_5: &defaults_2_4_5
+    2_4_10: &defaults_2_4_10
       working_directory: ~/wealthsimple
       docker:
-        - image: circleci/ruby:2.4.5
+        - image: circleci/ruby:2.4.10
           environment:
             RAILS_ENV: test
 
-    2_5_3: &defaults
+    2_5_8: &defaults
       working_directory: ~/wealthsimple
       docker:
-        - image: circleci/ruby:2.5.3
+        - image: circleci/ruby:2.5.8
           environment:
             RAILS_ENV: test
 
@@ -46,8 +46,8 @@ references:
           --format documentation
 
 jobs:
-  checkout_and_bundle_2_4_5:
-    <<: *defaults_2_4_5
+  checkout_and_bundle_2_4_10:
+    <<: *defaults_2_4_10
     environment:
       COVERALLS_REPO_TOKEN: ''
     steps:
@@ -109,17 +109,20 @@ workflows:
   version: 2
   build:
     jobs:
-      - checkout_and_bundle_2_4_5:
+      - checkout_and_bundle_2_4_10:
           context: wealthsimple
       - checkout_and_bundle:
           context: wealthsimple
       - rspec:
+          context: wealthsimple
           requires:
             - checkout_and_bundle
       - lint_check:
+          context: wealthsimple
           requires:
             - checkout_and_bundle
       - vulnerability_check:
+          context: wealthsimple
           requires:
             - checkout_and_bundle
       - release:
@@ -128,10 +131,10 @@ workflows:
             branches:
               only: master
           requires:
-          - checkout_and_bundle_2_4_5
-          - rspec
-          - lint_check
-          - vulnerability_check
+            - checkout_and_bundle_2_4_10
+            - rspec
+            - lint_check
+            - vulnerability_check
 
   security-audit:
     triggers:
@@ -145,5 +148,6 @@ workflows:
       - checkout_and_bundle:
           context: wealthsimple
       - vulnerability_check:
+          context: wealthsimple
           requires:
             - checkout_and_bundle

data/.rubocop_todo.yml

@@ -1,42 +1,42 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-07-09 13:48:32 -0400 using RuboCop version 0.72.0.
+# on 2021-01-07 16:23:09 UTC using RuboCop version 1.6.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
+# Offense count: 19
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 147
+
 # Offense count: 2
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
 Metrics/AbcSize:
   Max: 22
 
 # Offense count: 1
-# Configuration parameters: CountComments.
-Metrics/ClassLength:
-  Max: 164
-
-# Offense count: 1
+# Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:
   Max: 10
 
-# Offense count: 17
-# Cop supports --auto-correct.
-# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
-# URISchemes: http, https
-Metrics/LineLength:
-  Max: 147
-
 # Offense count: 3
-# Configuration parameters: CountComments, ExcludedMethods.
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended, extended
 Metrics/MethodLength:
   Max: 28
 
 # Offense count: 1
+# Configuration parameters: IgnoredMethods.
 Metrics/PerceivedComplexity:
   Max: 11
 
-# Offense count: 5
-# Configuration parameters: CustomTransform, IgnoreMethods.
+# Offense count: 9
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
 RSpec/FilePath:
   Exclude:
     - 'spec/configuration_spec.rb'
@@ -45,8 +45,8 @@ RSpec/FilePath:
     - 'spec/message_type/aws_event_spec.rb'
     - 'spec/message_type/sns_message_spec.rb'
     - 'spec/queue_poller_spec.rb'
-    - 'spec/topic_publisher_spec.rb'
     - 'spec/rollbar_spec.rb'
+    - 'spec/topic_publisher_spec.rb'
     - 'spec/version_spec.rb'
 
 # Offense count: 14
@@ -70,3 +70,10 @@ RSpec/VerifiedDoubles:
 Style/GuardClause:
   Exclude:
     - 'lib/pheme/queue_poller.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/IfUnlessModifier:
+  Exclude:
+    - 'lib/pheme/queue_poller.rb'
+    - 'pheme.gemspec'

data/CHANGELOG.md

@@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 4.0.3 - 2021-03-08
+- Update documentation to include instructions on directly publishing to SQS
+
+## 4.0.2 - 2020-12-17
+- Bump local dev version, rubocop fixes, add backstage catalog file + sonarqube project settings
+
+## 4.0.1 - 2020-03-23
+### Fixes
+- Fixes 4.0.0. Instead of expecting message attributes in the message from `poll`, retrieves them from the right place.
+
+## 4.0.0 - 2020-03-18
+### Breaking Changes
+- Add the ability for SQS to receive message attributes
+- `handle` function of `QueuePoller` now takes a third parameter `message_attributes`
+- blocks or `MessageHandler`s passed to `QueuePoller` can use this `message_attributes`
+
+## 3.4.0 - 2020-03-17
+### Added
+- Add the ability to pass message attributes to SNS
+
 ## 3.3.0 - 2019-07-09
 ### Added
 - Add more friendly options for initializing queue pollers by either:

data/README.md

@@ -39,3 +39,9 @@ end
 See https://github.com/wealthsimple/pheme/tree/master/spec/support for example implementations of each class.
 
 TODO: write better usage instructions.
+
+### Handling SQS messages
+
+Pheme expects that the SQS messages it is handling will have first been published to an SNS topic
+before being sent to the SQS queue. This means if the service publishing messages is publishing them
+**directly** to the SQS queue, that service must nest the message payload underneath a `Message` property.

data/catalog-info.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: pheme
+  description: Ruby SNS publisher + SQS message handler
+  tags:
+    - ruby
+    - open-source
+spec:
+  type: library
+  lifecycle: production

data/lib/pheme/configuration.rb

@@ -19,8 +19,7 @@ module Pheme
     ATTRIBUTES = %i[sns_client sqs_client logger].freeze
     OPTIONAL_ATTRIBUTES = %i[rollbar].freeze
 
-    attr_accessor(*ATTRIBUTES)
-    attr_accessor(*OPTIONAL_ATTRIBUTES)
+    attr_accessor(*ATTRIBUTES, *OPTIONAL_ATTRIBUTES)
 
     def initialize
       @logger ||= Logger.new(STDOUT) # rubocop:disable Lint/DisjunctiveAssignmentInConstructor

data/lib/pheme/message_handler.rb

@@ -1,10 +1,11 @@
 module Pheme
   class MessageHandler
-    attr_reader :message, :metadata, :timestamp
+    attr_reader :message, :metadata, :message_attributes, :timestamp
 
-    def initialize(message:, metadata: {})
+    def initialize(message:, metadata: {}, message_attributes: {})
       @message = message
       @metadata = metadata
+      @message_attributes = message_attributes
     end
 
     def handle

data/lib/pheme/queue_poller.rb

@@ -1,6 +1,7 @@
 require_relative 'compression'
 
 module Pheme
+  # rubocop:disable Metrics/ClassLength
   class QueuePoller
     include Compression
 
@@ -51,6 +52,7 @@ module Pheme
       end
     end
 
+    # rubocop:disable Metrics/AbcSize
     def poll
       time_start = log_polling_start
       queue_poller.poll(poller_configuration) do |queue_message|
@@ -59,7 +61,8 @@ module Pheme
           begin
             content = parse_body(queue_message)
             metadata = parse_metadata(queue_message)
-            with_optional_connection_pool_block { handle(content, metadata) }
+            message_attributes = parse_message_attributes(queue_message)
+            with_optional_connection_pool_block { handle(content, metadata, message_attributes) }
             queue_poller.delete_message(queue_message)
             log_delete(queue_message)
             @messages_processed += 1
@@ -73,6 +76,7 @@ module Pheme
       end
       log_polling_end(time_start)
     end
+    # rubocop:enable Metrics/AbcSize
 
     # returns queue_message.body as hash,
     # stores and parses get_content to body[:content]
@@ -105,6 +109,16 @@ module Pheme
       { timestamp: message_body['Timestamp'], topic_arn: message_body['TopicArn'] }
     end
 
+    def parse_message_attributes(queue_message)
+      message_body = JSON.parse(queue_message.body)
+      message_attributes = {}
+      message_body['MessageAttributes']&.each do |key, value|
+        message_attributes[key.to_sym] = coerce_message_attribute(value)
+      end
+
+      message_attributes
+    end
+
     def get_metadata(message_body)
       message_body.except('Message', 'Records')
     end
@@ -123,11 +137,11 @@ module Pheme
       RecursiveOpenStruct.new({ wrapper: parsed_body }, recurse_over_arrays: true).wrapper
     end
 
-    def handle(message, metadata)
+    def handle(message, metadata, message_attributes)
       if @message_handler
-        @message_handler.new(message: message, metadata: metadata).handle
+        @message_handler.new(message: message, metadata: metadata, message_attributes: message_attributes).handle
       elsif @block_message_handler
-        @block_message_handler.call(message, metadata)
+        @block_message_handler.call(message, metadata, message_attributes)
       else
         raise NotImplementedError
       end
@@ -135,9 +149,21 @@ module Pheme
 
     private
 
-    def with_optional_connection_pool_block
+    def coerce_message_attribute(value)
+      case value['Type']
+      when 'Binary', 'String'
+        value['Value']
+      when 'Number', 'String.Array'
+        JSON.parse(value['Value'])
+      else
+        Pheme.logger.info("Unsupported custom data type")
+        value["Value"]
+      end
+    end
+
+    def with_optional_connection_pool_block(&block)
       if connection_pool_block
-        ActiveRecord::Base.connection_pool.with_connection { yield }
+        ActiveRecord::Base.connection_pool.with_connection(&block)
       else
         yield
       end
@@ -192,4 +218,5 @@ module Pheme
       }.to_json)
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/pheme/topic_publisher.rb

@@ -34,7 +34,7 @@ module Pheme
       raise NotImplementedError
     end
 
-    def publish(message, sns_client: Pheme.configuration.sns_client)
+    def publish(message, sns_client: Pheme.configuration.sns_client, message_attributes: nil)
       payload = {
         message: "#{self.class} publishing message to #{topic_arn}",
         body: message,
@@ -42,7 +42,8 @@ module Pheme
         topic_arn: topic_arn,
       }
       Pheme.logger.info(payload.to_json)
-      sns_client.publish(topic_arn: topic_arn, message: serialize(message))
+
+      sns_client.publish(topic_arn: topic_arn, message: serialize(message), message_attributes: message_attributes)
     end
 
     def serialize(message)

data/lib/pheme/version.rb

@@ -1,3 +1,3 @@
 module Pheme
-  VERSION = '3.3.3'.freeze
+  VERSION = '4.0.3'.freeze
 end

data/pheme.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
   s.add_dependency "smarter_csv", "~> 1"
 
   s.add_development_dependency 'bundler'
-  s.add_development_dependency 'coveralls'
+  s.add_development_dependency 'coveralls', "~> 0.8"
   s.add_development_dependency 'git'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rspec'

data/sonar-project.properties

@@ -0,0 +1,4 @@
+sonar.projectKey=pheme
+sonar.ruby.coverage.reportPaths=coverage/.resultset.json
+sonar.sources=lib
+sonar.tests=spec

data/spec/queue_poller_spec.rb

@@ -14,7 +14,7 @@ describe Pheme::QueuePoller do
   let(:message_id) { SecureRandom.uuid }
 
   context 'base poller' do
-    subject { described_class.new(queue_url: queue_url).handle(nil, nil) }
+    subject { described_class.new(queue_url: queue_url).handle(nil, nil, nil) }
 
     it 'does not implement handle' do
       expect { subject }.to raise_error(NotImplementedError)
@@ -90,13 +90,13 @@ describe Pheme::QueuePoller do
     context 'when handling messages' do
       context 'when doing it the old way, via the handle function' do
         it 'uses the handle function by default' do
-          expect { described_class.new(queue_url: queue_url).handle(nil, nil) }.to raise_error(NotImplementedError)
+          expect { described_class.new(queue_url: queue_url).handle(nil, nil, nil) }.to raise_error(NotImplementedError)
         end
       end
 
       context 'when given a message_handler as parameter' do
         it 'uses default when given nil' do
-          expect { described_class.new(queue_url: queue_url, message_handler: nil).handle(nil, nil) }.to raise_error(NotImplementedError)
+          expect { described_class.new(queue_url: queue_url, message_handler: nil).handle(nil, nil, nil) }.to raise_error(NotImplementedError)
         end
 
         it 'uses default when given invalid message_handler' do
@@ -106,9 +106,11 @@ describe Pheme::QueuePoller do
         it 'uses handler when given one' do
           mock_handler = double('MessageHandler')
           allow(mock_handler).to receive(:handle)
-          allow(ExampleMessageHandler).to receive(:new).with(message: 'message', metadata: 'metadata').and_return(mock_handler)
+          allow(ExampleMessageHandler).to receive(:new).
+            with(message: 'message', metadata: 'metadata', message_attributes: 'message_attributes').
+            and_return(mock_handler)
 
-          described_class.new(queue_url: queue_url, message_handler: ExampleMessageHandler).handle('message', 'metadata')
+          described_class.new(queue_url: queue_url, message_handler: ExampleMessageHandler).handle('message', 'metadata', 'message_attributes')
           expect(mock_handler).to have_received(:handle).once
         end
       end
@@ -117,12 +119,12 @@ describe Pheme::QueuePoller do
         it 'uses handler when given one' do
           mock_handler = spy('MessageHandler')
 
-          poller = described_class.new(queue_url: queue_url) do |message, metadata|
-            mock_handler.process(message, metadata)
+          poller = described_class.new(queue_url: queue_url) do |message, metadata, message_attributes|
+            mock_handler.process(message, metadata, message_attributes)
           end
-          poller.handle('message', 'metadata')
+          poller.handle('message', 'metadata', 'message_attributes')
 
-          expect(mock_handler).to have_received(:process).with('message', 'metadata').once
+          expect(mock_handler).to have_received(:process).with('message', 'metadata', 'message_attributes').once
         end
 
         it 'fails on invalid handler' do
@@ -134,6 +136,62 @@ describe Pheme::QueuePoller do
     end
   end
 
+  describe '#parse_message_attributes' do
+    subject { described_class.new(queue_url: queue_url).parse_message_attributes(queue_message) }
+
+    context 'when no message_attributes' do
+      it { is_expected.to eq({}) }
+    end
+
+    context 'when message attributes' do
+      let(:queue_message) do
+        OpenStruct.new(
+          body: {
+            Message: message,
+            MessageAttributes: {
+              key: { Type: data_type, Value: string_value },
+            },
+          }.to_json,
+          message_id: message_id,
+        )
+      end
+
+      context 'when data_type is String' do
+        let(:data_type) { 'String' }
+        let(:string_value) { 'some-string-value' }
+        let(:expected_value) { 'some-string-value' }
+
+        its([:key]) { is_expected.to eq(expected_value) }
+      end
+
+      context 'when data_type is Number' do
+        let(:data_type) { 'Number' }
+
+        context 'and it is an int' do
+          let(:string_value) { '1' }
+          let(:expected_value) { 1 }
+
+          its([:key]) { is_expected.to eq(expected_value) }
+        end
+
+        context 'and it is a float' do
+          let(:string_value) { '1.5' }
+          let(:expected_value) { 1.5 }
+
+          its([:key]) { is_expected.to eq(expected_value) }
+        end
+      end
+
+      context 'when data_type is String.array' do
+        let(:data_type) { 'String.Array' }
+        let(:string_value) { '["sdfsdf",1,2,null,true,false]' }
+        let(:expected_value) { ["sdfsdf", 1, 2, nil, true, false] }
+
+        its([:key]) { is_expected.to eq(expected_value) }
+      end
+    end
+  end
+
   describe "#parse_body" do
     subject { poller.parse_body(queue_message) }
 
@@ -305,6 +363,7 @@ describe Pheme::QueuePoller do
         expect(ExampleMessageHandler).to receive(:new).with(
           message: RecursiveOpenStruct.new(message),
           metadata: { timestamp: timestamp, topic_arn: topic_arn },
+          message_attributes: {},
         )
         subject.poll
       end
@@ -359,7 +418,7 @@ describe Pheme::QueuePoller do
 
       let(:queue_message) { OpenStruct.new(body: { 'Records' => records }.to_json) }
       let(:records) do
-        [{ 'eventVersion' => '2.0', 'eventSource': 'aws:s3' }]
+        [{ 'eventVersion' => '2.0', eventSource: 'aws:s3' }]
       end
 
       before do

data/spec/spec_helper.rb

@@ -35,4 +35,4 @@ def use_default_configuration!
   end
 end
 
-Dir["./spec/support/**/*.rb"].each { |f| require f }
+Dir["./spec/support/**/*.rb"].sort.each { |f| require f }

data/spec/support/example_queue_poller.rb

@@ -3,10 +3,10 @@ class ExampleQueuePoller < Pheme::QueuePoller
     super(queue_url: queue_url, **kwargs)
   end
 
-  def handle(message, metadata)
+  def handle(message, metadata, message_attributes)
     case message.status
     when 'complete', 'rejected'
-      ExampleMessageHandler.new(message: message, metadata: metadata).handle
+      ExampleMessageHandler.new(message: message, metadata: metadata, message_attributes: message_attributes).handle
     else
       raise ArgumentError, "Unknown message status: #{message.status}"
     end

data/spec/support/example_queue_poller_with_inlined_handler.rb

@@ -1,5 +1,6 @@
-Pheme::QueuePoller.new(queue_url: 'http://mock_url.test') do |message, metadata|
+Pheme::QueuePoller.new(queue_url: 'http://mock_url.test') do |message, metadata, message_attributes|
   # handle the message
   pp message
   pp metadata
+  pp message_attributes
 end

data/spec/topic_publisher_spec.rb

@@ -36,10 +36,12 @@ describe Pheme::TopicPublisher do
       expect(Pheme.configuration.sns_client).to receive(:publish).with({
         topic_arn: "arn:aws:sns:whatever",
         message: { id: "id-0", status: "complete" }.to_json,
+        message_attributes: nil,
       })
       expect(Pheme.configuration.sns_client).to receive(:publish).with({
         topic_arn: "arn:aws:sns:whatever",
         message: { id: "id-1", status: "complete" }.to_json,
+        message_attributes: nil,
       })
       subject.publish_events
     end
@@ -54,6 +56,7 @@ describe Pheme::TopicPublisher do
         expect(Pheme.configuration.sns_client).to receive(:publish).with({
           topic_arn: topic_arn,
           message: message,
+          message_attributes: nil,
         })
         subject.publish(message)
       end
@@ -65,6 +68,7 @@ describe Pheme::TopicPublisher do
           expect(sns_client).to receive(:publish).with({
             topic_arn: topic_arn,
             message: message,
+            message_attributes: nil,
           })
           subject.publish(message, sns_client: sns_client)
         end
@@ -89,6 +93,7 @@ describe Pheme::TopicPublisher do
             with({
               topic_arn: topic_arn,
               message: compressed_message,
+              message_attributes: nil,
             }),
         )
 

data/spec/version_spec.rb

@@ -4,7 +4,7 @@ describe Pheme do
   def get_version(git, branch = 'HEAD')
     git.grep('VERSION = ', 'lib/*/version.rb', { object: branch }).
       map { |_sha, matches| matches.first[1] }.
-      map(&method(:parse_version)).
+      map { |version_string| parse_version(version_string) }.
       reject(&:nil?).
       first
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pheme
 version: !ruby/object:Gem::Version
-  version: 3.3.3
+  version: 4.0.3
 platform: ruby
 authors:
 - Peter Graham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-07 00:00:00.000000000 Z
+date: 2021-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -98,16 +98,16 @@ dependencies:
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -240,6 +240,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- catalog-info.yaml
 - lib/pheme.rb
 - lib/pheme/compression.rb
 - lib/pheme/configuration.rb
@@ -252,6 +253,7 @@ files:
 - lib/pheme/topic_publisher.rb
 - lib/pheme/version.rb
 - pheme.gemspec
+- sonar-project.properties
 - spec/configuration_spec.rb
 - spec/logger_spec.rb
 - spec/message_handler_spec.rb