pheme 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bb62b021efbad7c81eabfe53f32641b051ab355
-  data.tar.gz: b45731fc261b7b7f5505e906f1d720cb99cc7259
+  metadata.gz: 0d342e13ae80846b1d4ac3949eb1e94d7f930ec4
+  data.tar.gz: 074d2404d1a56ba15250c45537aa08c3b6f295cc
 SHA512:
-  metadata.gz: f3bbbff1509ad02bd38216a4748a7c74cf11deb751b394644aced095789e16e698768163f29d7adeab4577ddfcf1d9081d9b84b95917372dec51b6653862a6c6
-  data.tar.gz: f1f5e2765cea3ef16e838c191a37c5f224f05d926fcb4ee18565ecc91a994857cc31e2bb6a625f2882a524ad0118cb2ebc3371c39cac859f78eecc09a4dd506b
+  metadata.gz: f722c25ea54c1d75906e23fa13232f8d071863f42c41c0f0b06263ead813ce4166935068d477ff399a7e1803f9b473fcf9d3228bec84f9ad1659989e8b02ad40
+  data.tar.gz: e5f3784d480fc108c0861e0705e74c554fb05eb6759a7549e002dc98c1fb38ff58649b7b03af70477fc8007dd6306a5b62db7653c8bca710d298cb83aa8549fe

data/lib/pheme/compression.rb

@@ -0,0 +1,27 @@
+require 'base64'
+require 'zlib'
+
+module Pheme
+  module Compression
+    def compress(body)
+      gz = Zlib::GzipWriter.new(StringIO.new)
+      gz << body
+      Base64.encode64(gz.close.string)
+    end
+
+    def decompress(body)
+      return Zlib::GzipReader.new(StringIO.new(Base64.decode64(body))).read if gzip?(body)
+      body
+    end
+
+    private
+
+    # https://tools.ietf.org/html/rfc1952#page-6
+    GZIP_MAGIC_NUMBER = "\037\213".unpack('n').freeze
+
+    def gzip?(body)
+      # Decode the first 4 bytes to compare with magic number
+      Base64.decode64(body[0..4]).unpack('n') == GZIP_MAGIC_NUMBER
+    end
+  end
+end

data/lib/pheme/queue_poller.rb

@@ -1,5 +1,9 @@
+require_relative 'compression'
+
 module Pheme
   class QueuePoller
+    include Compression
+
     attr_accessor :queue_url, :queue_poller, :connection_pool_block, :format, :max_messages, :poller_configuration
 
     def initialize(queue_url:, connection_pool_block: false, max_messages: nil, format: :json, poller_configuration: {}, sqs_client: nil)
@@ -84,7 +88,7 @@ module Pheme
     end
 
     def get_content(body)
-      body['Message']
+      decompress(body['Message'])
     end
 
     def parse_csv(message_contents)

data/lib/pheme/topic_publisher.rb

@@ -1,5 +1,20 @@
+require_relative 'compression'
+
 module Pheme
   class TopicPublisher
+    include Compression
+
+    #
+    # Constant with message size limit.
+    # The message size also includes some metadata: 'name' and 'type'.
+    # We give ourselves a buffer for this metadata.
+    #
+    # Source: https://docs.aws.amazon.com/sns/latest/dg/SNSMessageAttributes.html#SNSMessageAttributesNTV
+    #
+    SNS_SIZE_LIMIT = 256.kilobytes
+    EXPECTED_METADATA_SIZE = 1.kilobyte
+    MESSAGE_SIZE_LIMIT = SNS_SIZE_LIMIT - EXPECTED_METADATA_SIZE
+
     attr_accessor :topic_arn
 
     def initialize(topic_arn:)
@@ -23,8 +38,11 @@ module Pheme
     end
 
     def serialize(message)
-      return message if message.is_a? String
-      message.to_json
+      message = message.to_json unless message.is_a? String
+
+      return compress(message) if message.bytesize > MESSAGE_SIZE_LIMIT
+
+      message
     end
   end
 end

data/lib/pheme/version.rb

@@ -1,3 +1,3 @@
 module Pheme
-  VERSION = "1.0.0".freeze
+  VERSION = "1.1.0".freeze
 end

data/spec/queue_poller_spec.rb

@@ -90,6 +90,17 @@ describe Pheme::QueuePoller do
         expect(subject.first.first.test).to eq('test')
       end
     end
+
+    context "with compressed body" do
+      let(:format) { :json }
+      let(:message) do
+        gz = Zlib::GzipWriter.new(StringIO.new)
+        gz << { test: 'test' }.to_json
+        Base64.encode64(gz.close.string)
+      end
+
+      its([:test]) { is_expected.to eq('test') }
+    end
   end
 
   describe "#poll" do

data/spec/topic_publisher_spec.rb

@@ -44,5 +44,29 @@ describe Pheme::TopicPublisher do
         subject.publish(message)
       end
     end
+
+    context 'with message too large' do
+      let(:topic_arn) { "arn:aws:sns:anything" }
+      let(:message) { 'x' * (described_class::MESSAGE_SIZE_LIMIT + 1) }
+
+      subject { Pheme::TopicPublisher.new(topic_arn: topic_arn).publish(message) }
+
+      let(:compressed_message) do
+        gz = Zlib::GzipWriter.new(StringIO.new)
+        gz << message
+        Base64.encode64(gz.close.string)
+      end
+
+      it "publishes gzipped, base64 encoded message" do
+        expect(Pheme.configuration.sns_client).to(
+          receive(:publish)
+            .with({
+                    topic_arn: topic_arn,
+                    message: compressed_message,
+                  }))
+
+        subject
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pheme
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Peter Graham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -157,6 +157,7 @@ files:
 - README.md
 - circle.yml
 - lib/pheme.rb
+- lib/pheme/compression.rb
 - lib/pheme/configuration.rb
 - lib/pheme/logger.rb
 - lib/pheme/message_handler.rb
@@ -199,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Ruby SNS publisher + SQS poller & message handler