pwwka 0.16.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d33c45520c16c18b56efbfcefd70a49252e3bd00
-  data.tar.gz: dac6fe4120e1dc388e49adb123330b9f8545cbcb
+SHA256:
+  metadata.gz: e9a3dc1069389bc19bef4c75f90900dd4580a739fb96865edbaa06a829700ad8
+  data.tar.gz: 4463babd7364e8084ebe87bdb57cccc1fdcc1262a435d3cf9540bdd77ef6a58b
 SHA512:
-  metadata.gz: 906bc065b933a8bc13525b454f8d6e1259414272cfabd8585305d4bc793f25e01598d4413064e10742fa4a31e7dacfa963418e58975b77763940fba5d5d84519
-  data.tar.gz: e88311d37274c9fb54c14f516be1f2a333c302380aeb838785c544addc3e0e4543b4cec3749fb4eccd8441b20fc93480cb6696383855231d9e158d27f423670d
+  metadata.gz: ddbba8fb8943d7ab931abd2fd38bd8840c30acd43404c51772e4aa0fe3d416afb197aa61dbd2d047d5701669630ec39bbcd41780fcbf5a070c301cbdd2c828ea
+  data.tar.gz: 1cba823b9fdccb53cfcbd2bc90e0a909fb428cfd9a763f02696466ccd187f10c8df4d9c6de1c3e2d31e6a7aa3a73e3cfe6b80c17099c25ed43c986b9a6d47aa6

data/.circleci/config.yml

@@ -3,105 +3,95 @@
 ---
 version: 2
 jobs:
-  ruby-2.5.0-rails-5.1:
+  ruby-2.5.1-rails-5.2:
     docker:
-    - image: circleci/ruby:2.5.0
+    - image: circleci/ruby:2.5.1
       environment:
-        BUNDLE_GEMFILE: Gemfile.rails-5.1
+        BUNDLE_GEMFILE: Gemfile.rails-5.2
+    - image: redis:2.8.12
+    - image: rabbitmq:3.5.6
     working_directory: "~/pwwka"
     steps:
     - checkout
-    - run: bundle install
+    - run: bundle install --full-index
     - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
         --format=doc
     - run:
-        name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
-        when: on_fail
-    - store_test_results:
-        path: "/tmp/test-results"
-  ruby-2.4.3-rails-5.1:
-    docker:
-    - image: circleci/ruby:2.4.3
-      environment:
-        BUNDLE_GEMFILE: Gemfile.rails-5.1
-    working_directory: "~/pwwka"
-    steps:
-    - checkout
-    - run: bundle install
-    - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
-        --format=doc
+        name: Run Additional CI Steps
+        command: if [ -e bin/additional-ci-steps ]; then bin/additional-ci-steps;
+          fi
     - run:
         name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
+        command: 'bundle exec y-notify #eng-platform'
         when: on_fail
     - store_test_results:
         path: "/tmp/test-results"
-  ruby-2.5.0-rails-5.0:
+  ruby-2.4.4-rails-5.2:
     docker:
-    - image: circleci/ruby:2.5.0
+    - image: circleci/ruby:2.4.4
       environment:
-        BUNDLE_GEMFILE: Gemfile.rails-5.0
+        BUNDLE_GEMFILE: Gemfile.rails-5.2
+    - image: redis:2.8.12
+    - image: rabbitmq:3.5.6
     working_directory: "~/pwwka"
     steps:
     - checkout
-    - run: bundle install
+    - run: bundle install --full-index
     - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
         --format=doc
     - run:
-        name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
-        when: on_fail
-    - store_test_results:
-        path: "/tmp/test-results"
-  ruby-2.4.3-rails-5.0:
-    docker:
-    - image: circleci/ruby:2.4.3
-      environment:
-        BUNDLE_GEMFILE: Gemfile.rails-5.0
-    working_directory: "~/pwwka"
-    steps:
-    - checkout
-    - run: bundle install
-    - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
-        --format=doc
+        name: Run Additional CI Steps
+        command: if [ -e bin/additional-ci-steps ]; then bin/additional-ci-steps;
+          fi
     - run:
         name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
+        command: 'bundle exec y-notify #eng-platform'
         when: on_fail
     - store_test_results:
         path: "/tmp/test-results"
-  ruby-2.5.0-rails-4.2:
+  ruby-2.5.1-rails-5.1:
     docker:
-    - image: circleci/ruby:2.5.0
+    - image: circleci/ruby:2.5.1
       environment:
-        BUNDLE_GEMFILE: Gemfile.rails-4.2
+        BUNDLE_GEMFILE: Gemfile.rails-5.1
+    - image: redis:2.8.12
+    - image: rabbitmq:3.5.6
     working_directory: "~/pwwka"
     steps:
     - checkout
-    - run: bundle install
+    - run: bundle install --full-index
     - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
         --format=doc
+    - run:
+        name: Run Additional CI Steps
+        command: if [ -e bin/additional-ci-steps ]; then bin/additional-ci-steps;
+          fi
     - run:
         name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
+        command: 'bundle exec y-notify #eng-platform'
         when: on_fail
     - store_test_results:
         path: "/tmp/test-results"
-  ruby-2.4.3-rails-4.2:
+  ruby-2.4.4-rails-5.1:
     docker:
-    - image: circleci/ruby:2.4.3
+    - image: circleci/ruby:2.4.4
       environment:
-        BUNDLE_GEMFILE: Gemfile.rails-4.2
+        BUNDLE_GEMFILE: Gemfile.rails-5.1
+    - image: redis:2.8.12
+    - image: rabbitmq:3.5.6
     working_directory: "~/pwwka"
     steps:
     - checkout
-    - run: bundle install
+    - run: bundle install --full-index
     - run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
         --format=doc
+    - run:
+        name: Run Additional CI Steps
+        command: if [ -e bin/additional-ci-steps ]; then bin/additional-ci-steps;
+          fi
     - run:
         name: Notify Pager Duty
-        command: bundle exec y-notify eng-platform
+        command: 'bundle exec y-notify #eng-platform'
         when: on_fail
     - store_test_results:
         path: "/tmp/test-results"
@@ -109,17 +99,13 @@ workflows:
   version: 2
   on-commit:
     jobs:
-    - ruby-2.5.0-rails-5.1:
+    - ruby-2.5.1-rails-5.2:
         context: org-global
-    - ruby-2.4.3-rails-5.1:
+    - ruby-2.4.4-rails-5.2:
         context: org-global
-    - ruby-2.5.0-rails-5.0:
+    - ruby-2.5.1-rails-5.1:
         context: org-global
-    - ruby-2.4.3-rails-5.0:
-        context: org-global
-    - ruby-2.5.0-rails-4.2:
-        context: org-global
-    - ruby-2.4.3-rails-4.2:
+    - ruby-2.4.4-rails-5.1:
         context: org-global
   scheduled:
     triggers:
@@ -130,15 +116,11 @@ workflows:
             only:
             - master
     jobs:
-    - ruby-2.5.0-rails-5.1:
-        context: org-global
-    - ruby-2.4.3-rails-5.1:
-        context: org-global
-    - ruby-2.5.0-rails-5.0:
+    - ruby-2.5.1-rails-5.2:
         context: org-global
-    - ruby-2.4.3-rails-5.0:
+    - ruby-2.4.4-rails-5.2:
         context: org-global
-    - ruby-2.5.0-rails-4.2:
+    - ruby-2.5.1-rails-5.1:
         context: org-global
-    - ruby-2.4.3-rails-4.2:
+    - ruby-2.4.4-rails-5.1:
         context: org-global

data/.ruby-version

@@ -1 +1 @@
-2.4.1
+2.5.1

data/Gemfile.rails-5.2

@@ -0,0 +1,8 @@
+# DO NOT MODIFY - this is managed by Git Reduce in goro
+#
+source 'https://gem.fury.io/me/'
+source 'https://www.rubygems.org'
+
+gemspec
+
+gem 'activesupport', '~> 5.2.0'

data/README.md

@@ -19,7 +19,7 @@ In your `Gemfile`:
 gem 'pwwka'
 ```
 
-(of course, you can always run `gem install pwwka` to install it without Bundler)
+or `gem install pwwka` if you aren't using a `Gemfile`.
 
 To run applications locally, you will need Rabbit installed.  The [installation guide](https://www.rabbitmq.com/download.html) is a great
 place to start.  This repo includes a `docker-compose.yml` file which will run Rabbit inside a Docker container.  It's used by the tests,
@@ -260,6 +260,26 @@ private
 end
 ```
 
+#### Payload Parsing
+
+By default, payloads are assumed to be JSON and are parsed before being sent to your `handle!` method (meaning: that method is
+given a `HashWithIndifferentAccess` of your payload).
+
+If you don't want this, for example if you are using XML or some other format, you can turn this feature off in your
+initializers:
+
+```ruby
+# config/initialisers/pwwka.rb
+require 'pwwka'
+
+Pwwka.configure do |config|
+  config.receive_raw_payload = true
+  # any other settings you might have
+end
+```
+
+In this case, your handler gets whatever Bunny returns, so you are on your own.
+
 #### Errors From Your Handler
 
 By default, handlers will log and ignore garbled payloads (basically payloads that fail to be parsed as JSON).  All other errors

data/build-matrix.json

@@ -1,4 +1,8 @@
 {
   "build_matrix": {
+    "additional_docker_images": [
+      "redis:2.8.12",
+      "rabbitmq:3.5.6"
+    ]
   }
 }

data/lib/pwwka.rb

@@ -11,7 +11,7 @@ module Pwwka
 
     def environment
       ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-    end   
+    end
   end
 
 end

data/lib/pwwka/configuration.rb

@@ -31,6 +31,7 @@ module Pwwka
       @keep_alive_on_handler_klass_exceptions = false
       @async_job_klass = Pwwka::SendMessageAsyncJob
       @default_prefetch = nil
+      @receive_raw_payload = false
     end
 
     def keep_alive_on_handler_klass_exceptions?
@@ -107,5 +108,34 @@ module Pwwka
     def default_prefetch=(val)
       @default_prefetch = val.nil? ? val : val.to_i
     end
+
+    # Set this if you don't want the payload parsed.  This can be useful is you are expecting a lot of malformed
+    # JSON or if you aren't using JSON at all.  Note that currently, setting this to true will prevent all
+    # payloads from being logged
+    def receive_raw_payload=(val)
+      @receive_raw_payload = val
+      @payload_parser = nil
+    end
+
+    # Returns a proc that, when called with the payload, parses it according to the configuration.
+    #
+    # By default, this will assume the payload is JSON, parse it, and return a HashWithIndifferentAccess.
+    def payload_parser
+      @payload_parser ||= if @receive_raw_payload
+                            ->(payload) { payload }
+                          else
+                            ->(payload) {
+                              ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(payload))
+                            }
+                          end
+    end
+
+    # True if we should omit the payload from the log
+    #
+    # ::level_of_message_with_payload the level of the message about to be logged
+    def omit_payload_from_log?(level_of_message_with_payload)
+      return true if @receive_raw_payload
+      Pwwka::Logging::LEVELS[Pwwka.configuration.payload_logging.to_sym] > Pwwka::Logging::LEVELS[level_of_message_with_payload.to_sym]
+    end
   end
 end

data/lib/pwwka/logging.rb

@@ -18,11 +18,11 @@ module Pwwka
     def logf(format,params)
       level = params.delete(:at) || Pwwka.configuration.log_level
       params[:payload] = params["payload"] if params["payload"]
-      params[:payload] = "[omitted]" if params[:payload] && LEVELS[Pwwka.configuration.payload_logging.to_sym] > LEVELS[level.to_sym]
+      if Pwwka.configuration.omit_payload_from_log?(level)
+        params[:payload] = "[omitted]" if params[:payload]
+      end
       message = format % params
       logger.send(level,message)
     end
-
-
   end
 end

data/lib/pwwka/receiver.rb

@@ -17,14 +17,18 @@ module Pwwka
       @topic_exchange    = @channel_connector.topic_exchange
     end
 
-    def self.subscribe(handler_klass, queue_name, routing_key: "#.#", block: true, prefetch: Pwwka.configuration.default_prefetch)
+    def self.subscribe(handler_klass, queue_name,
+                       routing_key: "#.#",
+                       block: true,
+                       prefetch: Pwwka.configuration.default_prefetch,
+                       payload_parser: Pwwka.configuration.payload_parser)
       raise "#{handler_klass.name} must respond to `handle!`" unless handler_klass.respond_to?(:handle!)
       receiver  = new(queue_name, routing_key, prefetch: prefetch)
       begin
         info "Receiving on #{queue_name}"
         receiver.topic_queue.subscribe(manual_ack: true, block: block) do |delivery_info, properties, payload|
           begin
-            payload = ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(payload))
+            payload = payload_parser.(payload)
             handler_klass.handle!(delivery_info, properties, payload)
             receiver.ack(delivery_info.delivery_tag)
             logf "Processed Message on %{queue_name} -> %{payload}, %{routing_key}", queue_name: queue_name, payload: payload, routing_key: delivery_info.routing_key
@@ -78,6 +82,5 @@ module Pwwka
       topic_exchange.delete
       channel_connector.connection_close
     end
-
   end
 end

data/lib/pwwka/version.rb

@@ -1,3 +1,3 @@
 module Pwwka
-  VERSION = '0.16.1'
+  VERSION = '0.17.0'
 end

data/spec/legacy/receiver_spec.rb

@@ -157,6 +157,36 @@ describe Pwwka::Receiver, :legacy do
 
     end
 
+    describe "#prepare_payload" do
+
+      context "when payload is JSON" do
+
+        it "returns a hash with indifferent access" do
+          payload = { something: "interesting" }.to_json
+
+          result = @receiver.prepare_payload(payload)
+
+          expect(result).to be_a(Hash)
+          expect(result).to have_key("something")
+          expect(result).to have_key(:something)
+        end
+
+      end
+
+      context "when the paylad is not JSON" do
+
+        it "defers the handling to a handler down the chain" do
+          payload = "<>"
+
+          result = @receiver.prepare_payload(payload)
+
+          expect(result).to eq(payload)
+        end
+
+      end
+
+    end
+
   end
 
 end

data/spec/support/test_configuration.rb

@@ -8,8 +8,13 @@ class TestConfiguration
   def initialize(docker_compose_file)
     yaml = YAML.load_file(docker_compose_file)
 
-    @resque_redis_port = (ENV["PWWKA_RESQUE_REDIS_PORT"] || yaml["services"]["resque"]["ports"].first.split(/:/)[0]).to_i
-    @rabbit_port       = (ENV["PWWKA_RABBIT_PORT"]       || yaml["services"]["rabbit"]["ports"].first.split(/:/)[0]).to_i
+    if ENV["CI"] == 'true'
+      @resque_redis_port = 6379
+      @rabbit_port       = 5672
+    else
+      @resque_redis_port = (ENV["PWWKA_RESQUE_REDIS_PORT"] || yaml["services"]["resque"]["ports"].first.split(/:/)[0]).to_i
+      @rabbit_port       = (ENV["PWWKA_RABBIT_PORT"]       || yaml["services"]["rabbit"]["ports"].first.split(/:/)[0]).to_i
+    end
   end
 
   def check_services

data/spec/unit/configuration_spec.rb

@@ -20,6 +20,28 @@ describe Pwwka::Configuration do
       expect(configuration.topic_exchange_name).to eq("pwwka.topics.production")
     end
   end
+
+  describe "#payload_parser" do
+    it "parses JSON by default" do
+      payload = { foo: { bar: 42 } }
+      expect(described_class.new.payload_parser.(payload.to_json)).to eq({ "foo" => { "bar" => 42 } })
+    end
+
+    it "setting receive_raw_payload to true pases through the raw payload" do
+      configuration.receive_raw_payload = true
+      payload = "<h1>This is some <blink>XML</blink></h1>"
+      expect(configuration.payload_parser.(payload)).to eq(payload)
+    end
+
+    it "setting receive_raw_payload to true then false restores the JSON-parsing" do
+      configuration.receive_raw_payload = true
+      payload = { foo: { bar: 42 } }
+      expect(configuration.payload_parser.(payload.to_json)).to eq(payload.to_json)
+      configuration.receive_raw_payload = false
+      expect(configuration.payload_parser.(payload.to_json)).to eq({ "foo" => { "bar" => 42 } })
+    end
+  end
+
   describe "#delayed_exchange_name" do
     it "is based on the Pwwka.environment" do
       expect(configuration.delayed_exchange_name).to eq("pwwka.delayed.production")

data/spec/unit/logging_spec.rb

@@ -68,6 +68,7 @@ describe Pwwka::Logging do
       ].each do |name|
         it "will strip payload (given as a #{name.class}) if configured" do
           Pwwka.configuration.payload_logging = :error
+          Pwwka.configuration.receive_raw_payload = false
           ForLogging.logf("This is the payload: %{payload}", name => { foo: "bar" })
           ForLogging.logf("This is also the payload: %{payload}", name => { foo: "bar" }, at: :error)
           expect(logger).to have_received(:info).with("This is the payload: [omitted]")
@@ -76,6 +77,16 @@ describe Pwwka::Logging do
 
         it "will strip payload (given as a #{name.class}) of errors, too" do
           Pwwka.configuration.payload_logging = :fatal
+          Pwwka.configuration.receive_raw_payload = false
+          ForLogging.logf("This is the payload: %{payload}", name => { foo: "bar" })
+          ForLogging.logf("This is also the payload: %{payload}", name => { foo: "bar" }, at: :error)
+          expect(logger).to have_received(:info).with("This is the payload: [omitted]")
+          expect(logger).to have_received(:error).with("This is also the payload: [omitted]")
+        end
+
+        it "will strip payload (given as a #{name.class}) if we AREN'T parsing payloads" do
+          Pwwka.configuration.payload_logging = :info
+          Pwwka.configuration.receive_raw_payload = true
           ForLogging.logf("This is the payload: %{payload}", name => { foo: "bar" })
           ForLogging.logf("This is also the payload: %{payload}", name => { foo: "bar" }, at: :error)
           expect(logger).to have_received(:info).with("This is the payload: [omitted]")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pwwka
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.17.0
 platform: ruby
 authors:
 - Stitch Fix Engineering
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-11 00:00:00.000000000 Z
+date: 2018-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -213,6 +213,7 @@ files:
 - Gemfile.rails-4.2
 - Gemfile.rails-5.0
 - Gemfile.rails-5.1
+- Gemfile.rails-5.2
 - LICENSE
 - README.md
 - Rakefile
@@ -286,7 +287,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Send and receive messages via RabbitMQ