messaging-stash 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 20f5e5944deef86579418b822f50db49ac4aafb5165681c80619cf581c85bcc9
+  data.tar.gz: 86e865833f15ec015d00942a1b39d43935c17140cf919a33d3b31700fe0b79bc
+SHA512:
+  metadata.gz: d5a84acaa60383dfff50b20fe2c500f2264b05410e3b40ee2624ecb2bae407661557781bad3aa4b3951b8dfb123a23c013d2c7a004ecd778d6cb547f1ded0e70
+  data.tar.gz: db50c87e2fb21c828a671510200b32089c2fbaa71c897c95d5ac98a9df3e0a2b3e1424910149b754d7a61edef7fdaba3ffb7f5c423254e83975d9ccb28e0b63b

data/.circleci/config.yml

@@ -0,0 +1,104 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.5.1
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+
+            bundle exec rspec --format progress \
+                            $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results
+
+  build-2.4.1:
+    docker:
+      - image: circleci/ruby:2.4.2
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+
+            bundle exec rspec --format progress \
+                            $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results
+
+workflows:
+  version: 2
+  build-multiple-targets:
+    jobs:
+      - build
+      - build-2.4.1

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in messaging-stash.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    messaging-stash (0.1.0)
+      phobos (>= 1.7.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    concurrent-ruby (1.0.5)
+    concurrent-ruby-ext (1.0.5)
+      concurrent-ruby (= 1.0.5)
+    diff-lcs (1.3)
+    exponential-backoff (0.0.2)
+    i18n (1.0.0)
+      concurrent-ruby (~> 1.0)
+    little-plugger (1.1.4)
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    minitest (5.11.3)
+    multi_json (1.13.1)
+    phobos (1.7.1)
+      activesupport (>= 3.0.0)
+      concurrent-ruby (>= 1.0.2)
+      concurrent-ruby-ext (>= 1.0.2)
+      exponential-backoff
+      logging
+      ruby-kafka (>= 0.3.14)
+      thor
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.5)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  messaging-stash!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -0,0 +1,35 @@
+# Messaging::Stash
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/messaging/stash`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'messaging-stash'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install messaging-stash
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/messaging-stash.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "messaging/stash"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/messaging/kafka/message_logger.rb

@@ -0,0 +1,43 @@
+module Messaging
+  module Kafka
+    module MessageLogger
+      class << self
+      
+        def message_published(topic, payload)
+          logger.info("Message Published. Topic: #{topic}, Payload: #{payload}")
+        end
+
+        def message_received(payload, metadata)
+          logger.info("Message Received. #{message_processed_or_received_string(payload, metadata)}")
+        end
+
+        def message_processed(payload, metadata)
+          logger.info("Message Processed. #{message_processed_or_received_string(payload, metadata)}")
+        end
+
+        def message_publish_failed(message, exception)
+          logger.error("Message Publish Error. Topic: #{message.topic}, Body: #{message.body_json}, Exception Class: #{exception.class} Exception: #{exception}")
+        end
+
+        def message_publish_list_failed(messages, exception)
+          logger.error("Message Publish List Error. Messages: #{messages}, Exception Class: #{exception.class}, Exception: #{exception}")
+        end
+
+        def message_buffer_flushed_on_shutdown
+          logger.info("Published any pending messages before shutdown.")
+        end
+
+        def logger
+          @logger ||= Logger.new(STDOUT)
+        end
+
+        private
+
+        def message_processed_or_received_string(payload, metadata)
+          "#{metadata[:handler]}. payload: #{payload}, metadata: #{metadata}"
+        end
+
+      end
+    end 
+  end
+end

data/lib/messaging/kafka/message_receiver.rb

@@ -0,0 +1,41 @@
+require 'json'
+module Messaging
+  module Kafka
+    class MessageReceiver < Messaging::Messages::MessageReceiver
+      include Phobos::Handler
+      include Phobos::Producer
+
+      ERROR_THRESHOLD = 5
+
+      def consume(payload, metadata)
+        begin
+          # metadata comes through as a hash, no need to parse
+          payload_hash =  JSON.parse(payload)
+          Messaging::Kafka::MessageLogger.message_received(payload_hash, metadata)
+          process(payload_hash, metadata)
+          Messaging::Kafka::MessageLogger.message_processed(payload_hash, metadata)
+        rescue JSON::ParserError => e
+          # A json parse error cannot be recovered from. Log the error and the body.
+          self.class.logger.error("MessageReceiver JSON::ParserError. payload: #{payload}, error: #{e}")
+        rescue => e
+          if metadata[:retry_count] > ERROR_THRESHOLD
+            # move the message to the poison topic if the threshold is reached.
+            # message = Messages::Events::MessagePoisoned.new(payload: payload_hash, listener: metadata[:handler])
+            # Messaging::Kafka::MessageSender.publish(message)
+            self.class.logger.error('Message poisoned')
+          else
+            raise
+          end
+        end
+      end
+
+      def process(payload, metadata)
+        raise NotImplementedError
+      end
+
+      def self.logger
+        @logger ||= Logger.new(STDOUT)
+      end
+    end
+  end
+end

data/lib/messaging/kafka/message_sender.rb

@@ -0,0 +1,92 @@
+require 'phobos'
+
+module Messaging
+  module Kafka
+    class MessageSender < Messaging::Messages::MessageSender
+      include Phobos::Producer
+      class << self
+        
+        # .publish will attempt to publish a single message synchronously. 
+        # A publish failure will not retry.
+        # Use sparingly. 
+        def publish(message)
+          return unless publish_message?
+          begin
+            producer.publish(message.topic, message.body_json)
+          rescue => e
+            Messaging::Kafka::MessageLogger.message_publish_failed(message, e)
+          end
+        end
+
+        # .publish_list will attempt to publish a single message synchronously. 
+        # A publish_list failure will not retry.
+        # Use sparingly.
+        def publish_list(messages)
+          return unless publish_message?
+          begin
+            message_list = build_messages(messages)
+            producer.publish_list(message_list)
+          rescue => e
+            Messaging::Kafka::MessageLogger.message_publish_list_failed(messages, e)
+          end
+        end
+
+        # .async_publish will buffer a message to be sent according to the asynchronous delivery configuration. 
+        # This method will return immediately.
+        def async_publish(message)
+          return unless publish_message?
+          async_publish_list([message])
+        end
+
+        # .async_publish_list will buffer a message to be sent according to the asynchronous delivery configuration. 
+        # This method will return immediately.
+        def async_publish_list(messages)
+          return unless publish_message?
+          message_list = build_messages(messages)
+          init_async_producer
+          producer.async_publish_list(message_list)
+        end
+
+        # Shutdown and publish any buffered messages to Kafka. 
+        # This should be called before the application closes.
+        def async_producer_shutdown
+          async_producers.each do |async_producer|
+            async_producer&.deliver_messages
+            async_producer&.shutdown
+          end
+          Messaging::Kafka::MessageLogger.message_buffer_flushed_on_shutdown
+        end
+
+        private 
+        
+        # Force Phobos to create a new asynchronous producer if it doesn't already exist and add it to our list.
+        # Phobos will maintain a producer for each thread publishing messages.
+        def init_async_producer
+          async_producer = producer.async_producer || producer.create_async_producer
+          @async_producers << async_producer unless async_producers.include?(async_producer)
+        end
+
+        def build_messages(messages)
+          messages.map do |message|
+            {
+              topic: message.topic,
+              payload: message.body_json,
+              key: nil
+            }
+          end
+        end
+
+        def publish_message?
+          true
+          # Flipper.enabled?(:publish_messages_to_kafka)
+        end
+
+        # A list of Phobos created asynchronous producers. 
+        # This is an array of Kafka::AsyncProducer from the ruby-kafka gem.
+        def async_producers
+          @async_producers ||= []
+        end
+      end
+    end
+  end
+end

data/lib/messaging/messages/message.rb

@@ -0,0 +1,40 @@
+module Messaging
+  module Messages
+    class Message
+
+      def self.publish(options = {})
+        message = self.new(options)
+        Messaging::Kafka::MessageSender.publish(message)
+      end
+
+      def self.async_publish(options = {})
+        message = self.new(options)
+        Messaging::Kafka::MessageSender.async_publish(message)
+      end
+
+      def initialize(options = {})
+        ;
+      end
+
+      def topic
+        ENV.fetch('KAFKA_PREFIX', '') + topic_name
+      end
+
+      def body_json
+        body.merge({ message_id: message_identifier }).to_json
+      end
+
+      def message_identifier
+        SecureRandom.uuid
+      end
+
+      def topic_name
+        raise NotImplementedError
+      end
+
+      def body
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/messaging/messages/message_receiver.rb

@@ -0,0 +1,10 @@
+module Messaging
+  module Messages
+    class MessageReceiver
+
+      def consume(payload, metadata)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/messaging/messages/message_sender.rb

@@ -0,0 +1,22 @@
+module Messaging
+  module Messages
+    class MessageSender
+
+      def self.publish(message)
+        raise NotImplementedError
+      end
+
+      def self.publish_list(messages)
+        raise NotImplementedError
+      end
+
+      def self.async_publish(message)
+        raise NotImplementedError
+      end
+
+      def self.async_publish_list(messages)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/messaging/stash.rb

@@ -0,0 +1,15 @@
+require "messaging/stash/version"
+require "messaging/messages/message_sender"
+require "messaging/messages/message_receiver"
+require "messaging/messages/message"
+require "messaging/kafka/message_logger"
+require "messaging/kafka/message_sender"
+require "messaging/kafka/message_receiver"
+require "phobos"
+require "logger"
+
+module Messaging
+  module Stash
+    # Your code goes here...
+  end
+end

data/lib/messaging/stash/version.rb

@@ -0,0 +1,5 @@
+module Messaging
+  module Stash
+    VERSION = "0.1.0"
+  end
+end

data/messaging-stash.gemspec

@@ -0,0 +1,43 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "messaging/stash/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "messaging-stash"
+  spec.version       = Messaging::Stash::VERSION
+  spec.authors       = [
+    "Alex Kaufman",
+    "Nicholas Balsamo"
+  ]
+  spec.email         = [
+    "alex@stashinvest.com",
+    "nbalsambo@stashinvest.com"
+  ]
+
+  spec.summary       = %q{Phobos client for stash}
+  spec.description   = %q{This contains all the code for using phobos in a ruby project}
+  spec.homepage      = "https://github.com/stashinvest"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency "phobos", ">= 1.7.1"
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: messaging-stash
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alex Kaufman
+- Nicholas Balsamo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: phobos
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.1
+description: This contains all the code for using phobos in a ruby project
+email:
+- alex@stashinvest.com
+- nbalsambo@stashinvest.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/messaging/kafka/message_logger.rb
+- lib/messaging/kafka/message_receiver.rb
+- lib/messaging/kafka/message_sender.rb
+- lib/messaging/messages/message.rb
+- lib/messaging/messages/message_receiver.rb
+- lib/messaging/messages/message_sender.rb
+- lib/messaging/stash.rb
+- lib/messaging/stash/version.rb
+- messaging-stash.gemspec
+homepage: https://github.com/stashinvest
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Phobos client for stash
+test_files: []