honest_pubsub 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc6a19fcd7590a3594ce7dce30c525ffad8d862e
+  data.tar.gz: 3904cd4bde4b1498a993d868a0fc7c8b8bd0cec6
+SHA512:
+  metadata.gz: 07e917724a4763ceea12384d38ecd153fbc22d9055b648eb6ead02832512491394b7d1ecb4690b66c670cf0afd17a907dba57ccb9fcdc6981ef662621c9565ad
+  data.tar.gz: b28f6207461e9c0f0b09d31fda4c2819fb92cdab523f0e96294198a27e2d41895ea63d0f90310c5d4fa1907d54f6b15b194ddca0a51c5ae217cd64955d0b07ea

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+.project
+.ruby-gemset
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rspec
+.idea
+test_pubsub.log

data/.ruby-version

@@ -0,0 +1 @@
+2.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in honest_pubsub.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Thanh Lim
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# HonestPubsub
+
+Simple Publishers and subscribers for Ruby using RabbitMQ
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'honest_pubsub'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install honest_pubsub
+
+You also need to install RabbitMQ
+
+    $ brew install rabbitmq
+
+## Usage
+
+There are two sides to this gem, publishers and subscribers
+
+### Publishers
+
+Publishing to a message is super simple
+
+    HonestPubsub.publish('user.created', { id: 3, name: 'Foo Bar', email: 'foobar@email.com')
+
+Additionally you can setup a context that is passed down to the subscribers. This context can be set up in a `before_filter`
+
+```ruby
+class ApplicationController < ActionController::Base
+    before_filter :setup_pubsub_context
+
+    private
+
+    def setup_pubsub_context
+        HonestPubsub::Context.setup_context
+          unique_id:       request.uuid,
+          orig_ip_address: request.remote_ip,
+          application:     "my_app/web:#{self.class.name.underscore}/#{action_name}",
+          user_id:         current_user.try(:id)
+    end
+end
+```
+
+Context is automatically cleared up by the gem for a Rails Application. Otherwise, you can call
+`HonestPubsub::Context.clear!`
+
+### Subscribers
+
+You can declare a subscriber class as follows
+
+```ruby
+class UserWelcomeSubscriber < HonestPubsub::Server::ClientWorker
+  subscribe_to "user_created" # The message prefix that you're subscribing to
+  # subscribe_to "user_created", on: 'welcome_emails_queue' # If you want to specify the queue name
+
+  def perform(context, payload)
+    # context is a hash that was put into HonestPubsub::Context
+    # Payload contains the data that was published
+    # My awesome logic goes here...
+  end
+end
+```
+
+You also run subscribers in a separate process to consume the message using the provided executable
+
+`bundle exec start_subscribers`
+
+You can execute `bundle exec start_subscribers --help` to see all the various options that it providers
+```
+Usage: bundle exec start_subscribers [options]
+    -P, --pidfile PATH               path to pidfile
+    -o, --only [SUBSCRIBERS]         comma separated name of subsriber classes that should be run
+    -r, --require [PATH|DIR]         Location of Rails application with workers or file to require
+    -v, --version                    Print version and exit
+
+```
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/start_subscribers

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/honest_pubsub/cli'
+
+begin
+  cli = HonestPubsub::CLI.instance
+  cli.parse
+  cli.run
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/config/pubsub.yml

@@ -0,0 +1,17 @@
+development:
+  # Documentation for parameters for rabbit and bunny is located here: http://rubybunny.info/articles/connecting.html
+  connection:
+    host: localhost
+    port: 5672
+    # username: rabbit
+    # password: rabbit
+    heartbeat: 60 # in seconds
+    log_level: 0
+    log_file: rabbit.log
+    network_recovery_interval: 10 # in seconds
+    continuation_timeout: 4000 # in milliseconds
+
+  logger:
+    enabled: true
+    level: warn
+    file: pubsub.log

data/honest_pubsub.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+# require all files in the folder?
+require 'honest_pubsub/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "honest_pubsub"
+  spec.version       = HonestPubsub::VERSION
+  spec.authors       = ["Thanh Lim"]
+  spec.email         = ["thanh@thehonestcompany.com"]
+  spec.description   = "Pub sub gem for Honest Company"
+  spec.summary       = "Pub sub gem for Honest Company"
+  spec.homepage      = ""
+  spec.license       = "Private"
+
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", ">= 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
+  spec.add_development_dependency "debugger"
+  spec.add_runtime_dependency "activesupport", ">= 3.2"
+  spec.add_runtime_dependency "awesome_print", "~> 1.2.0"
+  spec.add_runtime_dependency "bunny", ">= 1.2"
+  spec.add_runtime_dependency "airbrake", ">= 3.1"
+  spec.add_runtime_dependency "hashie", ">= 1.2"
+  spec.add_runtime_dependency "json", ">= 1.8"
+  spec.add_runtime_dependency "celluloid", ">= 0.15"
+  spec.add_runtime_dependency "celluloid-io", ">= 0.15"
+
+
+end

data/lib/honest_pubsub/cli.rb

@@ -0,0 +1,94 @@
+# This class handles parsing and running the command line interface for executing subscribers
+
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'honest_pubsub/server'
+
+module HonestPubsub
+  class CLI
+    include Singleton
+
+    attr_accessor :pidfile, :subscribers
+
+    # Method to support parsing of arguments passed through the command line
+    def parse(args = ARGV)
+      optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: bundle exec start_subscribers [options]"
+        opts.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          @pidfile = arg
+        end
+
+        opts.on("-o", "--only [SUBSCRIBERS]", "comma separated name of subsriber classes that should be run") do |subscribers|
+          @subscribers = subscribers.split(/\,/)
+        end
+
+        opts.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+          @require_path = arg
+        end
+
+        opts.on '-v', '--version', "Print version and exit" do |arg|
+          puts "HonestPubsub #{HonestPubsub::VERSION}"
+          abort
+        end
+      end
+
+      optparse.parse!(args)
+    end
+
+    def run
+      load_environment
+      write_pidfile
+      load_subscribers
+    end
+
+    def require_path
+      @require_path || "."
+    end
+
+
+    # @return [Array] returns array of subscriber classes that will be executed by the CLI
+    def subscriber_classes
+      if subscribers.present?
+        subscribers.map(&:constantize)
+      else
+        HonestPubsub::Server::ClientWorker.class_variable_get(:@@registered_subscribers)
+      end
+    end
+
+    def remove_pid
+      return unless pidfile
+      File.delete(pidfile) if File.exist?(pidfile)
+    end
+
+    private
+
+    def load_environment
+      if require_path
+        raise ArgumentError, "#{require_path} does not exist" unless File.exist?(require_path)
+      end
+
+      if File.directory?(require_path)
+        require 'rails'
+        require File.expand_path("#{require_path}/config/environment.rb")
+        ::Rails.application.eager_load!
+      else
+        require require_path
+      end
+    end
+
+    def write_pidfile
+      return unless pidfile
+      File.open(pidfile, 'w') do |f|
+        f.puts Process.pid
+      end
+    end
+
+    def load_subscribers
+      HonestPubsub::Server::SubscriberServer.new(subscriber_classes).start
+    end
+  end
+end
+
+

data/lib/honest_pubsub/configuration.rb

@@ -0,0 +1,38 @@
+module HonestPubsub
+  class Configuration
+    @@conf = nil
+
+    def self.configure_with(environment_name, yaml_file = nil)
+      @@yaml_file = yaml_file.nil? ? "config/pubsub.yml" : yaml_file
+      @@all_conf = Hashie::Mash.new(YAML.load_file(@@yaml_file) )
+      @@conf = @@all_conf[environment_name.to_sym]
+      self
+    end
+
+    def self.configuration
+      self.configure_with(self.environment) if @@conf.nil?
+      @@conf
+    end
+
+    def self.environment
+      val = ENV["RAILS_ENV"] || ENV["RACK_ENV"]
+      val = if val.present?
+        val.to_sym
+      else
+        if defined?(Rails)
+          Rails.env
+        else
+          raise "No environment can be found for configuration!"
+        end
+      end
+    end
+
+    def self.application_name
+      @application_name ||= if defined?(Rails)
+        Rails.application.class.name.to_s.gsub("::Application", '')
+      else
+        'honest_pubsub'
+      end.downcase
+    end
+  end
+end

data/lib/honest_pubsub/context.rb

@@ -0,0 +1,70 @@
+module HonestPubsub
+  class Context
+    # acceptable values
+    # employee - employee id that the context could have
+    # user_id - ender user id of call
+    # unique_id - unique identifier (could be request.uuid most of the time)
+    # orig_ip_address - ip address of originating requester
+    def initialize(params_hash = {})
+      @data = Hashie::Mash.new(params_hash)
+    end
+
+    def user_id
+      @data.user_id
+    end
+
+    def employee_id
+      @data.employee_id
+    end
+
+    def as_json
+      @data.as_json
+    end
+
+    def unique_id
+      @data.unique_id
+    end
+
+    def originating_ip_address
+      @data.orig_ip_address
+    end
+
+    def serialize
+      @data.as_json
+    end
+
+    def application
+      @data.application
+    end
+
+    def self.instance
+      Thread.current['pubsub_context'] || {}
+    end
+
+    def self.setup_context(attrs = {})
+      Thread.current['pubsub_context'] = self.new(attrs)
+    end
+
+    def self.clear!
+      Thread.current['pubsub_context'] = nil
+    end
+
+    # USE AS INFREQUENTLY AS POSSIBLE.  Only use it if you absolutely must
+    # as really, we want context in the object, and not the kitchen sink,
+    # which is what this will turn out to be if we use regularly use this
+    # mechanism.
+    def method_missing(method_name, *arguments, &block)
+      if @data.key?(method_name)
+        @data[method_name]
+      else
+        super
+      end
+
+    end
+
+    def self.from_json(data_hash)
+      ::HonestPubsub::Context.new(data_hash)
+    end
+
+  end
+end

data/lib/honest_pubsub/db_logger.rb

@@ -0,0 +1,52 @@
+
+# Internal database logger that saves the publish message before it is send
+# and also acts as a sink for messages.
+module HonestPubsub
+  class DbLogger
+
+    def initialize
+      @config = Configuration.configuration[:mongo]
+      if @config.present?
+        @client = Mongo::MongoClient.new(@config[:host], @config[:port])
+      else
+        @client = Mongo::MongoClient.new
+      end
+
+      @db = @client.db("logger")
+
+    end
+
+    def start_subscribe
+      subscriber = ::HonestPubsub::Subscriber.new("")
+      subscriber.start("logger") do |info, properties, contents|
+        # Write to mongo the contents and the routing_key, with the routing_key being indexed
+        routing_key = info[:routing_key]
+        chunks = routing_key.split(".")
+        # Logger will split the data into collections based upon the first parameter of the routing_key (initial roll out will only use 'honest')
+        collection_name = chunks[1] || chunks[0]
+        collection = @db.collection(collection_name)
+        # no need for safe writes.  need to write to a file log as well.
+        collection.insert({routing_key: routing_key, cts: contents, ts: properties[:timestamp]})
+        # make sure that we always index these two items in mongo
+        # it's a small price of a call, so no big deal
+        collection.ensure_index( {routing_key:1, ts:1} )
+      end
+    end
+
+    def log_publish(routing_key, message)
+      collection_name = "publisher"
+      collection = @db.collection(collection_name)
+
+      # no need for safe writes.  need to write to a file log as well.
+      collection.insert({routing_key: routing_key, cts: message[:payload], ts: message[:ts], pub: message[:pub]})
+      # make sure that we always index these two items in mongo
+      # it's a small price of a call, so no big deal
+      collection.ensure_index( {routing_key:1, ts:1} )
+
+    end
+
+    def teardown
+      @subscriber.teardown
+    end
+  end
+end

data/lib/honest_pubsub/exceptions/payload_error.rb

@@ -0,0 +1,2 @@
+class PayloadError < StandardError
+end

data/lib/honest_pubsub/logger.rb

@@ -0,0 +1,49 @@
+# Internal log class that is used to log messages before sending, after receiving, and failure to send
+module HonestPubsub
+  class Logger
+    def initialize(enable_publish_logging = true)
+      @enabled = HonestPubsub::Configuration.configuration[:logger][:enabled]
+      @enabled = enable_publish_logging if @enabled.nil?
+    end
+
+    def log_publish(routing_key, message)
+      return unless @enabled
+      # generate tags needed
+      tags = ["PUBLISH", message[:ts], message[:pub], message[:v], routing_key]
+
+      # FIX!!! -thl
+      # Could logging like this be too slow?
+      # Or should it be threaded?
+      # We'll need to benchmark, as we don't want this to get too slow.
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+
+      # TODO: -thl
+      # Log it into mongo as well?
+    end
+
+    def failed_publish(routing_key, properties, message)
+      tags = ["FAILED_SEND", message[:ts], message[:pub], message[:v], routing_key]
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+    end
+
+    def log_receive(routing_key, message)
+      tags = ["RECEIVED", message[:ts], message[:pub], message[:v], routing_key, Process::pid]
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+    end
+
+    def log_service(service_name, log_level, message)
+      tags = ["SERVICE", service_name, Process::pid]
+      log_method = log_level.to_sym.to_proc
+      logger.tagged(tags) { log_method.call(logger) { message.as_json } }
+    end
+
+    def teardown
+      logger.close
+    end
+
+    private
+    def logger
+      @logger ||= ActiveSupport::TaggedLogging.new(HonestPubsub.logger)
+    end
+  end
+end

data/lib/honest_pubsub/logging.rb

@@ -0,0 +1,42 @@
+require 'logger'
+
+# Manages a logger that can be used throughout the pubsub gem
+# The logger can be set to any object that quacks like a logger including the
+# Rails logger if so desired
+#
+# The logger can be set in the initializer of a program or anywhere throughout as
+# HonestPubsub.logger = Rails.logger for example
+
+module HonestPubsub
+  module Logging
+    # Sets the logger
+    #
+    # @param logger The logger to use throughout the pubsub gem
+    # @return The logger we use throughout the gem
+    def self.logger=(logger)
+      raise StandardError("Can't set logger to nil") unless logger.present?
+      @logger = logger
+    end
+
+    # Gets the logger
+    # If no logger is defined when this method is called it will return a standard
+    # ruby logger
+    #
+    # @return The logger we use throughout the gem
+    def self.logger 
+      config = HonestPubsub::Configuration.configuration[:logger]     
+      @logger ||= create_logger( {}.
+        merge( config[:level].present? ? { log_level: ::Logger::Severity.const_get(config[:level].upcase) } : {} ).
+        merge( config[:file].present? ? { log_target: config[:file] } : {} ) )
+    end
+
+    # Builds a standard ruby Logger
+    #
+    # @return Returns the logger at the end of the method
+    def self.create_logger( options = {} )
+      @logger = ::Logger.new(options.fetch(:log_target){ STDOUT })
+      @logger.level = options.fetch(:log_level){ ::Logger::INFO }
+      @logger
+    end
+  end
+end

data/lib/honest_pubsub/message.rb

@@ -0,0 +1,50 @@
+
+require "hashie"
+require "json"
+
+module HonestPubsub
+  class Message
+    @@message_version = "1"
+    attr_reader :ts
+    attr_reader :pub
+    attr_reader :version
+    attr_reader :payload
+    attr_reader :context
+
+    def initialize
+    end
+
+    # Context object should be passed into the call.
+    def serialize(context, routing_key, payload)
+      @ts = Time.now.to_i
+      @pub = "#{routing_key}:#{Socket.gethostname()}:#{Process::pid}"
+      @version = @@message_version
+      @payload = payload
+      @context = context
+      to_hash
+    end
+
+    def parse(envelope)
+      contents = Hashie::Mash.new(::JSON.parse(envelope))
+      @ts = contents[:ts]
+      @pub = contents[:pub]
+      # Version used for messaging can get updated if we need to add extra header information, and need
+      # to move each section of the library separately.
+      @version = contents[:v]
+      @payload = contents[:payload]
+      @context = ::HonestPubsub::Context.from_json(contents[:context])
+      to_hash
+    end
+
+    def to_hash
+      ::Hashie::Mash.new({
+        ts: @ts,
+        pub: @pub,
+        v: @@message_version,
+        payload: @payload,
+        context: @context.as_json
+      })
+    end
+
+  end
+end

data/lib/honest_pubsub/middleware.rb

@@ -0,0 +1,14 @@
+# Middleware to ensure that we clear the context with every request
+
+module HonestPubsub
+  class Middleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      HonestPubsub::Context.clear!
+      @app.call(env)
+    end
+  end
+end