emque-producing 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc3013b37568f755a5f258d0c18b3a8e346ac219
+  data.tar.gz: 28aad21a17a39be148e7d22611df8b4d992c85f6
+SHA512:
+  metadata.gz: 997635efddaa40fe279c246a3862917666b4c4d154b6809c9c69e7fe6e3a4281ffac27e8a1638c8556467f764be2cdb62a9b8d67edaf90331541e2d581b733da
+  data.tar.gz: 5cebfd302c30b90c1863555cd5b7fce581d67d1278310a05d6ad13f4bf71e573bf9a50311204da6a74e7108de3d7f735654db85905639bf621b14f982e68dfda

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    emque-producing (0.0.2)
+      oj (~> 2.10.2)
+      virtus (~> 1.0.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    amq-protocol (1.9.2)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    bunny (1.4.1)
+      amq-protocol (>= 1.9.2)
+    coderay (1.1.0)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    equalizer (0.0.9)
+    ice_nine (0.11.0)
+    method_source (0.8.2)
+    oj (2.10.2)
+    poseidon (0.0.4)
+    pry (0.9.12.4)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.1.1)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.5)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.2)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.1)
+    slop (3.4.7)
+    thread_safe (0.3.4)
+    virtus (1.0.3)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  bunny (~> 1.4.1)
+  emque-producing!
+  poseidon (= 0.0.4)
+  pry
+  rake
+  rspec (~> 3.1)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 TeamSnap
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+[ ![Codeship Status for
+teamsnap/emque-producing](https://www.codeship.io/projects/2ca7fd90-1785-0132-5f9d-7ab39a5c8237/status)](https://www.codeship.io/projects/34115)
+
+# Emque Producing
+
+Define and send messages with Ruby to a variety of [message brokers](http://en.wikipedia.org/wiki/Message_broker).
+Currently supported message brokers are [RabbitMQ](https://www.rabbitmq.com) and
+[Kafka](http://kafka.apache.org/).
+
+This is a library that pairs nicely with [Emque
+Consuming](https://www.github.com/teamsnap/emque-consuming), a framework for
+consuming and routing messages to your code.
+
+## Installation
+
+Add these lines to your application's Gemfile, depending on your message broker:
+
+    # for RabbitMQ, bunny is used
+    gem "emque-producing"
+    gem "bunny", "~> 1.4.1"
+
+or
+
+    # for Kafka, poseidon is used
+    gem "emque-producing"
+    gem "poseidon", "0.0.4"
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install emque-producing
+
+## Usage
+
+    # configure (likely in a Rails initializer)
+    require 'emque-producing'
+    Emque::Producing.configure do |c|
+      c.app_name = "app"
+      c.publishing_adapter = :rabbitmq
+      c.rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
+      #c.kafka_options = { :seed_brokers => [localhost:9092],
+      #                    :producer_options => {} }
+      c.error_handlers << Proc.new {|ex,context|
+       # notify/log
+      }
+    end
+
+    # create a message class
+    class MyMessage
+      include Emque::Producing::Message
+      topic "topic1"
+      message_type "mymessage.new"
+
+      attribute :first_property, Integer, :required => true
+      attribute :another_property, String, :required => true
+    end
+
+    # produce message
+    message = MyMessage.new({:first_property => 1, :another_property => "another"})
+    message.publish
+
+## Requirements
+
+* Ruby 1.9.3 or higher
+* RabbitMQ 3.x
+* Bunny 1.4.x
+* Kafka 0.8.1
+* Poseidon 0.0.4
+
+## Tests
+
+To run tests...
+
+```
+rspec
+```
+
+If you would like to test the gem as part of your client, you can update the
+configuration option `publish_messages` to false like so:
+```ruby
+    Emque::Producing.configure do |c|
+      c.publish_messages = false
+    ...other options
+    end 
+```
+This will prevent Emque from actually attempting to make the connection to your
+adapter which may be convenient in the case of CI environments.
+
+## Contributing
+
+1. Fork it ( http://github.com/teamsnap/emquemessages/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+  task :default => :spec
+rescue LoadError
+  # no rspec available
+  # end
+end

data/emque-producing.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib/', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'emque/producing/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "emque-producing"
+  spec.version       = Emque::Producing::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["Emily Dobervich", "Ryan Williams"]
+  spec.email         = ["emily@teamsnap.com", "ryan.williams@teamsnap.com"]
+  spec.summary       = %q{Define and send messages to a variety of message brokers}
+  spec.description   = %q{Define and send messages to a variety of message brokers}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+  spec.required_ruby_version = '>= 1.9.3'
+
+  # Manifest
+  spec.files         = `git ls-files`.split("\n")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "oj", "~> 2.10.2"
+  spec.add_dependency "virtus", "~> 1.0.3"
+
+  spec.add_development_dependency "bundler", "~> 1.0"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.1"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "poseidon", "0.0.4"
+  spec.add_development_dependency "bunny", "~> 1.4.1"
+end

data/lib/emque-producing.rb

@@ -0,0 +1 @@
+require "emque/producing"

data/lib/emque/producing.rb

@@ -0,0 +1,8 @@
+require "virtus"
+require "oj"
+require "emque/producing/version"
+require "emque/producing/producing"
+require "emque/producing/configuration"
+require "emque/producing/logging"
+require "emque/producing/publisher/base"
+require "emque/producing/message/message"

data/lib/emque/producing/configuration.rb

@@ -0,0 +1,24 @@
+module Emque
+  module Producing
+    class Configuration
+      attr_accessor :app_name
+      attr_accessor :publishing_adapter
+      attr_accessor :kafka_options
+      attr_accessor :rabbitmq_options
+      attr_accessor :error_handlers
+      attr_accessor :log_publish_message
+      attr_accessor :publish_messages
+
+      def initialize
+        @app_name = ""
+        @publishing_adapter = :rabbitmq
+        @error_handlers = []
+        @log_publish_message = false
+        @publish_messages = true
+        @kafka_options = { :seed_brokers => ["localhost:9092"],
+                           :producer_options => {} }
+        @rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
+      end
+    end
+  end
+end

data/lib/emque/producing/logging.rb

@@ -0,0 +1,25 @@
+require "logger"
+
+module Emque
+  module Producing
+    class Logging
+      def self.initialize_logger(log_target = STDOUT)
+        @logger = Logger.new(log_target)
+        @logger.level = Logger::INFO
+        @logger
+      end
+
+      def self.logger
+        defined?(@logger) ? @logger : initialize_logger
+      end
+
+      def self.logger=(log)
+        @logger = log || Logger.new("/dev/null")
+      end
+
+      def logger
+        Emque::Producing::Logging.logger
+      end
+    end
+  end
+end

data/lib/emque/producing/message/message.rb

@@ -0,0 +1,146 @@
+require 'securerandom'
+
+module Emque
+  module Producing
+    module Message
+      InvalidMessageError = Class.new(StandardError)
+      MessagesNotSentError = Class.new(StandardError)
+
+      module ClassMethods
+        def topic(name)
+          @topic = name
+        end
+
+        def read_topic
+          @topic
+        end
+
+        def message_type(name)
+          @message_type = name
+        end
+
+        def read_message_type
+          @message_type
+        end
+
+        def private_attribute(name, coercion=nil, opts={})
+          @private_attrs ||= []
+          @private_attrs << name
+          attribute(name, coercion, opts)
+        end
+
+        def private_attrs
+          Array(@private_attrs)
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, Virtus.model)
+        base.send(:attribute, :partition_key, String, :default => nil, :required => false)
+      end
+
+      def add_metadata
+        {
+          :metadata =>
+          {
+            :host => host_name,
+            :app => app_name,
+            :topic => topic,
+            :created_at => formatted_time,
+            :uuid => uuid,
+            :type => message_type,
+            :partition_key => partition_key
+          }
+        }.merge(public_attributes)
+      end
+
+      def topic
+        self.class.read_topic
+      end
+
+      def message_type
+        self.class.read_message_type
+      end
+
+      def valid?
+        invalid_attributes.empty? && topic && message_type
+      end
+
+      def invalid_attributes
+        invalid_attrs = self.class.attribute_set.inject([]) do |attrs, attr|
+          attrs << attr.name if attr.required? && self.attributes.fetch(attr.name).nil?
+          attrs
+        end
+        Array(invalid_attrs) - self.class.private_attrs
+      end
+
+      def to_json
+        data = self.add_metadata
+        Oj.dump(data, :mode => :compat)
+      end
+
+      def publish(publisher=Emque::Producing.publisher)
+        log "publishing...", true
+        if valid?
+          log "valid...", true
+          if Emque::Producing.configuration.publish_messages
+            sent = publisher.publish(topic, message_type, to_json, partition_key)
+            log "sent #{sent}"
+            raise MessagesNotSentError.new unless sent
+          end
+        else
+          log "failed...", true
+          raise InvalidMessageError.new(invalid_message)
+        end
+      end
+
+      private
+
+      def invalid_message
+        if !topic
+          "A topic is required"
+        elsif !message_type
+          "A message type is required"
+        else
+          "Required attributes #{invalid_attributes} are missing."
+        end
+      end
+
+      def host_name
+        Socket.gethostbyname(Socket.gethostname).first
+      end
+
+      def formatted_time
+        DateTime.now.new_offset(0).to_time.utc.iso8601
+      end
+
+      def uuid
+        SecureRandom.uuid
+      end
+
+      def app_name
+        Emque::Producing.configuration.app_name || raise("Messages must have an app name configured.")
+      end
+
+      def log(message, include_message = false)
+        if Emque::Producing.configuration.log_publish_message
+          message = "#{message} #{to_json}" if include_message
+          Emque::Producing.logger.info("MESSAGE LOG: #{message}")
+        end
+      end
+
+      def public_attributes
+        public = self.class.attribute_set.select do |attr|
+          attr && !self.class.private_attrs.include?(attr.name)
+        end.map(&:name)
+        slice_attributes(*public)
+      end
+
+      def slice_attributes(*keys)
+        keys.map!(&:to_sym)
+        attributes.select { |key, value| keys.include?(key) }
+      end
+    end
+  end
+end

data/lib/emque/producing/producing.rb

@@ -0,0 +1,43 @@
+module Emque
+  module Producing
+    class << self
+      attr_accessor :publisher
+      attr_writer :configuration
+
+      def configure
+        yield(configuration)
+      end
+
+      def configuration
+        @configuration ||= Emque::Producing::Configuration.new
+      end
+
+      def host_name
+        Socket.gethostbyname(Socket.gethostname).first
+      end
+
+      def publisher
+        return @publisher unless @publisher.nil?
+
+        if (configuration.publishing_adapter == :kafka)
+          require "emque/producing/publisher/kafka"
+          @publisher = Emque::Producing::Publisher::Kafka.new
+        elsif (configuration.publishing_adapter == :rabbitmq)
+          require "emque/producing/publisher/rabbitmq"
+          @publisher = Emque::Producing::Publisher::RabbitMq.new
+        else
+          raise "No publisher configured"
+        end
+        @publisher
+      end
+
+      def logger
+        Emque::Producing::Logging.logger
+      end
+
+      def logger=(log)
+        Emque::Producing::Logging.logger = log
+      end
+    end
+  end
+end

data/lib/emque/producing/publisher/base.rb

@@ -0,0 +1,23 @@
+module Emque
+  module Producing
+    module Publisher
+      class Base
+        def host_name
+          Socket.gethostbyname(Socket.gethostname).first
+        end
+
+        def handle_error(e)
+          Emque::Producing.configuration.error_handlers.each do |handler|
+            begin
+              handler.call(e, nil)
+            rescue => ex
+              Emque::Producing.logger.error "Producer error hander raised an error"
+              Emque::Producing.logger.error ex
+              Emque::Producing.logger.error Array(ex.backtrace).join("\n")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/producing/publisher/kafka.rb

@@ -0,0 +1,25 @@
+require "poseidon"
+
+module Emque
+  module Producing
+    module Publisher
+      class Kafka < Emque::Producing::Publisher::Base
+        def initialize
+          @producer = Poseidon::Producer.new(
+            Emque::Producing.configuration.kafka_options[:seed_brokers],
+            "producer_#{host_name}_#{Process.pid}",
+            Emque::Producing.configuration.kafka_options[:producer_options])
+        end
+
+        def publish(topic, message_type, message, key = nil)
+          begin
+            msg = Poseidon::MessageToSend.new(topic, message, key)
+            @producer.send_messages([msg])
+          rescue => e
+            handle_error(e)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/producing/publisher/rabbitmq.rb

@@ -0,0 +1,58 @@
+require "bunny"
+require "thread"
+
+module Emque
+  module Producing
+    module Publisher
+      class RabbitMq < Emque::Producing::Publisher::Base
+        CONN = Bunny
+          .new(Emque::Producing.configuration.rabbitmq_options[:url])
+          .tap { |conn|
+            conn.start
+          }
+
+        CHANNEL_POOL = Queue
+          .new
+          .tap { |queue|
+            20.times { |i| queue << CONN.create_channel }
+          }
+
+        def publish(topic, message_type, message, key = nil)
+          ch = CHANNEL_POOL.pop
+          ch.open if ch.closed?
+          begin
+            exchange = ch.fanout(topic, :durable => true, :auto_delete => false)
+
+            # Assumes all messages are mandatory in order to let callers know if
+            # the message was not sent. Uses publisher confirms to wait.
+            ch.confirm_select
+            sent = true
+            exchange.on_return do |return_info, properties, content|
+              sent = false
+            end
+
+            exchange.publish(
+              message,
+              :mandatory => true,
+              :persistent => true,
+              :type => message_type,
+              :app_id => Emque::Producing.configuration.app_name,
+              :content_type => "application/json")
+
+            success = ch.wait_for_confirms
+            unless success
+              Emque::Producing.logger.warn("RabbitMQ Publisher: message was nacked")
+              ch.nacked_set.each do |n|
+                Emque::Producing.logger.warn("message id: #{n}")
+              end
+            end
+
+            return sent
+          ensure
+            CHANNEL_POOL << ch unless ch.nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/producing/version.rb

@@ -0,0 +1,5 @@
+module Emque
+  module Producing
+    VERSION = "0.0.2"
+  end
+end

data/spec/bin/kafka-run-class.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+  echo "USAGE: $0 classname [opts]"
+  exit 1
+fi
+
+if [-z "$SCALA_VERSION" ]; then
+  SCALA_VERSION=2.8.0
+fi
+
+# assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency"
+for file in $KAFKA_PATH/core/target/scala-$SCALA_VERSION/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $KAFKA_PATH/perf/target/scala-$SCALA_VERSION/kafka*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+# classpath addition for release
+for file in $KAFKA_PATH/libs/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $KAFKA_PATH/kafka*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+if [ -z "$KAFKA_OPTS" ]; then
+  KAFKA_OPTS="-Xmx512M -server  -Dlog4j.configuration=file:$KAFKA_PATH/config/log4j.properties"
+fi
+
+if [  $JMX_PORT ]; then
+  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+exec $JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@"

data/spec/kafka_spec_helper.rb

@@ -0,0 +1,33 @@
+ROOT_DIRECTORY = File.absolute_path(File.dirname(__FILE__) + "/../")
+
+require "test_cluster"
+require "spec_helper"
+
+unless File.directory?(File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1"))
+  puts "\033[0;32m"
+  puts "*" * 83
+  puts "Downloading kafka"
+  puts "*" * 83
+  puts "\033[0;0m"
+
+  system(
+    "cd #{ROOT_DIRECTORY} && curl https://archive.apache.org/dist/kafka/0.8.1/kafka_2.8.0-0.8.1.tgz | tar xz"
+  )
+end
+
+ENV["KAFKA_PATH"] = File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1")
+ENV["SCALA_VERSION"] = "2.8.0"
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    JavaRunner.remove_tmp
+    JavaRunner.set_kafka_path!
+    $tc = TestCluster.new
+    $tc.start
+    sleep 5
+  end
+
+  config.after(:suite) do
+    $tc.stop
+  end
+end

data/spec/producing/configuration_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Emque::Producing::Configuration do
+  subject { Emque::Producing::Configuration.new }
+
+  it "provides default values" do
+    expect(subject.app_name).to eq ""
+    expect(subject.error_handlers).to eq []
+    expect(subject.kafka_options[:seed_brokers]).to eq ["localhost:9092"]
+  end
+
+  it "allows app_name to be overwritten" do
+    subject.app_name = "my app"
+    expect(subject.app_name).to eq "my app"
+  end
+
+  it "allows seed_brokers to be overwritten" do
+    subject.kafka_options[:seed_brokers] = ["kafka1:9092", "kafka2:9092"]
+    expect(subject.kafka_options[:seed_brokers]).to eq ["kafka1:9092", "kafka2:9092"]
+  end
+end

data/spec/producing/message/message_spec.rb

@@ -0,0 +1,88 @@
+require "spec_helper"
+require "virtus"
+require "emque/producing/message/message"
+
+class TestMessage
+  include Emque::Producing::Message
+
+  topic "queue"
+  message_type "queue.new"
+
+  attribute :test_id, Integer, :required => true
+  private_attribute :extra, String, :default => "value"
+end
+
+class MessageNoTopic
+  include Emque::Producing::Message
+  message_type "testing"
+end
+
+class MessageNoType
+  include Emque::Producing::Message
+  topic "testing"
+end
+
+describe Emque::Producing::Message do
+  before do
+    Emque::Producing.configure do |c|
+      c.app_name = "apiv3"
+    end
+  end
+
+  describe "#to_json" do
+    it "creates the metadata" do
+      message = TestMessage.new(:test_id => 1)
+      metadata = message.add_metadata[:metadata]
+      expect(metadata[:app]).to eql("apiv3")
+    end
+
+    it "can be transformed to json" do
+      message = Oj.load(TestMessage.new().to_json)
+      expect(message["metadata"]["app"]).to eql("apiv3")
+    end
+
+    it "includes valid attributes in json" do
+      produced_message = TestMessage.new(:test_id => 1)
+      json = produced_message.to_json
+      consumed_message = Oj.load(json)
+      expect(consumed_message["test_id"]).to eql(1)
+    end
+  end
+
+  describe "#retry" do
+    pending
+  end
+
+  it "validates the message for missing attributes" do
+    message = TestMessage.new()
+    expect(message).to_not be_valid
+  end
+
+  it "raises a useful message when trying to send an invalid message" do
+    message = TestMessage.new()
+    expected_error = Emque::Producing::Message::InvalidMessageError
+    expect{message.publish(->{})}.to raise_error(expected_error)
+  end
+
+  it "validates that the message has a topic" do
+    message = MessageNoTopic.new
+    expected_error = Emque::Producing::Message::InvalidMessageError
+    expect{message.publish(->{})}.to raise_error(expected_error, "A topic is required")
+  end
+
+  it "validates that the message has a message type" do
+    message = MessageNoType.new
+    expected_error = Emque::Producing::Message::InvalidMessageError
+    expect{message.publish(->{})}.to raise_error(expected_error, "A message type is required")
+  end
+
+  it "applys a uuid per message" do
+    message = TestMessage.new()
+    expect(message.add_metadata[:metadata][:uuid]).to_not be_nil
+  end
+
+  it "has the sub type in the metadata" do
+    message = TestMessage.new()
+    expect(message.add_metadata[:metadata][:type]).to eql("queue.new")
+  end
+end

data/spec/producing/producing_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Emque::Producing do
+
+end

data/spec/producing/publisher_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+require "emque/producing/publisher/kafka"
+
+describe Emque::Producing::Publisher do
+  describe "#publish" do
+    context "when error handler raises an exception" do
+      it "handles the exception" do
+        expect_any_instance_of(Poseidon::Producer). to receive(:send_messages).and_raise
+        Emque::Producing.configure do |c|
+          c.error_handlers << Proc.new {|ex,context|
+            raise "something"
+          }
+        end
+        Emque::Producing.logger = nil
+        publisher = Emque::Producing::Publisher::Kafka.new
+        publisher.publish("mytopic", "message.type", "mymessage")
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+$TESTING = true
+
+require "pry"
+require "emque-producing"
+
+ENV["EMQUE_ENV"] = "test"
+
+module VerifyAndResetHelpers
+  def verify(object)
+    RSpec::Mocks.proxy_for(object).verify
+  end
+
+  def reset(object)
+    RSpec::Mocks.proxy_for(object).reset
+  end
+end
+
+RSpec.configure do |config|
+  config.order = "random"
+
+  config.include VerifyAndResetHelpers
+end

data/spec/test_cluster.rb

@@ -0,0 +1,201 @@
+# from https://github.com/bpot/poseidon
+
+require 'daemon_controller'
+
+class TestCluster
+  attr_reader :broker, :zookeeper
+  def initialize
+    @zookeeper = ZookeeperRunner.new
+    @broker = BrokerRunner.new(0, 9092)
+  end
+
+  def start
+    @zookeeper.start
+    @broker.start
+  end
+
+  def stop
+    @zookeeper.stop
+    @broker.stop
+  end
+end
+
+class JavaRunner
+  def self.remove_tmp
+    FileUtils.rm_rf("#{ROOT_DIRECTORY}/tmp")
+  end
+
+  def self.set_kafka_path!
+    JavaRunner.kafka_path = File.join(ROOT_DIRECTORY, "kafka-0.8.0-src")
+  end
+
+  def self.kafka_path=(kafka_path)
+    @kafka_path = kafka_path
+  end
+
+  def self.kafka_path
+    @kafka_path
+  end
+
+  attr_reader :pid
+  def initialize(id, start_cmd, port, properties = {})
+    @id = id
+    @properties = properties
+    @pid = nil
+    @start_cmd = start_cmd
+    @port = port
+  end
+
+  def start
+    write_properties
+    run
+  end
+
+  def stop
+    daemon_controller.stop
+  end
+
+  def without_process
+    stop
+    begin
+      yield
+    ensure
+      start
+      sleep 5
+    end
+  end
+
+  private
+
+  def daemon_controller
+    @dc ||= DaemonController.new(
+      :identifier => @id,
+      :start_command => "#{@start_cmd} #{config_path} >>#{log_path} 2>&1 & echo $! > #{pid_path}",
+      :ping_command => [:tcp, '127.0.0.1', @port],
+      :pid_file => pid_path,
+      :log_file => log_path,
+      :start_timeout => 25
+    )
+  end
+
+  def run
+    FileUtils.mkdir_p(log_dir)
+    FileUtils.mkdir_p(pid_dir)
+    daemon_controller.start
+  end
+
+  def write_properties
+    FileUtils.mkdir_p(config_dir)
+    File.open(config_path, "w+") do |f|
+      @properties.each do |k,v|
+        f.puts "#{k}=#{v}"
+      end
+    end
+  end
+
+  def pid_path
+    "#{pid_dir}/#{@id}.pid"
+  end
+
+  def pid_dir
+    "#{file_path}/pid"
+  end
+
+  def log_path
+    "#{log_dir}/#{@id}.log"
+  end
+
+  def log_dir
+    "#{file_path}/log"
+  end
+
+  def config_path
+    "#{config_dir}/#{@id}.properties"
+  end
+
+  def config_dir
+    "#{file_path}/config"
+  end
+
+  def file_path
+    ROOT_DIRECTORY + "/tmp/"
+  end
+end
+
+class BrokerRunner
+  DEFAULT_PROPERTIES = {
+    "broker.id" => 0,
+    "port" => 9092,
+    "num.network.threads" => 2,
+    "num.io.threads" => 2,
+    "socket.send.buffer.bytes" => 1048576,
+    "socket.receive.buffer.bytes" => 1048576,
+    "socket.request.max.bytes" => 104857600,
+    "log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs",
+    "num.partitions" => 1,
+    "log.flush.interval.messages" => 10000,
+    "log.flush.interval.ms" => 1000,
+    "log.retention.hours" => 168,
+    "log.segment.bytes" => 536870912,
+    "log.cleanup.interval.mins" => 1,
+    "zookeeper.connect" => "localhost:2181",
+    "zookeeper.connection.timeout.ms" => 1000000,
+    "kafka.metrics.polling.interval.secs" => 5,
+    "kafka.metrics.reporters" => "kafka.metrics.KafkaCSVMetricsReporter",
+    "kafka.csv.metrics.dir" => "#{ROOT_DIRECTORY}/tmp/kafka_metrics",
+    "kafka.csv.metrics.reporter.enabled" => "false",
+  }
+
+  def initialize(id, port, partition_count = 1)
+    @id   = id
+    @port = port
+    @jr = JavaRunner.new("broker_#{id}",
+                         "#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh kafka.Kafka",
+                         port,
+                         DEFAULT_PROPERTIES.merge(
+                           "broker.id" => id,
+                           "port" => port,
+                           "log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs_#{id}",
+                           "num.partitions" => partition_count
+                         ))
+  end
+
+  def pid
+    @jr.pid
+  end
+
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+
+  def without_process
+    @jr.without_process { yield }
+  end
+end
+
+class ZookeeperRunner
+  def initialize
+    @jr = JavaRunner.new("zookeeper",
+                         "#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain",
+                         2181,
+                         :dataDir => "#{ROOT_DIRECTORY}/tmp/zookeeper",
+                         :clientPort => 2181,
+                         :maxClientCnxns => 0)
+  end
+
+  def pid
+    @jr.pid
+  end
+
+  def start
+    @jr.start
+  end
+
+  def stop
+    @jr.stop
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: emque-producing
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Emily Dobervich
+- Ryan Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.2
+- !ruby/object:Gem::Dependency
+  name: virtus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+description: Define and send messages to a variety of message brokers
+email:
+- emily@teamsnap.com
+- ryan.williams@teamsnap.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- emque-producing.gemspec
+- lib/emque-producing.rb
+- lib/emque/producing.rb
+- lib/emque/producing/configuration.rb
+- lib/emque/producing/logging.rb
+- lib/emque/producing/message/message.rb
+- lib/emque/producing/producing.rb
+- lib/emque/producing/publisher/base.rb
+- lib/emque/producing/publisher/kafka.rb
+- lib/emque/producing/publisher/rabbitmq.rb
+- lib/emque/producing/version.rb
+- spec/bin/kafka-run-class.sh
+- spec/kafka_spec_helper.rb
+- spec/producing/configuration_spec.rb
+- spec/producing/message/message_spec.rb
+- spec/producing/producing_spec.rb
+- spec/producing/publisher_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Define and send messages to a variety of message brokers
+test_files:
+- spec/bin/kafka-run-class.sh
+- spec/kafka_spec_helper.rb
+- spec/producing/configuration_spec.rb
+- spec/producing/message/message_spec.rb
+- spec/producing/producing_spec.rb
+- spec/producing/publisher_spec.rb
+- spec/spec_helper.rb
+- spec/test_cluster.rb