queue_client 1.0.0

data/.ruby-gemset

@@ -0,0 +1 @@
+queue-client

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.9.3-p448

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    queue_client (1.0.0)
+      amqp (= 0.9.8)
+      bunny (= 0.9.0.pre7)
+      eventmachine (= 1.0.0)
+      oj (= 2.0.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    amq-client (0.9.12)
+      amq-protocol (>= 1.2.0)
+      eventmachine
+    amq-protocol (1.3.0)
+    amqp (0.9.8)
+      amq-client (~> 0.9.5)
+      amq-protocol (>= 0.9.4)
+      eventmachine
+    bunny (0.9.0.pre7)
+      amq-protocol (>= 1.2.0)
+    diff-lcs (1.1.3)
+    eventmachine (1.0.0)
+    oj (2.0.5)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  queue_client!
+  rspec (>= 2.12.0)

data/lib/queue_client.rb

@@ -0,0 +1,35 @@
+require 'oj'
+require 'bunny'
+
+module Viki
+  module Queue
+    @host = 'localhost'
+    @port = 5672
+    @username = 'guest'
+    @password = 'guest'
+    class << self
+      attr_accessor :host, :port, :username, :password, :client_name, :_service
+    end
+
+    def self.configure(&block)
+      block.call self
+      raise "Viki::Queue.client_name not set" unless client_name
+      nil
+    end
+
+    def self.service
+      Viki::Queue._service ||= Service.new
+    end
+
+    def self.reconnect
+      Viki::Queue._service.stop
+      Viki::Queue._service = Service.new({routing: Viki::Queue._service.routing})
+    end
+  end
+end
+
+require_relative 'viki/queue/logger'
+require_relative 'viki/queue/message'
+require_relative 'viki/queue/runner'
+require_relative 'viki/queue/service'
+require_relative 'viki/queue/version'

data/lib/viki/queue/logger.rb

@@ -0,0 +1,9 @@
+module Viki::Queue
+  module Logger
+    def self.log(message, e = nil)
+      puts "#{message}\n"
+      puts "\t#{e.to_s}\n\tstack: #{e.backtrace.join("\n\t")}" unless e.nil?
+      return 0
+    end
+  end
+end

data/lib/viki/queue/message.rb

@@ -0,0 +1,56 @@
+module Viki::Queue
+  module Message
+    def create_message(resource, id, payload=nil)
+      write(prepare_message([:create, resource, id, payload]))
+    end
+
+    def update_message(resource, id, payload=nil)
+      write(prepare_message([:update, resource, id, payload]))
+    end
+
+    def delete_message(resource, id, payload=nil)
+      write(prepare_message([:delete, resource, id, payload]))
+    end
+
+    def bulk(*events)
+      write_many(events.map { |e| prepare_message(e) })
+    end
+
+    private
+
+    def write(payload)
+      routing_key = "#{Viki::Queue._service.routing}.#{payload[:resource]}.#{payload[:action]}"
+      message = Oj.dump(payload, mode: :compat)
+      opts = {routing_key: routing_key, timestamp: Time.now.to_i, persistent: true}
+      attemps = 5
+      while attemps > 0
+        begin
+          send_to_queue(message, opts)
+          break
+        rescue Exception => e
+          attemps -= 1
+          if attemps > 0
+            sleep(0.5)
+            Viki::Queue.reconnect()
+          else
+            Viki::Queue::Logger.log('Failed to write in queue. Exiting.', e)
+          end
+        end
+      end
+    end
+
+    def send_to_queue(message, opts)
+      Viki::Queue._service.exchange.publish(message, opts)
+    end
+
+    def write_many(payload)
+      payload.each { |p| write(p) }
+    end
+
+    def prepare_message(payload)
+      m = {action: payload[0], resource: payload[1], id: payload[2], _meta: {client_name: Viki::Queue.client_name}}
+      m[:payload] = payload[3] unless payload[3].nil?
+      m
+    end
+  end
+end

data/lib/viki/queue/runner.rb

@@ -0,0 +1,64 @@
+require 'amqp'
+
+module Viki::Queue
+  module Runner
+    def run(queue, router, config={})
+      config = {iterations: 1, fail_pause: 10}.merge(config)
+
+      begin
+        EventMachine.run do
+          connection = AMQP.connect({
+            host: Viki::Queue.host,
+            port: Viki::Queue.port,
+            username: Viki::Queue.username,
+            password: Viki::Queue.password})
+          channel = AMQP::Channel.new(connection)
+          loops = 0
+          channel.prefetch(1).queue(queue, :durable => true).subscribe(:ack => true) do |metadata, message|
+            processed = false
+            for i in 1..10 do
+              begin
+                payload = Oj.load(message, symbol_keys: true)
+                if payload[:_meta]
+                  payload[:_meta][:timestamp] = metadata.timestamp
+                else
+                  payload[:_meta] = {timestamp: metadata.timestamp}
+                end
+
+                if process(router, payload) == true
+                  processed = true
+                  metadata.ack
+                  break
+                end
+              rescue => e
+                router.error(e)
+              end
+              sleep(config[:fail_pause])
+            end
+
+            unless processed
+              router.error("Failed to process message: #{message}")
+              connection.close { EventMachine.stop }
+            end
+
+            loops += 1
+            if loops == config[:iterations]
+              connection.close { EventMachine.stop }
+            end
+          end
+        end
+      rescue Interrupt
+        puts "Queue is interrupt. Good night!"
+      end
+    end
+
+    private
+
+    def process(router, message)
+      unless message.nil?
+        method = "#{message[:action]}_#{message[:resource]}"
+        router.send(method, message)
+      end
+    end
+  end
+end

data/lib/viki/queue/service.rb

@@ -0,0 +1,51 @@
+module Viki::Queue
+  class Service
+    include Viki::Queue::Message
+    include Viki::Queue::Runner
+
+    attr_accessor :routing, :exchange
+
+    def initialize(opts={})
+      @connection = Bunny.new({
+        host: Viki::Queue.host,
+        port: Viki::Queue.port,
+        username: Viki::Queue.username,
+        password: Viki::Queue.password,
+        keepalive: true,
+        threaded: false,
+        socket_timeout: 0,
+        connect_timeout: 0})
+      @connection.start
+      @channel = @connection.create_channel
+      @exchange = @channel.topic("general", durable: true)
+      @routing = opts.fetch(:routing, 'resources')
+    end
+
+    def stop
+      @connection.stop
+    rescue
+    end
+
+    def subscribe(queue, resources)
+      resources.each do |r|
+        @channel.queue(queue, durable: true).bind(@exchange, :routing_key => "#{@routing}.#{r}.#")
+      end
+    end
+
+    def unsubscribe(queue, resources)
+      resources.each do |r|
+        @channel.queue(queue, durable: true).unbind(@exchange, :routing_key => "#{@routing}.#{r}.#")
+      end
+    end
+
+    def route(route)
+      return if route.nil? or route == ""
+      route = route[0...-1] if route[-1] == '.'
+      @routing = route
+    end
+
+    def delete(queue)
+      @channel.queue(queue, durable: true).delete()
+    end
+  end
+end

data/lib/viki/queue/version.rb

@@ -0,0 +1,5 @@
+module Viki
+  module Queue
+    VERSION = "1.0.0"
+  end
+end

data/queue_client.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "viki/queue/version"
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Viki"]
+  gem.email         = ["engineering@viki.com"]
+  gem.description   = %q{A client for Viki's queue}
+  gem.homepage      = "https://github.com/viki-org/queue-client"
+  gem.summary       = %q{A client for Viki's queue}
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "queue_client"
+  gem.require_paths = ["lib"]
+  gem.version       = Viki::Queue::VERSION
+
+  gem.add_development_dependency "rspec", ">= 2.12.0"
+
+  gem.add_runtime_dependency "bunny", "0.9.0.pre7"
+  gem.add_runtime_dependency "eventmachine", "1.0.0"
+  gem.add_runtime_dependency "amqp", "0.9.8"
+  gem.add_runtime_dependency "oj", "2.0.5"
+end

data/readme.md

@@ -0,0 +1,106 @@
+#Queue
+
+## Configuration
+    Viki::Queue.configure do |config|
+      config.client_name = "my_app"
+      config.host = "queue.dev.viki.io"
+      config.port = "5672"
+      config.username = "username"
+      config.password = "password"
+    end
+
+It defaults to localhost queues.
+
+## Initialize
+    queue_service = Viki::Queue.service
+
+## Subscribe
+Before being able to consume from a queue, you must first subscribe to some resources:
+
+    queue_service.subscribe('gaia-applications', ['application', 'user'])
+
+This creates a queue named *gaia_applications* which will monitor the *application* and *user* resources, i.e. it will subscribe to `resources.application.#` and `resources.user.#`. It is possible to change the root of the route path by using the `.route(new_route)` command as described below (see *routing*).
+
+## Unsubscribe
+You can unsubscribe from some resources:
+
+    queue_service.unsubscribe('gaia-applications', ['user'])
+
+This leaves a queue named *gaia_applications* which only monitors the *application* resource, i.e. it will subscribe to `resources.application.#`. It is possible to change the root of the route path by using the `.route(new_route)` command as described below (see *routing*).
+
+## Deletion
+You can delete a queue and all of its queued events:
+
+    queue_service.delete('gaia_applications')
+
+## Routing
+By default, all the messages will be routed under `resources.RESOURCE_NAME.ACTION`, e.g. `resources.videos.create`.It is possible to modify the root of the route.
+
+    queue_service.route('delayed_jobs.subbing')
+    queue_service.create_message(:compile, '3v')
+
+This commands will create a message under the route `delayed_jobs.subbing.compile.create`.
+
+On the other hand, by default subscriptions are routed via `resources.RESOURCE_NAME.#`, e.g. `resources.videos.#`.
+
+    queue_service.route('services.gaia')
+    queue_service.subscribe('gaia-consumer', [apps])
+
+This will create a new queue called `gaia-consumer` that is subscribed to `services.gaia.apps.#`.
+
+*Note* that the part `.resource.action` is always present in both examples, changing the route only affects the root of it.
+
+#Messages
+
+  Messages have always the following structure:
+
+    action: mandatory, it will always be 'create', 'update' or 'delete'
+    resource: mandatory
+    id: mandatory
+    payload: optional attribute.
+
+## Writing
+Use the `create_message`, `update_message` and `delete_message` methods:
+
+    queue_service.create_message(:application, '38a')
+    queue_service.update_message(:user, '9003u')
+    queue_service.delete_message(:container, '50c')
+
+An optional payload parameter can be supplied:
+
+    queue_service.create_message(:application, '38a', {name: 'gaia'})
+
+This will create a message in the queue that looks like:
+
+    {action: 'create', resource: 'application', id: '38a', payload: {name: 'gaia'} }
+
+### Bulk write
+Multiple events can be sent at once:
+
+    queue_service.bulk([:create, :video, '1v'], [:update, :user, '1u', 'optional_payload'], [:delete, :container, '1c'])
+
+The events will be queued in-order.
+
+## Consumption
+
+Consumption involves using the built-in runner and providing a routing class:
+
+    Class GaiaRouter
+      def self.delete_application(event)
+        # do something
+        true
+      end
+      def self.error(error)
+        # handle
+      end
+    end
+    queue_service.run(QUEUE_NAME, GaiaRouter)
+
+The method names look like `ACTION_RESOURCE`, where `ACTION` can be `create`, `update` or `delete`. The `RESOURCE` can be any resource. *Note* that if noone is registered for listening the kind of resource you are sending, the message will just be dropped. There's no need to `close` the queue, simply return true when the event has been successfully processed. *Note* that a message is only acknowledged to the queue when the processing method returns true.
+
+If an exception is raised while processing the message, the runner will call the error method of the router with the exception. Afterwards, the runner will wait before trying to reprocess the same message again.
+
+`run` takes a 3rd optional argument to configure the runner, possible values are:
+
+* `iterations`: The number of iterations to run, i.e. of messages to process. 0 means loop forever (DEFAULT 1).
+* `fail_pause`: Seconds to pause until trying to reprocess a failed message (DEFAULT 10)

data/spec/queue/authentication_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Viki::Queue do
+  describe 'authentication' do
+    it 'should fail is wrong username/pasword is given' do
+      Viki::Queue.configure do |c|
+        c.username = 'wrong'
+        c.password = 'alsowrong'
+      end
+      expect{ Viki::Queue.service.create_message(:kitten, '22k') }.to raise_error(Bunny::PossibleAuthenticationFailureError)
+    end
+  end
+end

data/spec/queue/write_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+
+describe Viki::Queue do
+  describe "write" do
+    it "writes a create event" do
+      assert({action: 'create', resource: 'kitten', id: '22k', _meta: {client_name: 'testing'}}) do
+        Viki::Queue.service.create_message(:kitten, '22k')
+      end
+    end
+
+    it "writes an update event" do
+      assert({action: 'update', resource: 'kitten', id: '22k', _meta: {client_name: 'testing'}}) do
+        Viki::Queue.service.update_message(:kitten, '22k')
+      end
+    end
+
+    it "writes a delete event" do
+      assert({action: 'delete', resource: 'kitten', id: '22k', _meta: {client_name: 'testing'}}) do
+        Viki::Queue.service.delete_message(:kitten, '22k')
+      end
+    end
+
+    it "sends payload when provided" do
+      assert({action: 'create', resource: 'kitten', id: '22k', payload: [1,2,3], _meta: {client_name: 'testing'}}) do
+        Viki::Queue.service.create_message(:kitten, '22k', [1,2,3])
+      end
+    end
+
+    it "writes multiple events at once" do
+      assert([
+        {action: 'create', resource: 'kitten', id: '22k', payload: 'some_payload', _meta: {client_name: 'testing'}},
+        {action: 'delete', resource: 'dogs', id: '34', _meta: {client_name: 'testing'}}], 2) do
+        Viki::Queue.service.bulk([:create, :kitten, '22k', 'some_payload'], [:delete, :dogs, '34'])
+      end
+    end
+
+    it "writes the client name" do
+      assert({action: 'create', resource: 'kitten', _meta: {client_name: 'testing'}, id: '22k', _meta: {client_name: 'testing'}}) do
+        orig = Viki::Queue.client_name
+        Viki::Queue.client_name = "testing"
+        Viki::Queue.service.create_message(:kitten, '22k')
+        Viki::Queue.client_name = orig
+      end
+    end
+
+    private
+
+    def assert(message, total=1)
+      EventMachine.run do
+        connection = AMQP.connect(host: Viki::Queue.host, port: Viki::Queue.port)
+        channel = AMQP::Channel.new(connection)
+        q = channel.queue("", exclusive: true)
+        routing_key = message.class == Array ? "resources.#" : "resources.#{message[:resource]}.#"
+        q.bind("general", routing_key: routing_key) do
+          yield
+        end
+        count = 0
+        q.subscribe do |m|
+          m = Oj.load(m, symbol_keys: true)
+          if message.class == Array
+            message.include?(m).should == true
+          else
+            m.should == message
+          end
+          count += 1
+          if count == total
+           connection.close { EventMachine.stop }
+         end
+        end
+      end
+    end
+
+  end
+end

data/spec/runner/run_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe Viki::Queue::Runner do
+  describe "run" do
+    it "polls the queue and noops a nil" do
+      fakeRouter = Class.new do
+        def self.create_application(e)
+          e[:resource].should == 'application'
+          e[:action].should == 'create'
+          e[:id].should == '22a'
+          true
+        end
+        def self.error(e)
+          puts(%{===> e: %s} % [(e).inspect])
+        end
+      end
+      q = Viki::Queue.service
+      q.subscribe('a_queue', ['application'])
+      q.create_message(:application, '22a')
+      q.run('a_queue', fakeRouter)
+      q.delete('a_queue')
+    end
+
+    it "includes the timestamp and the existing meta" do
+      fakeRouter = Class.new do
+        def self.create_application(e)
+          meta = e.delete(:_meta)
+          meta.include?(:timestamp).should == true
+          meta[:client_name].should == "testing"
+          true
+        end
+        def self.error(e)
+          puts(%{===> e: %s} % [(e).inspect])
+        end
+      end
+      q = Viki::Queue.service
+      q.subscribe('a_queue', ['application'])
+      orig = Viki::Queue.client_name
+      Viki::Queue.client_name = "testing"
+      q.create_message(:application, '22a')
+      Viki::Queue.client_name = orig
+      q.run('a_queue', fakeRouter)
+      q.delete('a_queue')
+    end
+
+    it "iterates multiple times" do
+      fakeRouter = Class.new do
+        class << self
+          attr_accessor :count
+        end
+        def self.create_application(e)
+          @count ||= 0
+          @count += 1
+          true
+        end
+        def self.update_video(e)
+          @count ||= 0
+          @count += 1
+          true
+        end
+        def self.error(e)
+          puts(%{===> e: %s} % [(e).inspect])
+        end
+      end
+      Time.stub(:now).and_return(Time.at(323232323))
+      q = Viki::Queue.service
+      q.subscribe('the-queue', ['application', 'video'])
+      q.create_message(:application, '22a')
+      q.update_message(:video, '1v')
+      q.run('the-queue', fakeRouter, iterations: 2)
+      fakeRouter.count.should == 2
+      q.delete('the-queue')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+require_relative '../lib/queue_client'
+require 'amqp'
+require 'json'
+Dir["./spec/support/**/*.rb"].each { |f| require f }
+
+RSpec.configure do |config|
+  config.before(:each) {
+    Viki::Queue.configure do |c|
+      c.host = 'localhost'
+      c.port = 5672
+      c.username = 'guest'
+      c.password = 'guest'
+      c.client_name = 'testing'
+    end
+  }
+end

data/t

@@ -0,0 +1,2 @@
+#!/bin/sh
+rspec --color spec

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: queue_client
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Viki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0.pre7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.0.pre7
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.8
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.5
+description: A client for Viki's queue
+email:
+- engineering@viki.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .ruby-gemset
+- .ruby-version
+- Gemfile
+- Gemfile.lock
+- lib/queue_client.rb
+- lib/viki/queue/logger.rb
+- lib/viki/queue/message.rb
+- lib/viki/queue/runner.rb
+- lib/viki/queue/service.rb
+- lib/viki/queue/version.rb
+- queue_client.gemspec
+- readme.md
+- spec/queue/authentication_spec.rb
+- spec/queue/write_spec.rb
+- spec/runner/run_spec.rb
+- spec/spec_helper.rb
+- t
+homepage: https://github.com/viki-org/queue-client
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A client for Viki's queue
+test_files:
+- spec/queue/authentication_spec.rb
+- spec/queue/write_spec.rb
+- spec/runner/run_spec.rb
+- spec/spec_helper.rb