captainu-tincan 0.7.0

data/lib/tincan/failure.rb

@@ -0,0 +1,67 @@
+require 'date'
+require 'json'
+require 'tincan/message'
+
+module Tincan
+  # Encapsulates a failed attempt at a message attempted from a Redis queue.
+  class Failure
+    attr_accessor :failed_at, :attempt_count, :message_id, :queue_name
+
+    # Creates a new instance of a notification with an object (usually an
+    # ActiveModel instance).
+    # @param [Integer] message_id The identifier for the message to retry.
+    # @param [String] queue_name The name of the queue in which this failure
+    #                 originally occurred.
+    # @return [Tincan::Message] An instance of this class.
+    def initialize(message_id = nil, queue_name = nil)
+      self.message_id = message_id
+      self.attempt_count = 0
+      self.failed_at = DateTime.now
+      self.queue_name = queue_name
+    end
+
+    # Gives a date and time when this object is allowed to be attempted again,
+    # derived from when it last failed, plus the number of attempts, in
+    # seconds.
+    # @return [DateTime] The date/time when this is allowed to be retried.
+    def attempt_after
+      failed_at + Rational((attempt_count * 10), 86400)
+    end
+
+    # Deserializes an object from a JSON string.
+    # @param [String] json A JSON string to be decoded.
+    # @return [Tincan::Failure] A deserialized failure.
+    def self.from_json(json)
+      from_hash(JSON.parse(json))
+    end
+
+    # Assigns keys and values to this object based on an already-deserialized
+    # JSON object.
+    # @param [Hash] hash A hash of properties and their values.
+    # @return [Tincan::Failure] A failure.
+    def self.from_hash(hash)
+      instance = new(hash['message_id'], hash['queue_name'])
+      instance.attempt_count = hash['attempt_count'].to_i
+      instance
+    end
+
+    # Generates a version of this failure as a JSON string.
+    # @return [String] A JSON-compliant marshalling of this instance's data.
+    def to_json(options = {})
+      Hash[%i(failed_at attempt_count message_id queue_name).map do |name|
+        [name, send(name)]
+      end].to_json(options)
+    end
+
+    # Object overrides
+
+    # Overrides equality operator to determine if all ivars are equal
+    def ==(other)
+      false unless other
+      checks = %i(failed_at attempt_count message_id queue_name).map do |p|
+        other.send(p) == send(p)
+      end
+      checks.all?
+    end
+  end
+end

data/lib/tincan/message.rb

@@ -0,0 +1,79 @@
+require 'date'
+require 'json'
+require 'active_support/inflector'
+
+module Tincan
+  # Encapsulates a message published to (and received from) a Redis "tincan"
+  # queue.
+  class Message
+    attr_accessor :object_name, :change_type, :object_data, :published_at
+
+    # Creates a new instance of a notification with an object (usually an
+    # ActiveModel instance).
+    # @param [Block] Takes an optional block to configure this instance.
+    # @return [Tincan::Message] An instance of this class.
+    def initialize
+      yield self if block_given?
+      self.published_at ||= DateTime.now
+    end
+
+    # Deserializes an object from a JSON string.
+    # @param [String] json A JSON string to be decoded.
+    # @return [Tincan::Message] A deserialized notification.
+    def self.from_json(json)
+      from_hash(JSON.parse(json))
+    end
+
+    # Assigns keys and values to this object based on an already-deserialized
+    # JSON object.
+    # @param [Hash] hash A hash of properties and their values.
+    # @return [Tincan::Message] A message.
+    def self.from_hash(hash)
+      instance = new do |i|
+        hash.each do |key, val|
+          val = DateTime.iso8601(val) if key == 'published_at'
+          i.send("#{key}=".to_sym, val)
+        end
+      end
+      instance
+    end
+
+    # Checks for proper change type and sets it if it's valid.
+    # @param [Symbol] value :create, :modify, or :delete.
+    def change_type=(value)
+      if %i(create modify delete).include?(value.to_sym)
+        @change_type = value.to_sym
+      else
+        fail ArgumentError, ':change_type must be :create, :modify or :delete'
+      end
+    end
+
+    # Generates a version of this notification as a JSON string.
+    # @return [String] A JSON-compliant marshalling of this instance's data.
+    def to_json(options = {})
+      # Note: at some point I may want to override how this is done. I think
+      # Rabl could definitely be of some use here.
+      Hash[%i(object_name change_type object_data published_at).map do |name|
+        [name, send(name)]
+      end].to_json(options)
+    end
+
+    # Object overrides
+
+    # Overrides equality operator to determine if all ivars are equal
+    def ==(other)
+      false unless other
+      checks = %i(object_name change_type object_data published_at).map do |p|
+        other.send(p) == send(p)
+      end
+      checks.all?
+    end
+
+    def to_s
+      vars = instance_variables.map do |v|
+        "#{v}=#{instance_variable_get(v).inspect}"
+      end.join(', ')
+      "<#{self.class}: #{vars}>"
+    end
+  end
+end

data/lib/tincan/receiver.rb

@@ -0,0 +1,167 @@
+require 'tincan/failure'
+require 'tincan/message'
+require 'redis'
+
+module Tincan
+  # An object whose purpose is to listen to a variety of Redis queues and fire
+  # off notifications when triggered.
+  class Receiver
+    attr_reader :config
+    attr_accessor :client_name, :listen_to, :redis_host, :redis_port,
+                  :namespace, :on_exception, :logger
+
+    # Lifecycle methods
+
+    # Creates and return a listener object, ready to listen. You can pass in
+    # either a hash or a block; the block takes priority.
+    # @param [Hash] options A list of keys/values to assign to this instance.
+    # @return [Tincan::Receiver] Self.
+    def initialize(options = {})
+      if block_given?
+        yield(self)
+      else
+        @config = options
+        ivars =  %i(client_name listen_to redis_host redis_port namespace
+                    on_exception logger)
+        ivars.each { |n| send("#{n}=".to_sym, @config[n]) }
+      end
+      self.redis_port ||= 6379
+    end
+
+    # Related objects
+
+    # The instance of a Redis communicator that can subscribe messages.
+    # @return [Redis] The Redis client used by this object.
+    def redis_client
+      @redis_client ||= ::Redis.new(host: redis_host, port: redis_port)
+    end
+
+    # Transactional (submission) methods
+
+    # Registers this receiver against a Redis set based on the object name.
+    # Looks like "namespace:object_name:receivers".
+    # @return [Tincan::Receiver] Self.
+    def register
+      listen_to.keys.each do |object_name|
+        receiver_list_key = key_for_elements(object_name, 'receivers')
+        logger.info "Registered against Tincan set #{receiver_list_key}"
+        redis_client.sadd(receiver_list_key, client_name)
+      end
+      self
+    end
+
+    # Handles putting a message identifier into a failed "retries" list.
+    # @param [Integer] message_id The identifier of the failed message.
+    # @param [String] original_list The name of the originating list.
+    # @return [Integer] The number of failed entries in the same list.
+    def store_failed_message(message_id, original_list)
+      logger.warn "Storing failure #{message_id} for list #{original_list}"
+      failure = Failure.new(message_id, original_list)
+      store_failure(failure)
+    end
+
+    # Handles putting a message identifier into a failed "retries" list.
+    # @param [Tincan::Failure] failure The failure to store.
+    # @return [Integer] The number of failed entries in the same list.
+    def store_failure(failure)
+      error_list = failure.queue_name.gsub('messages', 'failures')
+      redis_client.rpush(error_list, failure.to_json)
+    end
+
+    # Message handling methods
+
+    # Iterates through stored lambdas for a given object, and passes the
+    # message to all of them.
+    # @param [String] object_name The object name gleamed from the list key.
+    # @param [Tincan::Message] message The Message generated from the JSON
+    #                          hash retrieved from Redis.
+    def handle_message_for_object(object_name, message)
+      logger.debug "Encountered #{object_name} message: #{message.object_data}"
+      listen_to[object_name.to_sym].each do |stored_lambda|
+        stored_lambda.call(message)
+      end
+    end
+
+    # Loop methods
+
+    # Registers and subscribes. That is all.
+    def listen
+      register
+      subscribe
+    end
+
+    # Formatting and helper methods
+
+    # Asks the instance of Redis for the proper JSON data for a message, and
+    # then turns that into a Tincan::Message.
+    # @param [Integer] message_id The numerical ID of the message to retrieve.
+    # @param [String] object_name The object name of the message to retrieve;
+    #                 this helps the receiver determine which list it was
+    #                 posted to.
+    # @return [Tincan::Message] An initialized Message, or nil if not found.
+    def message_for_id(message_id, object_name)
+      key = key_for_elements(object_name, 'messages', message_id)
+      json = redis_client.get(key)
+      return nil unless json
+      Message.from_json(json)
+    end
+
+    # A flattened list of message list keys, in the format of
+    # "namespace:object_name:client:messages".
+    # @return [Array] An array of object_names formatted with client name, like
+    #                 "namespace:object_name:client:messages".
+    def message_list_keys
+      @message_list_keys ||= listen_to.keys.map do |object_name|
+        %w(messages failures).map do |type|
+          key_for_elements(object_name, client_name, type)
+        end
+      end.flatten
+    end
+
+    private
+
+    # Loops on a blocking pop call to a series of message lists on Redis and
+    # yields a given block when triggered. Handles turning JSON back into
+    # notification messages. Uses `BLPOP` as a blocking pop, indefinitely,
+    # until we get a message.
+    def subscribe
+      logger.info 'Awaiting new messages from Tincan.'
+      loop do
+        begin
+          message_list, content = redis_client.blpop(message_list_keys)
+          object_name = message_list.split(':')[1]
+          message_type = message_list.split(':').last
+
+          if message_type == 'messages'
+            message = message_for_id(content, object_name)
+          elsif message_type == 'failures'
+            failure = Failure.from_json(content)
+            if failure.attempt_after > DateTime.now
+              store_failure(failure)
+              next
+            end
+            message = message_for_id(failure.message_id, object_name)
+          end
+
+          handle_message_for_object(object_name, message) if message
+        rescue Interrupt
+          logger.warn 'Encountered interrupt.'
+          raise
+        rescue Exception => e
+          logger.warn "Encountered exception #{e}."
+          on_exception.call(e, {}) if on_exception
+          next unless content
+          failure ||= Failure.new(content, message_list)
+          failure.attempt_count += 1
+          store_failure(failure)
+        end
+      end
+    end
+
+    # Joins a variadic set of elements along with a namespace with colons.
+    # @return [String] A joined string to be used as a Redis key.
+    def key_for_elements(*elements)
+      ([namespace] + elements).join(':')
+    end
+  end
+end

data/lib/tincan/sender.rb

@@ -0,0 +1,101 @@
+require 'tincan/message'
+require 'redis'
+
+module Tincan
+  # An object whose purpose is to send messages to a given series of Redis
+  # message queues for those receiving them.
+  class Sender
+    attr_reader :config
+    attr_accessor :redis_host, :redis_port, :namespace
+
+    # Lifecycle methods
+
+    # Creates and return a sender object, ready to send. You can pass in
+    # either a hash or a block; the block takes priority.
+    # @param [Hash] options A list of keys/values to assign to this instance.
+    # @return [Tincan::Receiver] Self.
+    def initialize(options = {})
+      if block_given?
+        yield(self)
+      else
+        @config = options
+        ivars =  %i(redis_host redis_port namespace)
+        ivars.each { |n| send("#{n}=".to_sym, @config[n]) }
+      end
+      self.redis_port ||= 6379
+    end
+
+    # Related objects
+
+    # The instance of a Redis communicator that can publish messages.
+    # @return [Redis] The Redis client used by this object.
+    def redis_client
+      @redis_client ||= ::Redis.new(host: redis_host, port: redis_port)
+    end
+
+    # Transactional (lookup) methods
+
+    # Asks Redis for the set of all active receivers and generates string
+    # keys for all of them. Formatted like "namespace:object:client:messages".
+    # @return [Array] An array of keys identifying all receiver pointer lists.
+    def keys_for_receivers(object_name)
+      receiver_list_key = key_for_elements(object_name, 'receivers')
+      receivers = redis_client.smembers(receiver_list_key)
+      receivers.map do |receiver|
+        key_for_elements(object_name, receiver, 'messages')
+      end
+    end
+
+    # Tells Redis to delete any pending messages for registered receivers, by
+    # essentially deleting the message key.
+    # @return [Boolean] True, just because.
+    def flush_all_queues_for_object(object_name)
+      keys_for_receivers(object_name).each do |key|
+        redis_client.del(key)
+      end
+      true
+    end
+
+    # Communication methods
+
+    # Bundles up an object in a message object and publishes it to the Redis
+    # host.
+    # @param [Tincan::Message] message The message to publish.
+    # @param [Symbol] change_type :create, :modify, or :delete.
+    # @return [Boolean] true if the operation was a success.
+    def publish(message)
+      identifier = identifier_for_message(message)
+      redis_client.set(primary_key_for_message(message), message.to_json)
+      keys_for_receivers(message.object_name.downcase).each do |key|
+        redis_client.rpush(key, identifier)
+      end
+      true
+    end
+
+    # Formatting and helper methods
+
+    # Generates an identifier to be used for a message. It's unique!
+    # @param [Tincan::Message] message The message for which to generate a
+    #                          unique identifier.
+    # @return [Integer] A unique identifier number for the message.
+    def identifier_for_message(message)
+      message.published_at.to_time.to_i
+    end
+
+    # Generates a key to be used as the primary destination key in Redis.
+    # @param [Tincan::Message] message The message to use in key generation.
+    # @return [String] A properly-formatted key to be used with Redis.
+    def primary_key_for_message(message)
+      identifier = identifier_for_message(message)
+      key_for_elements(message.object_name.downcase, 'messages', identifier)
+    end
+
+    private
+
+    # Joins a variadic set of elements along with a namespace with colons.
+    # @return [String] A joined string to be used as a Redis key.
+    def key_for_elements(*elements)
+      ([namespace] + elements).join(':')
+    end
+  end
+end

data/lib/tincan/version.rb

@@ -0,0 +1,3 @@
+module Tincan
+  VERSION = '0.7.0'
+end

data/license.markdown

@@ -0,0 +1,22 @@
+Copyright (c) 2014 CaptainU, LLC.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/readme.markdown

@@ -0,0 +1,120 @@
+# Tincan
+
+[![Build Status](https://travis-ci.org/captainu/tincan.svg?branch=master)](https://travis-ci.org/captainu/tincan) [![Code Climate](https://codeclimate.com/github/captainu/tincan.png)](https://codeclimate.com/github/captainu/tincan) [![Code Climate](https://codeclimate.com/github/captainu/tincan/coverage.png)](https://codeclimate.com/github/captainu/tincan)
+
+Provides an easy way to register senders and receivers on a reliable Redis message queue, to be used in lieu of Redis's own pub/sub commands (which are connection-reliant). This uses Redis's lists and sets using a defined and namespaced series of keys that allows for a *sender* to publish structured notification messages to a Redis server, which then get referenced in multiple receiver-specific lists, all of which are being watched by a client running a blocking pop (Redis's `BLPOP` command). These clients, known as *receivers*, handle the messages and route them to any number of custom-defined Ruby lambdas.
+
+This is a Ruby implementation of [David Marquis's outstanding post](http://davidmarquis.wordpress.com/2013/01/03/reliable-delivery-message-queues-with-redis/) about reliable delivery message queues with Redis. It borrows heavily from the amazing [Sidekiq](https://github.com/mperham/sidekiq)'s design regarding its command-line and deployment interface.
+
+See below for some usage examples (more coming soon).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+``` ruby
+gem 'captainu-tincan', github: 'captainu/tincan', require: 'tincan'
+```
+
+And then execute:
+
+``` bash
+$ bundle install
+```
+
+## Usage
+
+### Sender
+
+``` ruby
+sender = Tincan::Sender.new do |config|
+  config.redis_host = 'localhost'
+  config.namespace = 'tincan' # Or whatever you'd like
+end
+
+# some_object here is something that responds to #to_json
+sender.publish(some_object, :create)
+```
+
+In order to reset a list of messages for all listening receivers, initiate a sender (like demonstrated above), and perform the following (where `some_object` is the singlar, underscored-and-downcased version of the name of the Class for which to nuke messages).
+
+``` ruby
+sender.flush_all_queues_for_object('some_object')
+```
+
+### Receiver
+
+``` ruby
+receiver = Tincan::Receiver.new do |config|
+  config.redis_host = 'localhost'
+  config.client_name = 'my-receiver'
+  config.namespace = 'tincan' # Same as above.
+  config.logger = ::Logger.new(STDOUT)
+  config.listen_to = {
+    college: [
+      ->(data) { SomeThing.handle_data(data) },
+      ->(data) { SomeOtherThing.handle_same_data(data) }
+    ],
+    college_team: [
+      -> (data) { AnotherThing.handle_this_data(data) }
+    ]
+  }
+  config.on_exception = lambda do |ex, context|
+    Airbrake.notify_or_ignore(ex, parameters: context)
+  end
+end
+
+# This call blocks and loops
+receiver.listen
+```
+
+#### Rails integration for Receiver
+
+Drop a `tincan.yml` file in your Rails project at `config/tincan.yml`, and make it look like this!
+
+``` yaml
+defaults: &defaults
+  redis_host: localhost
+  listen_to:
+    college:
+      - SomeModel.update_from_tincan
+    college_team:
+      - SomeOtherModel.update_from_tincan
+
+development:
+  <<: *defaults
+  namespace: tincan-development
+  client_name: your_app-dev
+
+test:
+  <<: *defaults
+  namespace: tincan-test
+  client_name: your_app-test
+
+production:
+  <<: *defaults
+  namespace: tincan
+  client_name: your_app
+```
+
+Then, a command-line `tincan` worker is just a few keystrokes away.
+
+``` bash
+$ bundle exec tincan
+```
+
+You can even daemonize it with `-d`. The command-line library is largely modeled after [Sidekiq](https://github.com/mperham/sidekiq), and is currently in dire need of some tests. Use at your own risk.
+
+For integration with Capistrano, see [capistrano-tincan](https://github.com/captainu/capistrano-tincan).
+
+## Contributing
+
+1. [Fork it](https://github.com/captainu/tincan/fork)!
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new pull request
+
+## Contributors
+
+- [Ben Kreeger](https://github.com/kreeger)