insque 0.3.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in insque.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 gropher
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,100 @@
+# Insque
+
+Instant queue. Background processing and message driven communication tool. Faster and simplier alternative to Resque.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'insque', :git => 'https://github.com/Gropher/Insque.git'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install insque
+
+## Usage
+
+At first you need to generate initializer and redis config file. Pass your sender name as parameter. 
+Sender name is the unique identifier of your instance of insque. You can use several instances of insque to create message driven communication system 
+
+    $ rails g insque:initializer somesender
+
+### Sending
+
+To broadcast message use:
+```ruby
+Insque.broadcast :message_name, {:params => {:some => :params}}
+```
+There is an easy way to use insque as background jobs processing. You can use `send_later` method to call any method of your rails models in background
+. You still need listener running to make this work.
+```ruby
+@model = MyModel.first
+@model.send_later :mymethod, 'some', 'params'
+```
+
+### Recieving
+
+To start recieving messages you need to:
+
+1. Create handler method in Insque module. First part of handler name is the name of the message sender.
+```ruby
+def somesender_message_name message
+  #TODO: Handle message somehow
+end
+```
+
+2. Call `listen` method in some background process or rake task:
+```ruby
+Insque.listen
+```
+
+   or just run `bundle exec rake insque:listener` from your console.
+
+3. Call `janitor` method in some background process or rake task. Janitor will reissue failed messages or report error if message fails again. Janitor treats message as failed if it was not processed for an hour after broadcast or reissue.
+```ruby
+Insque.janitor
+```
+
+   or just run `bundle exec rake insque:janitor` from your console.
+
+### Daemonizing
+
+If you want to run insque listener as a daemon consider using [foreman](https://github.com/ddollar/foreman) for this.
+
+Add foreman to your `Gemfile`:
+
+    gem 'foreman-capistrano'
+
+Install it:
+
+    $ bundle install  
+
+Create `Procfile`:
+
+    listener: bundle exec rake insque:listener
+    janitor: bundle exec rake insque:janitor
+
+Run foreman from your console:
+    
+    $ bundle exec foreman start
+
+For production use modify your capistrano deploy script somewhat like this:
+    
+    set :default_environment, {'PATH' => "/sbin:$PATH"}  # Optional. Useful if you get errors because start or restart command not found
+    set :foreman_concurrency, "\"listener=4,janitor=2\"" # How many processes of each type do you want
+    require 'foreman/capistrano'
+    
+    after "deploy", "foreman:export"              # Export Procfile as a set of upstart jobs
+    after "deploy", "foreman:restart"             # You will need upstart installed on your server for this to work.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+

data/insque.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/insque/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["gropher"]
+  gem.email         = ["grophen@gmail.com"]
+  gem.description   = "Instant queue. Background processing and message driven communication tool. Faster and simplier alternative to Resque."
+  gem.summary       = "Instant queue. Background processing and message driven communication tool. Faster and simplier alternative to Resque."
+  gem.homepage      = "https://github.com/Gropher/Insque"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "insque"
+  gem.require_paths = ["lib"]
+  gem.version       = Insque::VERSION
+  
+  gem.add_dependency('redis', '>= 2.2.2' )
+end

data/lib/generators/insque/initializer_generator.rb

@@ -0,0 +1,18 @@
+module Insque
+  module Generators
+    class InitializerGenerator < ::Rails::Generators::Base
+      argument :sender_name, :type => :string
+      
+      desc 'Create a sample Insque initializer and redis config file'
+      
+      def self.source_root
+        @source_root ||= File.join(File.dirname(__FILE__), 'templates')
+      end
+
+      def create_initializer
+        template 'insque.erb', 'config/initializers/insque.rb'
+        template 'redis.yml', 'config/redis.yml'
+      end
+    end
+  end
+end

data/lib/generators/insque/templates/insque.erb

@@ -0,0 +1,13 @@
+Insque.sender = '<%= sender_name %>'
+REDIS_CONFIG = YAML.load(File.open("#{Rails.root}/config/redis.yml"))[Rails.env]
+$redis = Redis.new(:host => REDIS_CONFIG["host"], :port => REDIS_CONFIG["port"])
+Insque.redis = $redis
+Insque.debug = true
+if defined?(PhusionPassenger)
+  PhusionPassenger.on_event(:starting_worker_process) do |forked|
+    if forked
+      $redis = Redis.new(:host => REDIS_CONFIG["host"], :port => REDIS_CONFIG["port"])
+      Insque.redis = $redis
+    end
+  end
+end

data/lib/generators/insque/templates/redis.yml

@@ -0,0 +1,9 @@
+development:
+  host: localhost
+  port: 6379
+production:
+  host: localhost
+  port: 6379
+test:
+  host: localhost
+  port: 6379

data/lib/insque/railtie.rb

@@ -0,0 +1,8 @@
+require 'insque'
+require 'rails'
+
+module Insque
+  class Railtie < Rails::Railtie
+    rake_tasks { load 'tasks/insque.rake' }
+  end
+end

data/lib/insque/version.rb

@@ -0,0 +1,3 @@
+module Insque
+  VERSION = "0.3.0"
+end

data/lib/insque.rb

@@ -0,0 +1,122 @@
+require "insque/version"
+require "redis"
+require "insque/railtie" if defined?(Rails)
+
+module Insque
+  def self.debug= debug
+    @debug = debug
+  end
+
+  def self.redis= redis
+    @redis = redis
+    @redis.select 7
+  end
+
+  def self.sender= sender
+    @sender = sender
+    @inbox = "insque_inbox_#{sender}"
+    @processing = "insque_processing_#{sender}"
+    create_send_later_handler
+  end
+
+  def self.broadcast message, params = nil, recipient = :any
+    keys = []
+    case recipient
+    when :any
+      keys = @redis.smembers 'insque_inboxes'
+    when :self
+      keys = [@inbox]
+    else
+      keys = recipient.is_a?(Array) ? recipient : [recipient]
+    end
+    value = { :message => "#{@sender}_#{message}", :params => params, :broadcasted_at => Time.now.utc }
+    log "SENDING: #{value.to_json} TO #{keys.to_json}" if @debug
+    @redis.multi do |r|
+      keys.each {|k| r.lpush k, value.to_json}
+    end
+  end
+
+  def self.listen worker_name=''
+    @redis.sadd 'insque_inboxes', @inbox
+    log "#{worker_name} START LISTENING #{@inbox}"
+    loop do
+      message = @redis.brpoplpush(@inbox, @processing, 0)
+      log "#{worker_name} RECEIVING: #{message}" if @debug
+      begin
+        parsed_message = JSON.parse message
+        send(parsed_message['message'], parsed_message) if self.respond_to? parsed_message['message']
+      rescue => e
+        log "#{worker_name} ========== BROKEN_MESSAGE: #{message} =========="
+        log e.inspect
+        log e.backtrace
+      end
+      @redis.lrem @processing, 0, message
+    end
+  end
+
+  def self.janitor
+    loop do
+      @redis.watch @processing
+      errors = []
+      restart = []
+      delete = []
+      @redis.lrange(@processing, 0, -1).each do |m|
+        begin
+          parsed_message = JSON.parse(m)
+          if parsed_message['restarted_at'] && DateTime.parse(parsed_message['restarted_at']) < 1.hour.ago.utc
+            errors << parsed_message 
+            delete << m
+          elsif DateTime.parse(parsed_message['broadcasted_at']) < 1.hour.ago.utc
+            restart << parsed_message.merge(:restarted_at => Time.now.utc)
+            delete << m
+          end
+        rescue
+          log "========== JANITOR_BROKEN_MESSAGE: #{m} =========="
+        end
+      end
+      result = @redis.multi do |r|
+        restart.each {|m| r.lpush @inbox, m.to_json }
+        delete.each {|m| r.lrem @processing, 0, m }
+      end
+      if result
+        errors.each {|m| log "ERROR: #{m.to_json}" }
+        restart.each {|m| log "RESTART: #{m.to_json}" }
+        log "CLEANING SUCCESSFULL"
+      else
+        log "CLEANING FAILED"
+      end
+      sleep(Random.rand * 10)
+    end
+  end
+
+  private
+  def self.log message
+    print "#{Time.now.utc} #{message}\n"
+    STDOUT.flush if @debug
+  end
+
+  def self.create_send_later_handler
+    define_singleton_method("#{@sender}_send_later") do |msg|
+      Kernel.const_get(msg['params']['class']).find(msg['params']['id']).send(msg['params']['method'], *msg['params']['args'])      
+    end
+  end
+end
+
+class ActiveRecord::Base
+  def send_later(method, *args)
+    Insque.broadcast :send_later, {:class => self.class.name, :id => id, :method => method, :args => args }, :self
+  end
+  
+  def self.acts_as_insque_crud(*args)
+    options = args.extract_options!
+    excluded = (options[:exclude] || []).map(&:to_s)
+
+    [:update, :create, :destroy].each do |action|
+      set_callback action, :after do
+        params = self.serializable_hash(options).delete_if {|key| (['created_at', 'updated_at'] + excluded).include? key}
+        Insque.broadcast :"#{self.class.to_s.underscore}_#{action}", params
+      end
+    end
+  end
+
+end

data/lib/tasks/insque.rake

@@ -0,0 +1,13 @@
+namespace :insque do
+  desc 'Starts insque listener'
+  task :listener => :environment do
+    trap('TERM') { puts "SIGTERM"; exit 0 }
+    Insque.listen
+  end
+
+  desc 'Starts insque janitor'
+  task :janitor => :environment do
+    trap('TERM') { puts "SIGTERM"; exit 0 }
+    Insque.janitor
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: insque
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+  prerelease: 
+platform: ruby
+authors:
+- gropher
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+description: Instant queue. Background processing and message driven communication
+  tool. Faster and simplier alternative to Resque.
+email:
+- grophen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- insque.gemspec
+- lib/generators/insque/initializer_generator.rb
+- lib/generators/insque/templates/insque.erb
+- lib/generators/insque/templates/redis.yml
+- lib/insque.rb
+- lib/insque/railtie.rb
+- lib/insque/version.rb
+- lib/tasks/insque.rake
+homepage: https://github.com/Gropher/Insque
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Instant queue. Background processing and message driven communication tool.
+  Faster and simplier alternative to Resque.
+test_files: []
+has_rdoc: