ruote-redis 2.2.0 → 2.3.0

data/CHANGELOG.txt

@@ -2,6 +2,16 @@
 = ruote-redis - CHANGELOG.txt
 
 
+== ruote-redis 2.3.0    released 2012/09/01
+
+- fix issue uncovered by Base#remove_process
+- fix update_rev (also update put_at)
+- Storage#reconnect (shortcut to @redis.reconnect)
+- further hardened Redis#keys_to_a
+- simplified storage initialization
+- using rpush and lpop for msgs (one del less per msg)
+
+
 == ruote-redis 2.2.0    released 2011/03/01
 
 - Ruote::Redis::Storage (instead of RedisStorage)

data/CREDITS.txt

@@ -15,5 +15,6 @@
 
 == many thanks to
 
-- the Redis team for their great project
+- the Redis authors and community
+- the redis.rb authors and community
 

data/LICENSE.txt

@@ -1,5 +1,5 @@
 
-Copyright (c) 2001-2011, John Mettraux, jmettraux@gmail.com
+Copyright (c) 2005-2012, John Mettraux, jmettraux@gmail.com
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.rdoc

@@ -21,7 +21,8 @@ The storage is instantiate by passing a redis-rb instance to the storage.
 
 Passing an em-redis instance might work, but I haven't tried.
 
-Tested with Redis 1.2.6 and 2.0.4 (redis-rb 2.1.1).
+Tested with Redis 2.0.1 (redis-rb 3.0.1) on Debian GNU/Linux.
+Tested with Redis 2.4.16 (redis-rb 3.0.1) on OSX snoleo.
 
 
 == running tests
@@ -31,20 +32,20 @@ assuming you have
   ruote/
   ruote-redis/
 
-start a redis server instance (port 6379) and then 
+start a redis server instance (port 6379) and then
 
 
 * unit tests :
 
 get into ruote/ and do
 
-  ruby test/unit/storage.rb -- --redis
+  RUOTE_STORAGE=redis ruby test/functional/storage.rb
 
 * functional tests :
 
 get into ruote/ and do
 
-  ruby test/functional/test.rb -- --redis
+  RUOTE_STORAGE=redis ruby test/functional/test.rb
 
 
 == license

data/Rakefile

@@ -5,7 +5,7 @@ require 'rubygems'
 
 require 'rake'
 require 'rake/clean'
-require 'rake/rdoctask'
+require 'rdoc/task'
 
 
 #

data/TODO.txt

@@ -6,4 +6,7 @@
 [x] 1 ::Redis instance per thread ? (instead of :thread_safe => true)
 
 [ ] resque participant ?
+[ ] sidekiq participant ?
+
+[ ] use flushdb
 

data/lib/ruote/redis/storage.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2005-2011, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2005-2012, John Mettraux, jmettraux@gmail.com
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files(the "Software"), to deal
@@ -22,8 +22,7 @@
 # Made in Japan.
 #++
 
-#require 'redis'
-  # now letting the end-user doing this require
+require 'redis'
 
 require 'rufus-json'
 require 'ruote/storage/base'
@@ -37,7 +36,7 @@ module Redis
   # A Redis storage for ruote.
   #
   # The constructor accepts two arguments, the first one is a Redis instance
-  #( see http://github.com/ezmobius/redis-rb ), the second one is the classic
+  # ( see http://github.com/ezmobius/redis-rb ), the second one is the classic
   # ruote engine options( see
   # http://ruote.rubyforge.org/configuration.html#engine )
   #
@@ -47,8 +46,7 @@ module Redis
   #
   #   engine = Ruote::Engine.new(
   #     Ruote::Worker.new(
-  #       Ruote::Redis::RedisStorage.new(
-  #         ::Redis.new(:db => 14, :thread_safe => true), {})))
+  #       Ruote::Redis::RedisStorage.new('db'=> 14, 'thread_safe' => true)))
   #
   #
   # == em-redis
@@ -60,49 +58,129 @@ module Redis
   # If you try and it works, feedback is welcome
   # http://groups.google.com/group/openwferu-users
   #
+  #
+  # == 'pop_count' option
+  #
+  # By default, when the worker queries this storage for msgs to process,
+  # the storage will try to pop 28 msgs. This number can be changed thanks
+  # to the 'pop_count' option, like in:
+  #
+  #   engine = Ruote::Engine.new(
+  #     Ruote::Worker.new(
+  #       Ruote::Redis::RedisStorage.new(
+  #         'db'=> 14, 'thread_safe' => true, 'pop_count' => 56)))
+  #
+  # Don't put too high a number, it increases the chance of msgs getting lost
+  # in case of the worker going down.
+  #
+  # (if there is a need to avoid such a scenario in the future,
+  # Redis' rpoplpush might come in handy).
+  #
   class Storage
 
     include Ruote::StorageBase
 
     attr_reader :redis
 
+    # Listing the redis options to differentiate them from ruote storage
+    # options.
+    #
+    REDIS_OPTIONS = %w[ host port path db thread_safe logger ]
+
     # A Redis storage for ruote.
     #
+    # Can be initialized in two ways
+    #
+    #   Ruote::Redis::Storage.new(
+    #     ::Redis.new(
+    #       :host => '127.0.0.1',
+    #       :db => 13,
+    #       :thread_safe => true))
+    #
+    # or
+    #
+    #   Ruote::Redis::Storage.new(
+    #     'host' => '127.0.0.1',
+    #     'db' => 13,
+    #     'thread_safe' => true)
+    #
+    # The first style is probably better avoided.
+    #
     def initialize(redis, options={})
 
+      if options == {} && redis.is_a?(Hash)
+
+        redis_options, options = redis.partition { |k, v|
+          REDIS_OPTIONS.include?(k.to_s)
+        }
+
+        redis_options = Hash[redis_options.collect { |k, v| [ k.to_sym, v ] }]
+        options = Hash[options]
+
+        redis = ::Redis.new(redis_options)
+      end
+
       @redis = redis
       @options = options
 
-      def @redis.keys_to_a(opt)
-        keys(opt) rescue []
+      @pop_count = @options['pop_count'] || 28
+
+      # Returns an array of the (String) keys that match the given pattern.
+      #
+      # Returns an empty array if anything goes wrong.
+      #
+      def @redis.keys_to_a(pattern)
+
+        if (a = (keys(pattern) rescue nil)).is_a?(Array)
+          a
+        else
+          []
+        end
       end
 
-      put_configuration
+      replace_engine_configuration(options)
     end
 
     # Returns true if the doc is successfully deleted.
     #
     def reserve(doc)
 
+      return true if doc['type'] == 'msgs'
+
       (@redis.del(key_for(doc)) == 1)
     end
 
     def put_msg(action, options)
 
       doc = prepare_msg_doc(action, options)
+      doc['put_at'] = Ruote.now_to_utc_s
 
-      @redis.set(key_for(doc), to_json(doc))
+      @redis.lpush('msgs', Rufus::Json.encode(doc))
 
       nil
     end
 
+    # Note: the worker argument is not used in this storage implementation.
+    #
+    def get_msgs
+
+      @redis.pipelined {
+        @pop_count.times { @redis.rpop('msgs') }
+      }.compact.collect { |d|
+        from_json(d)
+      }
+    end
+
     def put_schedule(flavour, owner_fei, s, msg)
 
       doc = prepare_schedule_doc(flavour, owner_fei, s, msg)
 
       return nil unless doc
 
-      @redis.set(key_for(doc), to_json(doc))
+      doc['_rev'] = '0'
+      doc['put_at'] = Ruote.now_to_utc_s
+
+      @redis.set(key_for(doc), Rufus::Json.encode(doc))
 
       doc['_id']
     end
@@ -141,9 +219,11 @@ module Redis
           #
           # put is successful (return nil)
           #
-          nrev = (rev.to_i + 1).to_s
-          @redis.set(key, to_json(doc.merge('_rev' => nrev)))
-          doc['_rev'] = nrev if opts[:update_rev]
+          doc = doc.send(
+            opts[:update_rev] ? :merge! : :merge,
+            { '_rev' => (rev.to_i + 1).to_s, 'put_at' => Ruote.now_to_utc_s })
+
+          @redis.set(key, Rufus::Json.encode(doc))
 
           nil
         end
@@ -204,7 +284,9 @@ module Redis
 
       elsif keys.first.is_a?(String)
 
-        keys.collect { |k| @redis.keys_to_a("#{type}/*!#{k}") }.flatten
+        keys.collect { |k|
+          @redis.keys_to_a("#{type}/*!#{k}#{type == 'schedules' ? '-*' : ''}")
+        }.flatten
 
       else #if keys.first.is_a?(Regexp)
 
@@ -225,13 +307,13 @@ module Redis
       limit = opts[:limit] || ids.length
       ids = ids[skip, limit]
 
-      docs = ids.length > 0 ? @redis.mget(*ids) : []
-      docs = docs.inject({}) do |h, doc|
-        if doc
-          doc = Rufus::Json.decode(doc)
-          h[doc['_id']] = doc
-        end
-        h
+      docs = ids.length > 0 && @redis.mget(*ids)
+      docs = docs.is_a?(Array) ? docs : []
+
+      docs = docs.each_with_object({}) do |doc, h|
+        next unless doc
+        doc = Rufus::Json.decode(doc)
+        h[doc['_id']] = doc
       end
 
       return docs.size if opts[:count]
@@ -252,15 +334,8 @@ module Redis
 
     def purge!
 
-      @redis.keys_to_a('*').each { |k| @redis.del(k) }
-    end
-
-    # Returns a String containing a representation of the current content of
-    # in this Redis storage.
-    #
-    def dump(type)
-
-      @redis.keys_to_a("#{type}/*").sort.join("\n")
+      2.times { @redis.flushdb rescue nil }
+        # 2 times to work around Redis::ProtocolError '3'
     end
 
     # Shuts this worker down.
@@ -295,6 +370,13 @@ module Redis
       @redis.keys_to_a("#{type}/*").each { |k| (@redis.del(k) rescue nil) }
     end
 
+    # Simply calls @redis.reconnect
+    #
+    def reconnect
+
+      @redis.reconnect
+    end
+
     protected
 
     LOCK_KEY = /-lock$/
@@ -309,20 +391,18 @@ module Redis
 
       loop do
 
-        r = @redis.setnx(kl, Time.now.to_f.to_s)
+        break if @redis.setnx(kl, Time.now.to_f.to_s) != false
+          # locking successful
 
-        if r == false
+        #
+        # already locked
 
-          t = @redis.get(kl)
+        t = @redis.get(kl)
 
-          @redis.del(kl) if t && Time.now.to_f - t.to_f > 60.0
-            # after 1 minute, locks time out
-
-          sleep 0.007 # let's try to lock again after a while
-        else
+        @redis.del(kl) if t && Time.now.to_f - t.to_f > 60.0
+          # after 1 minute, locks time out
 
-          break # lock acquired
-        end
+        sleep 0.007 # let's try to lock again after a while
       end
 
       #@redis.expire(kl, 2)
@@ -354,23 +434,6 @@ module Redis
 
       s ? Rufus::Json.decode(s) : nil
     end
-
-    def to_json(doc, opts={})
-
-      Rufus::Json.encode(
-        opts[:delete] ? nil : doc.merge('put_at' => Ruote.now_to_utc_s))
-    end
-
-    # Don't put configuration if it's already in
-    #
-    # (prevent storages from trashing configuration...)
-    #
-    def put_configuration
-
-      return if get('configurations', 'engine')
-
-      put({ '_id' => 'engine', 'type' => 'configurations' }.merge(@options))
-    end
   end
 
   #

data/lib/ruote/redis/version.rb

@@ -1,8 +1,32 @@
+#--
+# Copyright (c) 2005-2012, John Mettraux, jmettraux@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files(the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
 
 module Ruote
 module Redis
 
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end
 end
 

data/ruote-redis.gemspec

@@ -1,9 +1,12 @@
-# encoding: utf-8
 
 Gem::Specification.new do |s|
 
   s.name = 'ruote-redis'
-  s.version = File.read('lib/ruote/redis/version.rb').match(/VERSION = '([^']+)'/)[1]
+
+  s.version = File.read(
+    File.expand_path('../lib/ruote/redis/version.rb', __FILE__)
+  ).match(/ VERSION *= *['"]([^'"]+)/)[1]
+
   s.platform = Gem::Platform::RUBY
   s.authors = [ 'John Mettraux' ]
   s.email = [ 'jmettraux@gmail.com' ]
@@ -12,7 +15,7 @@ Gem::Specification.new do |s|
   s.summary = 'Redis storage for ruote (a Ruby workflow engine)'
   s.description = %q{
 Redis storage for ruote (a Ruby workflow engine)
-}
+  }
 
   #s.files = `git ls-files`.split("\n")
   s.files = Dir[
@@ -21,7 +24,7 @@ Redis storage for ruote (a Ruby workflow engine)
     '*.gemspec', '*.txt', '*.rdoc', '*.md'
   ]
 
-  s.add_runtime_dependency 'redis', '2.1.1'
+  s.add_runtime_dependency 'redis'#, '>= 2.2.2'
   s.add_runtime_dependency 'ruote', ">= #{s.version}"
 
   s.add_development_dependency 'rake'

data/test/bm/loaded.rb

@@ -0,0 +1,38 @@
+
+$: << File.expand_path('../../../lib', __FILE__)
+
+require 'test/unit'
+
+require 'rufus-json/automatic'
+require 'ruote'
+require 'ruote-redis'
+
+ruote =
+  Ruote::Dashboard.new(
+    Ruote::Worker.new(
+      Ruote::Redis::Storage.new('db' => 15, 'thread_safe' => true)))
+
+ruote.storage.purge!
+#ruote.noisy = true
+
+ruote.register 'toto' do |workitem|
+  workitem.fields['seen'] = Time.now.to_s
+end
+
+t = Time.now
+
+50.times do
+  wfid = ruote.launch(Ruote.define do
+    toto; toto; toto; toto; toto
+  end)
+  p wfid
+end
+
+loop do
+  count = ruote.storage.get_many('expressions', nil, :count => true)
+  p count
+  break if count < 1
+end
+
+p Time.now - t
+

data/test/bughunt/launch_gun.rb

@@ -0,0 +1,75 @@
+
+#
+# testing ruote-redis
+#
+# Started in Narita, finished in Santa Barbara
+#
+# Fri Feb  3 14:47:04 JST 2012
+#
+
+$:.unshift(File.expand_path('../../../../ruote/lib', __FILE__))
+$:.unshift(File.expand_path('../../../lib', __FILE__))
+
+require 'rubygems'
+
+require 'pp'
+
+require 'rufus-json/automatic'
+require 'redis'
+
+require 'ruote'
+#require 'ruote/storage/fs_storage'
+require 'ruote-redis'
+
+N = 1000
+
+$dash = Ruote::Dashboard.new(
+  Ruote::Worker.new(
+    Ruote::Redis::Storage.new(
+      'db' => 13, 'thread_safe' => true)))
+    #Ruote::HashStorage.new))
+    #Ruote::FsStorage.new('work')))
+
+p $dash.storage_participant.size
+$dash.storage.purge!
+#p $dash.processes.size
+p $dash.storage_participant.size
+
+#exit 0
+
+$dash.register 'toto', Ruote::StorageParticipant
+
+$pdef = Ruote.define do
+  sequence do
+    sequence do
+      toto
+    end
+  end
+end
+
+$wfids = []
+
+N.times do |i|
+  $wfids << $dash.launch($pdef)
+  print (i % 10 == 0) ? i : '.'; STDOUT.flush
+  #sleep 0.001 # jams...
+  #sleep 0.005
+end
+
+puts
+puts 'launch done'
+
+t = Time.now
+
+while (n = $dash.storage_participant.size) < N
+  sleep 0.050
+  p n if (Time.now - t) > 60.0
+  break if (Time.now - t) > 2 * 60.0
+end
+
+puts "took #{Time.now - t}s"
+p $dash.storage_participant.size
+
+$dash.shutdown
+$dash.storage.purge!
+

data/test/bughunt/put_gun.rb

@@ -0,0 +1,36 @@
+
+$:.unshift(File.expand_path('../../../../ruote/lib', __FILE__))
+$:.unshift(File.expand_path('../../../lib', __FILE__))
+
+require 'rubygems'
+
+require 'pp'
+
+require 'redis'
+
+N = 10_000
+
+redis = ::Redis.new('db' => 12, 'thread_safe' => true)
+
+p redis.del('put_gun')
+
+N.times do
+  msg = (Time.now.to_f.to_s + '__') * 50
+  redis.rpush('put_gun', msg)
+end
+
+p redis.llen('put_gun')
+
+results = []
+
+loop do
+  x = redis.lpop('put_gun')
+  break unless x
+  results << x
+end
+
+p results.size
+p results.sort.uniq.size
+
+# it's always N, redis is great
+

data/test/{functional_connection.rb → connection.rb}

@@ -21,11 +21,13 @@ class RrLogger
 end
 
 
-def new_storage (opts)
+def new_storage(opts)
+
+  #Ruote::Redis::Storage.new(
+  #  ::Redis.new(:db => 14, :thread_safe => true, :logger => RrLogger.new),
+  #  opts)
 
   Ruote::Redis::Storage.new(
-    ::Redis.new(:db => 14, :thread_safe => true),
-    #::Redis.new(:db => 14, :thread_safe => true, :logger => RrLogger.new),
-    opts)
+    opts.merge('db' => 14, 'thread_safe' => true, 'pop_count' => 28))
 end
 

metadata

@@ -1,119 +1,108 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ruote-redis
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 2
-  - 2
-  - 0
-  version: 2.2.0
+version: !ruby/object:Gem::Version
+  version: 2.3.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - John Mettraux
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-03-01 00:00:00 +09:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-09-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: redis
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 1
-        - 1
-        version: 2.1.1
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: ruote
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruote
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 2
-        - 0
-        version: 2.2.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-description: "\n\
-  Redis storage for ruote (a Ruby workflow engine)\n"
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "\nRedis storage for ruote (a Ruby workflow engine)\n  "
+email:
 - jmettraux@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - Rakefile
 - lib/ruote/redis/storage.rb
 - lib/ruote/redis/version.rb
 - lib/ruote/redis.rb
 - lib/ruote-redis.rb
-- test/functional_connection.rb
+- test/bm/loaded.rb
+- test/bughunt/launch_gun.rb
+- test/bughunt/put_gun.rb
+- test/connection.rb
 - ruote-redis.gemspec
 - CHANGELOG.txt
 - CREDITS.txt
 - LICENSE.txt
 - TODO.txt
 - README.rdoc
-has_rdoc: true
 homepage: http://ruote.rubyforge.org
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: ruote
-rubygems_version: 1.3.7
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Redis storage for ruote (a Ruby workflow engine)
 test_files: []
-