ruote-couch 2.1.9 → 2.1.10

data/CHANGELOG.txt

@@ -2,7 +2,15 @@
 = ruote-couch - CHANGELOG.txt
 
 
-== ruote-couch 2.1.9    not yet released
+== ruote-couch 2.1.10    released 2010/06/15
+
+- storage#ids not returning design doc ids anymore
+- now using CouchDB 0.11 continuous&all_docs
+- multi-worker hardened
+- adapted #query_workitems to ruote 2.1.10
+
+
+== ruote-couch 2.1.9    released 2010/03/22
 
 - trusting rufus-jig : not closing couches anymore
 - CouchStorage#clear and purge cleanup

data/README.rdoc

@@ -5,6 +5,8 @@ A storage implementation for ruote 2.1.x.
 
 Warning : this is a very naive implementation for now. Not many optimizations...
 
+Note : CouchDB >= 0.11
+
 
 == running tests
 

data/Rakefile

@@ -33,7 +33,7 @@ CouchDB storage for ruote 2.1 (ruby workflow engine)
   gem.test_file = 'test/test.rb'
 
   gem.add_dependency 'ruote', ">= #{Ruote::Couch::VERSION}"
-  gem.add_dependency 'rufus-jig', '>= 0.1.17'
+  gem.add_dependency 'rufus-jig', '>= 0.1.18'
   gem.add_development_dependency 'yard'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'jeweler'

data/TODO.txt

@@ -1,11 +1,14 @@
 
 [x] ct_2 --couch says put(gone) is returning nil... :(
 [o] ?feed=longpool&heartbeat=x (parallel thread or smarter scheduler ?)
+[o] remove re_put_ok (rufus-jig >= 0.1.18)
 
 [ ] view for ats
 [ ] view for crons
 
 [ ] jig / r-couch should not rely on exceptions
 
-[ ] issue with cron '* * * * *' and long-polling
+=================================================+
+[ ] issue with cron '* * * * *' and long-polling |
+=================================================+
 

data/lib/ruote/couch/database.rb

@@ -37,9 +37,12 @@ module Ruote::Couch
 
     attr_reader :type
 
-    def initialize (host, port, type, name, re_put_ok=true)
+    attr_reader :couch
 
-      opts = { :re_put_ok => re_put_ok }
+    def initialize (host, port, type, name)
+
+      #opts = { :re_put_ok => re_put_ok }
+      opts = {}
       #opts[:timeout] = TODO
 
       @couch = Rufus::Jig::Couch.new(host, port, name, opts)
@@ -69,7 +72,13 @@ module Ruote::Couch
 
     def delete (doc)
 
-      @couch.delete(doc)
+      r = @couch.delete(doc)
+
+      #p [ :del, doc['_id'], Thread.current.object_id.to_s[-3..-1], r.nil? ]
+      Thread.pass
+        # without this, test/functional/ct_0 fails after 1 to 10 runs...
+
+      r
     end
 
     def get_many (key, opts)
@@ -84,13 +93,15 @@ module Ruote::Couch
       rs.select { |doc| ! doc['_id'].match(/^\_design\//) }
     end
 
+    DESIGN_DOC_REGEX = /^\_design\//
+
     # Returns a sorted list of the ids of all the docs in this database.
     #
     def ids
 
       rs = @couch.get('_all_docs')
 
-      rs['rows'].collect { |r| r['id'] }
+      rs['rows'].collect { |r| r['id'] }.reject { |i| i.match(DESIGN_DOC_REGEX) }
     end
 
     def dump
@@ -280,10 +291,9 @@ module Ruote::Couch
 
       hwis = hwis.select { |hwi| hwi['fei']['wfid'] == wfid } if wfid
 
-      hwis = hwis.select { |hwi|
-        Ruote::StorageParticipant.matches?(hwi, pname, criteria) }
-
-      hwis.collect { |hwi| Ruote::Workitem.new(hwi) }
+      hwis.select { |hwi|
+        Ruote::StorageParticipant.matches?(hwi, pname, criteria)
+      }
     end
 
     # Returns the design document that goes with this class of database

data/lib/ruote/couch/storage.rb

@@ -22,6 +22,7 @@
 # Made in Japan.
 #++
 
+require 'thread'
 require 'ruote/storage/base'
 require 'ruote/couch/version'
 require 'ruote/couch/database'
@@ -63,7 +64,7 @@ module Couch
       @prefix = options['couch_prefix'] || options['prefix'] || ''
       @prefix = "#{@prefix}_" if @prefix.size > 0
 
-      @zeroes = 21 # maybe make it an option
+      #@zeroes = 21 # maybe make it an option
       @timeout = options['couch_timeout'] || 60
 
       @dbs = {}
@@ -78,14 +79,21 @@ module Couch
         @host, @port, 'errors', "#{@prefix}ruote_errors")
 
       @dbs['expressions'] = WfidIndexedDatabase.new(
-        @host, @port, 'expressions', "#{@prefix}ruote_expressions", false)
+        #@host, @port, 'expressions', "#{@prefix}ruote_expressions", false)
+        @host, @port, 'expressions', "#{@prefix}ruote_expressions")
 
       @dbs['workitems'] = WorkitemDatabase.new(
         @host, @port, 'workitems', "#{@prefix}ruote_workitems")
 
       put_configuration
 
-      @zero_msgs_offset = @zeroes
+      #@zero_msgs_offset = @zeroes
+      @msgs_thread = nil
+      @msgs_queue = ::Queue.new
+
+      @schedules_thread = nil
+      @schedules_queue = ::Queue.new
+      @schedules = nil
     end
 
     def put (doc, opts={})
@@ -142,9 +150,12 @@ module Couch
 
     def shutdown
 
-      @poller.kill if @poller
-
       #@dbs.values.each { |db| db.shutdown }
+
+      #@poller.kill if @poller
+
+      @msgs_thread.kill rescue nil
+      @schedules_thread.kill rescue nil
     end
 
     # Mainly used by ruote's test/unit/ut_17_storage.rb
@@ -152,7 +163,8 @@ module Couch
     def add_type (type)
 
       @dbs[type] = Database.new(
-        @host, @port, type, "#{@prefix}ruote_#{type}", false)
+        #@host, @port, type, "#{@prefix}ruote_#{type}", false)
+        @host, @port, type, "#{@prefix}ruote_#{type}")
     end
 
     # Nukes a db type and reputs it (losing all the documents that were in it).
@@ -193,54 +205,43 @@ module Couch
     #
     def get_msgs
 
-      if @zero_msgs_offset > 0
+      ensure_msgs_thread_is_running
 
-        msgs = get_many(
-          'msgs', nil, :limit => 300
-        ).sort { |a, b|
-          a['put_at'] <=> b['put_at']
-        }
+      msgs = []
 
-        @zero_msgs_offset = @zero_msgs_offset - 1 if msgs.size == 0
-        return msgs
+      while @msgs_queue.size > 0
+        msgs << @msgs_queue.pop
       end
 
-      @zero_msgs_offset = @zeroes
-
-      schedules = get_many('schedules')
-
-      next_at = schedules.collect { |s| s['at'] }.sort.first
-      delta = next_at ? (Time.parse(next_at) - Time.now) : nil
-
-      #p [ delta, @timeout ]
+      msgs
+    end
 
-      return [] if delta && delta < 5.0
+    def get_schedules (delta, now)
 
-      last_seq = @dbs['msgs'].get('_changes')['last_seq']
+      ensure_schedules_thread_is_running
 
-      timeout = delta ? delta - 3.0 : -1.0
-      timeout = (timeout < 0.0 || timeout > @timeout) ? @timeout : timeout
+      if @schedules.nil?
 
-      #p [ Time.now, :last_seq, last_seq, :timeout, timeout ]
+        # NOTE : the problem with this approach is that ALL the schedules
+        # are stored in memory. Most of the time it's not a problem, but
+        # for people will lots of schedules...
 
-      begin
+        @schedules = get_many('schedules')
+        @schedules = @schedules.inject({}) { |h, s| h[s['_id']] = s; h }
+      end
 
-        @poller = Thread.current
+      while @schedules_queue.size > 0
 
-        @dbs['msgs'].get(
-          "_changes?feed=longpoll&heartbeat=60000&since=#{last_seq}",
-          :timeout => timeout)
-            # block until there is a change in the 'msgs' db
+        deleted, s = @schedules_queue.pop
 
-      rescue Exception => e
-      #rescue Rufus::Jig::TimeoutError => te
-      #  p [ :caught, e.class ]
-      #  e.backtrace.each { |l| puts l }
-      ensure
-        @poller = nil
+        if deleted
+          @schedules.delete(s['_id'])
+        else
+          @schedules[s['_id']] = s
+        end
       end
 
-      []
+      filter_schedules(@schedules.values, now)
     end
 
     protected
@@ -253,6 +254,30 @@ module Couch
 
       put(conf)
     end
+
+    def ensure_msgs_thread_is_running
+
+      status = @msgs_thread ? @msgs_thread.status : -1
+      return if status == 'run' || status == 'sleep'
+
+      @msgs_thread = Thread.new do
+        @dbs['msgs'].couch.on_change do |_, deleted, doc|
+          @msgs_queue << doc unless deleted
+        end
+      end
+    end
+
+    def ensure_schedules_thread_is_running
+
+      status = @schedules_thread ? @schedules_thread.status : -1
+      return if status == 'run' || status == 'sleep'
+
+      @schedules_thread = Thread.new do
+        @dbs['schedules'].couch.on_change do |_, deleted, doc|
+          @schedules_queue << [ deleted, doc ]
+        end
+      end
+    end
   end
 end
 end

data/lib/ruote/couch/version.rb

@@ -1,7 +1,7 @@
 
 module Ruote
 module Couch
-  VERSION = '2.1.9'
+  VERSION = '2.1.10'
 end
 end
 

data/ruote-couch.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ruote-couch}
-  s.version = "2.1.9"
+  s.version = "2.1.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John Mettraux"]
-  s.date = %q{2010-03-23}
+  s.date = %q{2010-06-15}
   s.description = %q{CouchDB storage for ruote 2.1 (ruby workflow engine)}
   s.email = %q{jmettraux@gmail.com}
   s.extra_rdoc_files = [
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
      "test/functional/base.rb",
      "test/functional/ft_0_long_polling.rb",
      "test/functional/test.rb",
-     "test/integration_connection.rb",
+     "test/functional_connection.rb",
      "test/test.rb",
      "test/unit/test.rb",
      "test/unit/ut_0_design_doc.rb"
@@ -51,21 +51,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<ruote>, [">= 2.1.9"])
-      s.add_runtime_dependency(%q<rufus-jig>, [">= 0.1.17"])
+      s.add_runtime_dependency(%q<ruote>, [">= 2.1.10"])
+      s.add_runtime_dependency(%q<rufus-jig>, [">= 0.1.18"])
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
-      s.add_dependency(%q<ruote>, [">= 2.1.9"])
-      s.add_dependency(%q<rufus-jig>, [">= 0.1.17"])
+      s.add_dependency(%q<ruote>, [">= 2.1.10"])
+      s.add_dependency(%q<rufus-jig>, [">= 0.1.18"])
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
     end
   else
-    s.add_dependency(%q<ruote>, [">= 2.1.9"])
-    s.add_dependency(%q<rufus-jig>, [">= 0.1.17"])
+    s.add_dependency(%q<ruote>, [">= 2.1.10"])
+    s.add_dependency(%q<rufus-jig>, [">= 0.1.18"])
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])

data/test/{integration_connection.rb → functional_connection.rb}

@@ -17,7 +17,18 @@ end
 
 require 'ruote/couch/storage'
 
-# TODO : maybe place delete_all_test_databases here
+
+unless $_RUOTE_COUCH_CLEANED
+
+  couch = Rufus::Jig::Couch.new('127.0.0.1', 5984)
+  %w[
+    configurations errors expressions msgs schedules variables workitems
+  ].each do |type|
+    couch.delete("/test_ruote_#{type}")
+  end
+  puts "(purged all /test_ruote_xxx databases)"
+  $_RUOTE_COUCH_CLEANED = true
+end
 
 
 def new_storage (opts)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 2
   - 1
-  - 9
-  version: 2.1.9
+  - 10
+  version: 2.1.10
 platform: ruby
 authors: 
 - John Mettraux
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-23 00:00:00 +09:00
+date: 2010-06-15 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,8 +27,8 @@ dependencies:
         segments: 
         - 2
         - 1
-        - 9
-        version: 2.1.9
+        - 10
+        version: 2.1.10
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -41,8 +41,8 @@ dependencies:
         segments: 
         - 0
         - 1
-        - 17
-        version: 0.1.17
+        - 18
+        version: 0.1.18
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -105,7 +105,7 @@ files:
 - test/functional/base.rb
 - test/functional/ft_0_long_polling.rb
 - test/functional/test.rb
-- test/integration_connection.rb
+- test/functional_connection.rb
 - test/test.rb
 - test/unit/test.rb
 - test/unit/ut_0_design_doc.rb