ruote-couch 2.1.10 → 2.1.11

data/CHANGELOG.txt

@@ -2,10 +2,20 @@
 = ruote-couch - CHANGELOG.txt
 
 
+== ruote-couch 2.1.11    released 2010/10/01
+
+- storage#shutdown back in business (for the sake of tests mostly)
+- get_many(x, y, :descending => true) support
+- adapted for engine.processes pagination
+- workitems.by_participant() :skip and :limit support
+- get_many(x, y, :count => true) support
+- a database.purge! that can't be mislead by cache issues
+
+
 == ruote-couch 2.1.10    released 2010/06/15
 
 - storage#ids not returning design doc ids anymore
-- now using CouchDB 0.11 continuous&all_docs
+- now using CouchDB 0.11 ?feed=continuous&include_docs=true
 - multi-worker hardened
 - adapted #query_workitems to ruote 2.1.10
 

data/CREDITS.txt

@@ -0,0 +1,18 @@
+
+= ruote-couch CREDITS.txt
+
+
+== authors
+
+- John Mettraux - http://github.com/jmettraux
+
+
+== contributors
+
+- Torsten Schoenebaum - http://github.com/tosch
+
+
+== many thanks to
+
+- the CouchDB team for their great project
+

data/README.rdoc

@@ -5,7 +5,7 @@ A storage implementation for ruote 2.1.x.
 
 Warning : this is a very naive implementation for now. Not many optimizations...
 
-Note : CouchDB >= 0.11
+Note : CouchDB >= 1.0.1
 
 
 == running tests

data/Rakefile

@@ -32,8 +32,9 @@ CouchDB storage for ruote 2.1 (ruby workflow engine)
 
   gem.test_file = 'test/test.rb'
 
-  gem.add_dependency 'ruote', ">= #{Ruote::Couch::VERSION}"
-  gem.add_dependency 'rufus-jig', '>= 0.1.18'
+  #gem.add_dependency 'ruote', ">= #{Ruote::Couch::VERSION}"
+  gem.add_dependency 'ruote', ">= 2.1.11"
+  gem.add_dependency 'rufus-jig', '>= 0.1.23'
   gem.add_development_dependency 'yard'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'jeweler'
@@ -61,33 +62,30 @@ end
 #
 # DOC
 
-begin
+#
+# make sure to have rdoc 2.5.x to run that
+#
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rd|
 
-  require 'yard'
+  rd.main = 'README.rdoc'
+  rd.rdoc_dir = 'rdoc/ruote-couch_rdoc'
 
-  YARD::Rake::YardocTask.new do |doc|
-    doc.options = [
-      '-o', 'html/ruote-couch', '--title',
-      "ruote-couch #{Ruote::Couch::VERSION}"
-    ]
-  end
+  rd.rdoc_files.include(
+    'README.rdoc', 'CHANGELOG.txt', 'CREDITS.txt', 'lib/**/*.rb')
 
-rescue LoadError
-
-  task :yard do
-    abort "YARD is not available : sudo gem install yard"
-  end
+  rd.title = "ruote-couch #{Ruote::Couch::VERSION}"
 end
 
 
 #
 # TO THE WEB
 
-task :upload_website => [ :clean, :yard ] do
+task :upload_rdoc => [ :clean, :rdoc ] do
 
   account = 'jmettraux@rubyforge.org'
   webdir = '/var/www/gforge-projects/ruote'
 
-  sh "rsync -azv -e ssh html/ruote-couch #{account}:#{webdir}/"
+  sh "rsync -azv -e ssh rdoc/ruote-couch_rdoc #{account}:#{webdir}/"
 end
 

data/TODO.txt

@@ -12,3 +12,6 @@
 [ ] issue with cron '* * * * *' and long-polling |
 =================================================+
 
+[ ] storage.close
+[ ] http://guide.couchdb.org/draft/notifications.html (filter schedules ?)
+

data/lib/ruote/couch/database.rb

@@ -39,11 +39,7 @@ module Ruote::Couch
 
     attr_reader :couch
 
-    def initialize (host, port, type, name)
-
-      #opts = { :re_put_ok => re_put_ok }
-      opts = {}
-      #opts[:timeout] = TODO
+    def initialize (host, port, type, name, opts={})
 
       @couch = Rufus::Jig::Couch.new(host, port, name, opts)
 
@@ -79,29 +75,49 @@ module Ruote::Couch
         # without this, test/functional/ct_0 fails after 1 to 10 runs...
 
       r
+
+    rescue Rufus::Jig::TimeoutError => te
+      true
     end
 
+    # The get_many used by msgs, configurations and variables.
+    #
     def get_many (key, opts)
 
-      rs = @couch.get("_all_docs?include_docs=true#{query_options(opts)}")
+      return query('_all_docs?include_docs=true') if ( ! key) && opts.size < 1
 
-      rs = rs['rows'].collect { |e| e['doc'] }
+      is = ids
 
-      rs = rs.select { |doc| doc['_id'].match(key) } if key
-        # naive...
+      return is.length if opts[:count]
 
-      rs.select { |doc| ! doc['_id'].match(/^\_design\//) }
-    end
+      is = is.reverse if opts[:descending]
 
-    DESIGN_DOC_REGEX = /^\_design\//
+      if key
+        keys = Array(key).map { |k| k.is_a?(String) ? "!#{k}" : k }
+        is = is.select { |i| Ruote::StorageBase.key_match?(keys, i) }
+      end
+
+      skip = opts[:skip] || 0
+      limit = opts[:limit] || is.length
+
+      is = is[skip, limit]
+
+      query_by_post('_all_docs?include_docs=true', is)
+
+      # TODO
+      # maybe _count come be of use here
+      # http://wiki.apache.org/couchdb/Built-In_Reduce_Functions
+    end
 
     # Returns a sorted list of the ids of all the docs in this database.
     #
     def ids
 
-      rs = @couch.get('_all_docs')
-
-      rs['rows'].collect { |r| r['id'] }.reject { |i| i.match(DESIGN_DOC_REGEX) }
+      @couch.get('_all_docs')['rows'].collect { |r|
+        r['id']
+      }.reject { |i|
+        DESIGN_DOC_REGEX.match(i)
+      }
     end
 
     def dump
@@ -116,18 +132,22 @@ module Ruote::Couch
       end
     end
 
-    #def shutdown
-    #  @couch.close
-    #end
-      # jig > 0.1.17 is OK without that
+    # Makes sure to close the HTTP connection down.
+    #
+    def shutdown
+
+      @couch.close
+    end
 
     # Deletes all the documents in this database.
     #
     def purge!
 
+      @couch.http.cache.clear
       @couch.get('_all_docs')['rows'].each do |row|
+        next if row['id'].match(/^\_design\//)
         doc = { '_id' => row['id'], '_rev' => row['value']['rev'] }
-        @couch.delete(doc) unless doc['_id'].match(/^\_design\//)
+        @couch.delete(doc)
       end
         #
         # which is faster than
@@ -144,22 +164,36 @@ module Ruote::Couch
       # nothing to do for a index-less database
     end
 
-    # (for now, the only option is :limit)
+    # These options are known and passed to CouchDB.
+    #
+    QUERY_OPTIONS = [ :skip, :limit, :descending ]
+
+    # :limit and :skip support
     #
     def query_options (opts)
 
-      opts = opts.select { |k, v| [ :limit, :skip ].include?(k) && v != nil }
+      opts = opts.select { |k, v| QUERY_OPTIONS.include?(k) && v != nil }
 
       s = opts.collect { |k, v| "#{k}=#{v}" }.join('&')
 
       s.length > 0 ? "&#{s}" : ''
     end
 
+    # Used by #get_many and #ids when filtering design documents out of
+    # '_all_docs'.
+    #
+    DESIGN_DOC_REGEX = /^\_design\//
+
+    def filter_design_docs (docs)
+
+      docs.reject { |d| DESIGN_DOC_REGEX.match(d['_id']) }
+    end
+
     def query (uri)
 
       rs = @couch.get(uri)
 
-      rs['rows'].collect { |e| e['doc'] }
+      filter_design_docs(rs['rows'].collect { |e| e['doc'] })
     end
 
     def query_by_post (uri, keys)
@@ -168,7 +202,7 @@ module Ruote::Couch
 
       rs = @couch.post(uri, keys)
 
-      rs['rows'].collect { |e| e['doc'] }.uniq
+      filter_design_docs(rs['rows'].collect { |e| e['doc'] }.uniq)
     end
   end
 
@@ -177,27 +211,23 @@ module Ruote::Couch
   #
   class WfidIndexedDatabase < Database
 
+    # The get_many used by errors, expressions and schedules.
+    #
     def get_many (key, opts)
 
-      if key && m = key.source.match(/!?(.+)\$$/)
-        # let's use the couch view...
-
-        query(
-          "_design/ruote/_view/by_wfid?key=%22#{m[1]}%22" +
-          "&include_docs=true#{query_options(opts)}")
+      return super(key, opts) unless key.is_a?(String)
 
-      else
-        # let's use the naive default implementation
+      # key is a wfid
 
-        super
-      end
+      query("_design/ruote/_view/by_wfid?key=%22#{key}%22&include_docs=true")
     end
 
     # Used by WorkitemDatabase#query
     #
     def by_wfid (wfid)
 
-      get_many(/!#{wfid}$/, {})
+      #get_many(/!#{wfid}$/, {})
+      get_many(wfid, {})
     end
 
     # Returns the design document that goes with this class of database
@@ -215,8 +245,12 @@ module Ruote::Couch
         '_id' => '_design/ruote',
         'views' => {
           'by_wfid' => {
-            'map' =>
-              'function (doc) { if (doc.fei) emit(doc.fei.wfid, null); }'
+            'map' => %{
+              function (doc) {
+                if (doc.wfid) emit(doc.wfid, null);
+                else if (doc.fei) emit(doc.fei.wfid, null);
+              }
+            }
           }
         }
       }
@@ -278,8 +312,7 @@ module Ruote::Couch
         return by_wfid(wfid) # if wfid
       end
 
-      return get_many(nil, {}).collect { |hwi| Ruote::Workitem.new(hwi) } \
-        if criteria.empty?
+      return get_many(nil, {}) if criteria.empty?
 
       cr = criteria.collect { |fname, fvalue| { fname => fvalue } }
 

data/lib/ruote/couch/storage.rb

@@ -47,47 +47,41 @@ module Couch
     # Hooks the storage to a CouchDB instance.
     #
     # The main option is 'couch_prefix', which indicate which prefix should be
-    # added to all the database names used by this storage.
+    # added to all the database names used by this storage. 'prefix' is accepted
+    # as well.
     #
-    # The option 'couch_timeout' is used what is the get_msgs timeout. This
-    # is the long-polling timeout. For functional test it is set to two seconds
-    # but for a production system, something like 10 minutes or 8 hours might
-    # be OK.
-    #
-    def initialize (host, port, options={})
+    def initialize (*args)
 
-      @host = host
-      @port = port
+      hc = Rufus::Jig::HttpCore.new(*args)
+        # leverage the argument parsing logic in there
 
-      @options = options
+      @host = hc.host
+      @port = hc.port
 
-      @prefix = options['couch_prefix'] || options['prefix'] || ''
-      @prefix = "#{@prefix}_" if @prefix.size > 0
+      @options = hc.options
 
-      #@zeroes = 21 # maybe make it an option
-      @timeout = options['couch_timeout'] || 60
+      @prefix = hc.options['couch_prefix'] || hc.options['prefix'] || ''
+      @prefix = "#{@prefix}_" if @prefix.size > 0
 
       @dbs = {}
 
-      %w[ msgs schedules configurations variables ].each do |type|
+      %w[ msgs configurations variables ].each do |type|
 
         @dbs[type] = Database.new(
-          @host, @port, type, "#{@prefix}ruote_#{type}")
+          @host, @port, type, "#{@prefix}ruote_#{type}", @options)
       end
 
-      @dbs['errors'] = WfidIndexedDatabase.new(
-        @host, @port, 'errors', "#{@prefix}ruote_errors")
+      %w[ errors expressions schedules ].each do |type|
 
-      @dbs['expressions'] = WfidIndexedDatabase.new(
-        #@host, @port, 'expressions', "#{@prefix}ruote_expressions", false)
-        @host, @port, 'expressions', "#{@prefix}ruote_expressions")
+        @dbs[type] = WfidIndexedDatabase.new(
+          @host, @port, type, "#{@prefix}ruote_#{type}", @options)
+      end
 
       @dbs['workitems'] = WorkitemDatabase.new(
-        @host, @port, 'workitems', "#{@prefix}ruote_workitems")
+        @host, @port, 'workitems', "#{@prefix}ruote_workitems", @options)
 
       put_configuration
 
-      #@zero_msgs_offset = @zeroes
       @msgs_thread = nil
       @msgs_queue = ::Queue.new
 
@@ -131,18 +125,6 @@ module Couch
       #@dbs.values.each { |db| db.shutdown }
     end
 
-    # Used when doing integration tests, removes all
-    # msgs, schedules, errors, expressions and workitems.
-    #
-    # NOTE that it doesn't remove engine variables (danger)
-    #
-    def clear
-
-      %w[ msgs schedules errors expressions workitems ].each do |type|
-        @dbs[type].purge!
-      end
-    end
-
     def dump (type)
 
       @dbs[type].dump
@@ -150,9 +132,7 @@ module Couch
 
     def shutdown
 
-      #@dbs.values.each { |db| db.shutdown }
-
-      #@poller.kill if @poller
+      @dbs.values.each { |db| db.shutdown }
 
       @msgs_thread.kill rescue nil
       @schedules_thread.kill rescue nil
@@ -179,11 +159,11 @@ module Couch
     # A provision made for workitems, allow to query them directly by
     # participant name.
     #
-    def by_participant (type, participant_name)
+    def by_participant (type, participant_name, opts)
 
       raise NotImplementedError if type != 'workitems'
 
-      @dbs['workitems'].by_participant(participant_name)
+      @dbs['workitems'].by_participant(participant_name, opts)
     end
 
     def by_field (type, field, value=nil)
@@ -195,7 +175,11 @@ module Couch
 
     def query_workitems (criteria)
 
-      @dbs['workitems'].query_workitems(criteria)
+      count = criteria.delete('count')
+
+      result = @dbs['workitems'].query_workitems(criteria)
+
+      count ? result.size : result.collect { |h| Ruote::Workitem.new(h) }
     end
 
     # Overwriting Ruote::StorageBase.get_msgs
@@ -205,9 +189,14 @@ module Couch
     #
     def get_msgs
 
+      mt = @msgs_thread
+
       ensure_msgs_thread_is_running
 
       msgs = []
+      2.times { msgs = get_many('msgs') } if mt != @msgs_thread
+        #
+        # seems necessary to avoid any msgs leak :-(
 
       while @msgs_queue.size > 0
         msgs << @msgs_queue.pop
@@ -234,6 +223,8 @@ module Couch
 
         deleted, s = @schedules_queue.pop
 
+        next unless s
+
         if deleted
           @schedules.delete(s['_id'])
         else
@@ -241,7 +232,7 @@ module Couch
         end
       end
 
-      filter_schedules(@schedules.values, now)
+      filter_schedules(@schedules.values.reject { |sch| sch['at'].nil? }, now)
     end
 
     protected

data/lib/ruote/couch/version.rb

@@ -1,7 +1,7 @@
 
 module Ruote
 module Couch
-  VERSION = '2.1.10'
+  VERSION = '2.1.11'
 end
 end
 

data/ruote-couch.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ruote-couch}
-  s.version = "2.1.10"
+  s.version = "2.1.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John Mettraux"]
-  s.date = %q{2010-06-15}
+  s.date = %q{2010-10-01}
   s.description = %q{CouchDB storage for ruote 2.1 (ruby workflow engine)}
   s.email = %q{jmettraux@gmail.com}
   s.extra_rdoc_files = [
@@ -18,6 +18,7 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     "CHANGELOG.txt",
+     "CREDITS.txt",
      "LICENSE.txt",
      "README.rdoc",
      "Rakefile",
@@ -29,7 +30,6 @@ Gem::Specification.new do |s|
      "lib/ruote/couch/version.rb",
      "ruote-couch.gemspec",
      "test/functional/base.rb",
-     "test/functional/ft_0_long_polling.rb",
      "test/functional/test.rb",
      "test/functional_connection.rb",
      "test/test.rb",
@@ -51,21 +51,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<ruote>, [">= 2.1.10"])
-      s.add_runtime_dependency(%q<rufus-jig>, [">= 0.1.18"])
+      s.add_runtime_dependency(%q<ruote>, [">= 2.1.11"])
+      s.add_runtime_dependency(%q<rufus-jig>, [">= 0.1.23"])
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
     else
-      s.add_dependency(%q<ruote>, [">= 2.1.10"])
-      s.add_dependency(%q<rufus-jig>, [">= 0.1.18"])
+      s.add_dependency(%q<ruote>, [">= 2.1.11"])
+      s.add_dependency(%q<rufus-jig>, [">= 0.1.23"])
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
     end
   else
-    s.add_dependency(%q<ruote>, [">= 2.1.10"])
-    s.add_dependency(%q<rufus-jig>, [">= 0.1.18"])
+    s.add_dependency(%q<ruote>, [">= 2.1.11"])
+    s.add_dependency(%q<rufus-jig>, [">= 0.1.23"])
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])

data/test/functional_connection.rb

@@ -9,18 +9,35 @@ require 'yajl' rescue require 'json'
 require 'rufus-json'
 Rufus::Json.detect_backend
 
-begin
-  require 'patron' unless ARGV.include?('--net')
-rescue LoadError
-  # then use 'net/http'
+unless $http_lib_loaded
+  begin
+    if ARGV.include?('--patron')
+      require 'patron'
+      puts ' : using patron'
+    elsif ARGV.include?('--netp')
+      require 'net/http/persistent'
+      puts ' : using net-http-persistent'
+    else
+      puts ' : using net/http'
+    end
+  rescue LoadError => le
+    # then use 'net/http'
+    puts ' : falling back to net/http'
+  end
+  $http_lib_loaded = true
 end
 
 require 'ruote/couch/storage'
 
+def _couch_url
+
+  File.read('couch_url.txt').strip rescue 'http://127.0.0.1:5984'
+end
+
 
 unless $_RUOTE_COUCH_CLEANED
 
-  couch = Rufus::Jig::Couch.new('127.0.0.1', 5984)
+  couch = Rufus::Jig::Couch.new(_couch_url)
   %w[
     configurations errors expressions msgs schedules variables workitems
   ].each do |type|
@@ -36,8 +53,7 @@ def new_storage (opts)
   opts ||= {}
 
   Ruote::Couch::CouchStorage.new(
-    '127.0.0.1',
-    5984,
-    opts.merge!('couch_prefix' => 'test', 'couch_timeout' => 1))
+    _couch_url,
+    opts.merge!('couch_prefix' => 'test', :basic_auth => %w[ admin admin ]))
 end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 2
   - 1
-  - 10
-  version: 2.1.10
+  - 11
+  version: 2.1.11
 platform: ruby
 authors: 
 - John Mettraux
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-15 00:00:00 +09:00
+date: 2010-10-01 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,8 +27,8 @@ dependencies:
         segments: 
         - 2
         - 1
-        - 10
-        version: 2.1.10
+        - 11
+        version: 2.1.11
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -41,8 +41,8 @@ dependencies:
         segments: 
         - 0
         - 1
-        - 18
-        version: 0.1.18
+        - 23
+        version: 0.1.23
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -92,6 +92,7 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - CHANGELOG.txt
+- CREDITS.txt
 - LICENSE.txt
 - README.rdoc
 - Rakefile
@@ -103,7 +104,6 @@ files:
 - lib/ruote/couch/version.rb
 - ruote-couch.gemspec
 - test/functional/base.rb
-- test/functional/ft_0_long_polling.rb
 - test/functional/test.rb
 - test/functional_connection.rb
 - test/test.rb

data/test/functional/ft_0_long_polling.rb

@@ -1,66 +0,0 @@
-
-#
-# testing ruote-couch
-#
-# Fri Mar 12 17:15:27 JST 2010
-#
-
-require File.join(File.dirname(__FILE__), 'base')
-
-
-class FtLongPollingTest < Test::Unit::TestCase
-
-  def setup
-
-    @engine =
-      Ruote::Engine.new(
-        Ruote::Worker.new(
-          Ruote::Couch::CouchStorage.new(
-            '127.0.0.1',
-            5984,
-            'couch_prefix' => 'test', 'couch_timeout' => 2 * 60)))
-  end
-
-  def teardown
-
-    @engine.shutdown
-    @engine.context.storage.purge!
-  end
-
-  def test_wait
-
-    trace = []
-
-    @engine.register_participant '.+' do |workitem|
-      trace << Time.now
-    end
-
-    pdef = Ruote.process_definition do
-      sequence do
-        alpha
-        wait '3m'
-        bravo
-      end
-    end
-
-    @engine.context.logger.noisy = true
-
-    wfid = @engine.launch(pdef)
-
-    sleep 15
-    assert_equal 1, trace.size
-
-    assert_not_nil @engine.context.storage.instance_variable_get(:@poller)
-
-    @engine.wait_for(wfid)
-
-    p trace
-
-    assert_equal 2, trace.size
-
-    delta = trace.last - trace.first
-
-    assert_in_delta 3.0 * 60, delta, 1.0
-  end
-end
-