ruote 0.9.19 → 0.9.20

data/lib/openwfe/expool/expstorage.rb

@@ -1,44 +1,27 @@
-#
 #--
-# Copyright (c) 2006-2008, Nicolas Modryzk and John Mettraux, OpenWFE.org
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# Copyright (c) 2006-2009, Nicolas Modryzk and John Mettraux, OpenWFE.org
 #
-# . Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
 #
-# . Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
 #
-# . Neither the name of the "OpenWFE" nor the names of its contributors may be
-#   used to endorse or promote products derived from this software without
-#   specific prior written permission.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 #
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
+# Made in Japan.
 #++
-#
-
-#
-# "made in Japan"
-#
-# Nicolas Modrzyk at openwfe.org
-# John Mettraux at openwfe.org
-#
 
-require 'fileutils'
 
 require 'rufus/lru'
 
@@ -59,13 +42,13 @@ module OpenWFE
 
     def observe_expool
 
+      return unless get_expression_pool
+
       get_expression_pool.add_observer(:update) do |channel, fei, fe|
-        ldebug { ":update  for #{fei}" }
         self[fei] = fe
       end
       get_expression_pool.add_observer(:remove) do |channel, fei|
-        ldebug { ":remove  for #{fei}" }
-        self.delete fei
+        self.delete(fei)
       end
     end
 
@@ -138,7 +121,7 @@ module OpenWFE
         if wfrevision and fei.wfrevision != wfrevision
 
       return false \
-        if cs and fei.sub_instance_id != ""
+        if cs and fei.sub_instance_id != ''
       return false \
         if wfid and fei.wfid != wfid
       return false \
@@ -202,8 +185,6 @@ module OpenWFE
 
       @cache = LruHash.new(size)
 
-      @real_storage = nil
-
       observe_expool
     end
 
@@ -212,7 +193,7 @@ module OpenWFE
       #ldebug { "[] size is #{@cache.size}" }
       #ldebug { "[] (sz #{@cache.size}) for #{fei.to_debug_s}" }
 
-      fe = @cache[fei.hash]
+      fe = @cache[fei.short_hash]
       return fe if fe
 
       #ldebug { "[] (reload) for #{fei.to_debug_s}" }
@@ -224,23 +205,23 @@ module OpenWFE
         return nil
       end
 
-      @cache[fei.hash] = fe
-
-      fe
+      @cache[fei.short_hash] = fe
     end
 
     def []= (fei, fe)
 
-      ldebug { "[]= caching #{fei}" }
-
-      @cache[fei.hash] = fe
+      #ldebug { "[]= caching #{fei}" }
+      @cache[fei.short_hash] = fe
     end
 
     def delete (fei)
 
-      @cache.delete fei.hash
+      @cache.delete(fei.short_hash)
     end
 
+    #
+    # returns the count of expressions currently cached here
+    #
     def length
 
       @cache.length
@@ -261,7 +242,9 @@ module OpenWFE
     #
     def find_expressions (options={})
 
-      get_real_storage.find_expressions options
+      options[:cache] = self
+
+      get_real_storage.find_expressions(options)
     end
 
     #
@@ -270,34 +253,39 @@ module OpenWFE
     #
     def fetch_root (wfid)
 
-      #
-      # at first, look in the cache
-
-      @cache.each do |hashed_fei, fexp|
+      @cache.values.find { |fexp|
+        fexp.fei.wfid == wfid and fexp.is_a?(DefineExpression)
+      } || get_real_storage.fetch_root(wfid)
+    end
 
-        return fexp \
-          if fexp.fei.wfid == wfid and fexp.is_a?(DefineExpression)
-      end
+    #
+    # Returns the expression corresponding to the fei if cached.
+    # Does not lookup in the underlying "real" storage (will therefore
+    # return nil if the expression is not cached.
+    #
+    def fetch (fei)
 
-      get_real_storage.fetch_root wfid
+      @cache[fei.short_hash]
     end
 
     protected
 
-      #
-      # Returns the "real storage" i.e. the storage that does the real
-      # persistence behind this "cache storage".
-      #
-      def get_real_storage
+    #
+    # Returns the "real storage" i.e. the storage that does the real
+    # persistence behind this "cache storage".
+    #
+    def get_real_storage
 
-        @application_context[:s_expression_storage__1]
-      end
+      @application_context[:s_expression_storage__1]
+    end
   end
 
   #
-  # Memory consuming in-memory storage.
+  # [memory consuming] in-memory storage.
   # No memory limit, puts everything in a Hash
   #
+  # USE ONLY FOR TESTS
+  #
   class InMemoryExpressionStorage < Hash
     include ServiceMixin
     include OwfeServiceLocator
@@ -305,13 +293,25 @@ module OpenWFE
 
     def initialize (service_name, application_context)
 
-      service_init service_name, application_context
+      service_init(service_name, application_context)
 
       observe_expool
     end
 
     alias :purge :clear
 
+    #--
+    #def [] (k)
+    #  super(k.short_hash)
+    #end
+    #def []= (k, v)
+    #  super(k.short_hash, v)
+    #end
+    #def delete (k)
+    #  super(k.short_hash)
+    #end
+    #++
+
     #
     # Finds expressions matching the given criteria (returns a list
     # of expressions).
@@ -353,9 +353,7 @@ module OpenWFE
     #
     def find_expressions (options={})
 
-      values.find_all do |fexp|
-        does_match? options, fexp
-      end
+      values.find_all { |fexp| does_match?(options, fexp) }
     end
 
     #
@@ -364,9 +362,7 @@ module OpenWFE
     #
     def fetch_root (wfid)
 
-      find_expressions(
-        :wfid => wfid,
-        :include_classes => DefineExpression)[0]
+      find_expressions(:wfid => wfid, :include_classes => DefineExpression)[0]
     end
 
   end

data/lib/openwfe/expool/fs_expstorage.rb

@@ -0,0 +1,302 @@
+#--
+# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
+
+require 'fileutils'
+require 'openwfe/expool/expstorage'
+require 'openwfe/expool/threaded_expstorage'
+
+
+module OpenWFE
+
+  #
+  # Stores the expressions (the pieces of process instances) in the file system.
+  #
+  # By default uses Ruby Marshalling. If the options :persist_as_yaml is set
+  # to true in the application context, YAML serialization will be used.
+  # YAML is more portable than Ruby Marshalling, but considerably slower.
+  #
+  # The default (Ruby Marshalling) is recommended. If you need to migrate from
+  # one version of Marshalling to the other, you can use work/pooltool.rb
+  # to migrate the whole set of expressions from one format to the other.
+  #
+  class FsExpressionStorage
+    include ServiceMixin
+    include OwfeServiceLocator
+    include ExpressionStorageBase
+
+    attr_accessor :persist_as_yaml, :suffix
+    attr_reader :basepath
+
+    def initialize (service_name, application_context)
+
+      service_init(service_name, application_context)
+
+      @basepath =
+        application_context[:expstorage_path] || get_work_directory + '/expool'
+
+      @persist_as_yaml = (application_context[:persist_as_yaml] == true)
+      @suffix = 'ruote'
+
+      observe_expool
+    end
+
+    #
+    # Stores an expression
+    #
+    def []= (fei, fexp)
+
+      d, fn = filename_for(fei)
+
+      FileUtils.mkdir_p(d) unless File.exist?(d)
+
+      File.open("#{d}/#{fn}", 'w') { |f| f.write(encode(fexp)) }
+    end
+
+    #
+    # Retrieves an expression
+    #
+    def [] (fei)
+
+      fexp = load_fexp(filename_for(fei, true))
+      fexp.application_context = @application_context if fexp
+      fexp
+    end
+
+    #
+    # Removes the expression from the storage
+    #
+    def delete (fei)
+
+      FileUtils.rm_f(filename_for(fei, true))
+    end
+
+    #
+    # Returns the count of expressions currently stored
+    #
+    def size
+
+      Dir["#{@basepath}/**/*.#{@suffix}"].size
+    end
+
+    #
+    # Finds expressions matching the given criteria (returns a list
+    # of expressions).
+    #
+    # This methods is called by the expression pool, it's thus not
+    # very "public" (not used directly by integrators, who should
+    # just focus on the methods provided by the Engine).
+    #
+    # :wfid ::
+    #   will list only one process,
+    #   <tt>:wfid => '20071208-gipijiwozo'</tt>
+    # :parent_wfid ::
+    #   will list only one process, and its subprocesses,
+    #   <tt>:parent_wfid => '20071208-gipijiwozo'</tt>
+    # :consider_subprocesses ::
+    #   if true, "process-definition" expressions
+    #   of subprocesses will be returned as well.
+    # :wfid_prefix ::
+    #   allows your to query for specific workflow instance
+    #   id prefixes. for example :
+    #   <tt>:wfid_prefix => "200712"</tt>
+    #   for the processes started in December.
+    # :include_classes ::
+    #   excepts a class or an array of classes, only instances of these
+    #   classes will be returned. Parent classes or mixins can be
+    #   given.
+    #   <tt>:includes_classes => OpenWFE::SequenceExpression</tt>
+    # :exclude_classes ::
+    #   works as expected.
+    # :wfname ::
+    #   will return only the expressions who belongs to the given
+    #   workflow [name].
+    # :wfrevision ::
+    #   usued in conjuction with :wfname, returns only the expressions
+    #   with a given workflow revision.
+    # :applied ::
+    #   if this option is set to true, will only return the expressions
+    #   that have been applied (exp.apply_time != nil).
+    #
+    def find_expressions (options={})
+
+      dir = if wfid = options[:wfid]
+        dir_for(wfid)
+      else
+        "#{@basepath}/**" # brute force
+      end
+
+      Dir["#{dir}/*.#{@suffix}"].inject([]) do |a, path|
+
+        fexp = load_fexp(path)
+
+        if fexp and does_match?(options, fexp)
+
+          fexp.application_context = @application_context
+          a << fexp
+        end
+
+        a
+      end
+    end
+
+    #
+    # An iterator on ALL expressions in the storage (only used by pooltool.ru)
+    #
+    def each
+
+      return unless block_given?
+
+      Dir["#{@basepath}/**/*.#{@suffix}"].each do |path|
+
+        fexp = load_fexp(path)
+
+        yield(fexp.fei, fexp)
+      end
+    end
+
+    #
+    # Dangerous ! Nukes the whole work/expool/ dir
+    #
+    def purge
+
+      FileUtils.rm_f(@basepath)
+    end
+
+    #
+    # Fetches the root expression of a process instance
+    #
+    def fetch_root (wfid)
+
+      dir = dir_for(wfid)
+
+      fexps = Dir["#{dir}/*.#{@suffix}"].collect { |path| load_fexp(path) }
+
+      root = fexps.find { |fexp|
+        fexp.fei.expid == '0' &&
+        fexp.fei.sub_instance_id == '' &&
+        fexp.is_a?(OpenWFE::DefineExpression)
+      }
+      root.application_context = @application_context
+      root
+    end
+
+    #
+    # Called by pooltool.ru
+    #
+    def close
+      # nothing to do
+    end
+
+    protected
+
+    #
+    # Encodes the flow expression (if the options :yaml_persistence is set
+    # to true in the application context or via #yaml= will save as
+    # YAML)
+    #
+    def encode (fexp)
+      @persist_as_yaml ? fexp.to_yaml : Marshal.dump(fexp)
+    end
+
+    #
+    # Loads the flow expression at the given path
+    #
+    def load_fexp (path)
+      return nil unless File.exist?(path)
+      File.open(path, 'r') { |f| decode(f.read) }
+    end
+
+    #
+    # Decodes the content of a file (reads YAML or Marshall binary
+    # indifferently)
+    #
+    def decode (s)
+      s[0, 5] == '--- !' ? YAML.load(s) : Marshal.load(s)
+    end
+
+    #
+    # Returns the directory path for a given workflow instance id
+    #
+    def dir_for (wfid)
+
+      wfid = FlowExpressionId.to_parent_wfid(wfid)
+      a_wfid = get_wfid_generator.split_wfid(wfid)
+
+      "#{@basepath}/#{a_wfid[-2]}/#{a_wfid[-1]}"
+    end
+
+    #
+    # Returns the pair dir / filename for an expression.
+    # If the optional arg join is set to true, will return the full path
+    # for the expression
+    #
+    def filename_for (fei, join=false)
+
+      r = if fei.wfid == '0'
+        [ @basepath, "engine_environment.#{@suffix}" ]
+      else
+        [
+          dir_for(fei.wfid),
+          "#{fei.workflow_instance_id}__#{fei.expression_id}_#{fei.expression_name}.#{@suffix}"
+        ]
+      end
+
+      join ? "#{r.first}/#{r.last}" : r
+    end
+
+  end
+
+  #
+  # YAML expression storage. Expressions (atomic pieces of process instances)
+  # are stored in a hierarchy of YAML files.
+  #
+  # DEPRECATED, use the plain FsExpressionStorage instead.
+  #
+  class YamlFileExpressionStorage < FsExpressionStorage
+
+    def initialize (service_name, application_context)
+      super
+      @persist_as_yaml = true
+      @suffix = 'yaml'
+    end
+  end
+
+  #
+  # With this extension of YmalFileExpressionStorage, persistence occurs
+  # in a separate thread, for a snappier response.
+  #
+  # Will probably get deprecated soon.
+  #
+  class ThreadedYamlFileExpressionStorage < YamlFileExpressionStorage
+    include ThreadedStorageMixin
+
+    def initialize (service_name, application_context)
+      super
+      start_queue
+    end
+  end
+
+end
+