openwferu 0.9.15 → 0.9.16

data/lib/openwfe/expool/expressionpool.rb

@@ -190,7 +190,7 @@ module OpenWFE
         # Returns the FlowExpressionId instance of the root expression of
         # the newly launched flow.
         #
-        def launch (launchitem, options)
+        def launch (launchitem, options={})
 
             #
             # prepare raw expression
@@ -207,7 +207,7 @@ module OpenWFE
                 # the process instance
 
             raw_expression = wrap_in_schedule(raw_expression, options) \
-                if options and options.size > 0
+                if options.size > 0
 
             fei = raw_expression.fei
 
@@ -234,18 +234,24 @@ module OpenWFE
             requesting_expression, sub_id, template, params=nil)
 
             rawexp = if template.is_a?(RawExpression)
+
+                template.application_context = @application_context
                 template
+
             elsif template.is_a?(FlowExpressionId)
-                fetch_expression(template)
+
+                fetch_expression template
+
             else
-                build_raw_expression(nil, template)
+
+                build_raw_expression nil, template
             end
 
             #raise "did not find subprocess in : #{template.to_s}" \
             #    unless rawexp
 
-            rawexp = rawexp.dup()
-            rawexp.fei = rawexp.fei.dup()
+            rawexp = rawexp.dup
+            rawexp.fei = rawexp.fei.dup
 
             if requesting_expression == nil
 
@@ -276,11 +282,11 @@ module OpenWFE
             #    "#{rawexp.fei.workflow_instance_id.to_s}"
             #end
 
-            env = rawexp.new_environment(params)
+            env = rawexp.new_environment params
                 #
                 # the new scope gets its own environment
 
-            rawexp.store_itself()
+            rawexp.store_itself
 
             rawexp
         end
@@ -303,22 +309,22 @@ module OpenWFE
             rawexp.fei
         end
 
-        #
+        #--
         # Evaluates a raw definition expression and
         # returns its body fei
         #
-        def evaluate (rawExpression, workitem)
-
-            exp = rawExpression.instantiate_real_expression workitem
-            fei = exp.evaluate workitem
-
-            #remove(rawExpression)
-                #
-                # not necessary, the raw expression gets overriden by
-                # the real expression
-
-            fei
-        end
+        #def evaluate (rawExpression, workitem)
+        #    #rawExpression = fetch_expression(rawExpression) \
+        #    #    if rawExpression.is_a?(FlowExpressionId)
+        #    exp = rawExpression.instantiate_real_expression workitem
+        #    fei = exp.evaluate workitem
+        #    #remove(rawExpression)
+        #        #
+        #        # not necessary, the raw expression gets overriden by
+        #        # the real expression
+        #    fei
+        #end
+        #++
 
         #
         # Applies a given expression (id or expression)
@@ -370,9 +376,9 @@ module OpenWFE
         #
         def cancel_expression (exp)
 
-            exp = fetch_expression(exp)
+            exp = fetch_expression exp
 
-            wi = cancel(exp)
+            wi = cancel exp
 
             if wi
                 reply_to_parent(exp, wi, false)
@@ -418,58 +424,6 @@ module OpenWFE
             ldebug { "forget() forgot      #{fei}" }
         end
 
-        #
-        # Pauses a process (sets its '/__paused__' variable to true).
-        #
-        def pause_process (wfid)
-
-            wfid = extract_wfid(wfid)
-
-            root_expression = fetch_root(wfid)
-
-            root_expression.set_variable(VAR_PAUSED, true)
-        end
-
-        #
-        # Restarts a process : removes its 'paused' flag (variable) and makes
-        # sure to 'replay' events (replies) that came for it while it was
-        # in pause.
-        #
-        def resume_process (wfid)
-
-            wfid = extract_wfid(wfid)
-
-            root_expression = fetch_root(wfid)
-
-            #
-            # remove 'paused' flag
-
-            root_expression.unset_variable(VAR_PAUSED)
-
-            #
-            # replay
-
-            journal = get_error_journal
-
-            # select PausedError instances in separate list
-
-            errors = journal.get_error_log(wfid)
-            error_class = PausedError.name
-            paused_errors = errors.select { |e| e.error_class == error_class }
-
-            return if paused_errors.size < 1
-
-            # remove them from current error journal
-
-            journal.remove_errors wfid, paused_errors
-
-            # replay select PausedError instances
-
-            paused_errors.each do |e|
-                journal.replay_at_error e
-            end
-        end
-
         #
         # Replies to the parent of the given expression.
         #
@@ -538,16 +492,16 @@ module OpenWFE
         #
         def update (flow_expression)
 
-            #ldebug { "update() for #{flow_expression.fei.to_debug_s}" }
+            ldebug { "update() for #{flow_expression.fei.to_debug_s}" }
 
-            t = Timer.new
+            #t = Timer.new
 
             onotify :update, flow_expression.fei, flow_expression
 
-            ldebug do 
-                "update() took #{t.duration} ms  " +
-                "#{flow_expression.fei.to_debug_s}"
-            end
+            #ldebug do 
+            #    "update() took #{t.duration} ms  " +
+            #    "#{flow_expression.fei.to_debug_s}"
+            #end
 
             flow_expression
         end
@@ -562,16 +516,21 @@ module OpenWFE
         def fetch (exp)
             synchronize do
 
-                fei = exp
-
                 #ldebug { "fetch() exp is of kind #{exp.class}" }
 
-                if exp.kind_of?(FlowExpression)
-                    fei = exp.fei 
+                fei = if exp.kind_of?(FlowExpression)
+
+                    exp.fei 
+
                 elsif not exp.kind_of?(FlowExpressionId)
+
                     raise \
                         "Cannot fetch expression with key : "+
                         "'#{fei}' (#{fei.class})"
+
+                else
+
+                    exp
                 end
 
                 #ldebug { "fetch() for  #{fei.to_debug_s}" }
@@ -588,7 +547,7 @@ module OpenWFE
         #
         def fetch_expression (exp)
 
-            exp, _fei = fetch(exp)
+            exp, fei = fetch(exp)
             exp
         end
 
@@ -598,6 +557,8 @@ module OpenWFE
         #
         def fetch_root (exp_or_wfid)
 
+            ldebug { "fetch_root() '#{exp_or_wfid.to_s}'" }
+
             return fetch_expression_with_wfid(exp_or_wfid) \
                 if exp_or_wfid.is_a?(String)
 
@@ -638,7 +599,7 @@ module OpenWFE
         def remove (exp)
 
             exp, _fei = fetch(exp) \
-                if exp.kind_of?(FlowExpressionId)
+                if exp.is_a?(FlowExpressionId)
 
             return unless exp
 
@@ -670,12 +631,12 @@ module OpenWFE
 
                 linfo { "reschedule() initiating..." }
 
-                get_expression_storage.each_of_kind(Schedulable) do |fe|
+                get_expression_storage.each_of_kind(Schedulable) do |fei, fe|
 
-                    #linfo { "reschedule() for  #{fe.fei.to_debug_s}..." }
-                    linfo { "reschedule() for  #{fe.fei.to_s}..." }
+                    #linfo { "reschedule() for  #{fei.to_debug_s}..." }
+                    linfo { "reschedule() for  #{fei.to_s}..." }
 
-                    onotify :reschedule, fe.fei
+                    onotify :reschedule, fei
 
                     fe.reschedule(get_scheduler)
                 end
@@ -762,7 +723,9 @@ module OpenWFE
 
             get_expression_storage.real_each(wfid_prefix) do |fei, fexp|
 
-                next unless fexp.is_a? DefineExpression
+                #ldebug { "list_processes() class is #{fexp.class.name}" }
+
+                next unless fexp.is_a?(DefineExpression)
 
                 next if not consider_subprocesses and fei.wfid.index(".")
 
@@ -779,7 +742,15 @@ module OpenWFE
         #
         def fetch_expression_with_wfid (wfid)
 
-            list_processes(false, wfid)[0]
+            #list_processes(false, wfid)[0]
+
+            ps = list_processes(false, wfid)
+
+            ldebug do 
+                "fetch_expression_with_wfid() '#{wfid}' found #{ps.size} items"
+            end
+
+            ps[0]
         end
 
         #
@@ -851,31 +822,29 @@ module OpenWFE
             #
             def do_apply (exp, workitem)
 
-                exp, fei = fetch(exp) if exp.kind_of?(FlowExpressionId)
+                exp, _fei = fetch(exp) if exp.is_a?(FlowExpressionId)
 
                 check_if_paused exp
 
-                #ldebug { "apply()  '#{fei}' (#{fei.class})" }
+                #ldebug { "apply()  '#{_fei}'" }
 
                 if not exp
 
-                    lwarn { "apply() cannot apply missing  #{fei.to_debug_s}" }
+                    lwarn do 
+                        "do_apply() cannot apply missing  #{_fei.to_debug_s}"
+                    end
+
                     return
 
-                    #raise "apply() cannot apply missing  #{fei.to_debug_s}"
+                    #raise "apply() cannot apply missing  #{_fei.to_debug_s}"
+                        # not very helpful anyway
                 end
 
-                #ldebug { "apply()  #{fei.to_debug_s}" }
-
-                #exp.apply_time = OpenWFE::now()
-                    #
-                    # this is done in RawExpression
-
                 workitem.flow_expression_id = exp.fei
 
                 onotify :apply, exp, workitem
 
-                exp.apply(workitem)
+                exp.apply workitem
             end
 
             #
@@ -941,10 +910,11 @@ module OpenWFE
                     else #oin
                         { "for" => oin }
                     end
+                    att["scheduler-tags"] = "scheduled-launch"
 
                     sle = get_expression_map.get_class(:sleep)
                     sle = sle.new(fei.dup, fei, nil, application_context, att)
-                    sle.fei.expression_id = "0.1"
+                    sle.fei.expression_id = "0.0"
                     sle.fei.expression_name = "sleep"
                     seq.children << sle.fei
                     seq.children << raw_expression.fei
@@ -968,6 +938,7 @@ module OpenWFE
                         { "every" => oevery }
                     end
                     att["name"] = "//cron_launch__#{fei.wfid}"
+                    att["scheduler-tags"] = "scheduled-launch"
 
                     cro = get_expression_map.get_class(:cron)
                     cro = cro.new(fei, nil, nil, application_context, att)
@@ -1128,7 +1099,7 @@ module OpenWFE
             #
             def build_raw_expression (launchitem, param)
 
-                procdef = determine_representation(param)
+                procdef = determine_representation param
 
                 #return procdef if procdef.is_a? RawExpression
 

data/lib/openwfe/expool/expstorage.rb

@@ -38,13 +38,13 @@
 # John Mettraux at openwfe.org
 #
 
-#require 'monitor'
 require 'fileutils'
 
 require 'openwfe/service'
 require 'openwfe/util/lru'
 require 'openwfe/flowexpressionid'
 
+
 module OpenWFE
 
     #
@@ -136,7 +136,7 @@ module OpenWFE
             fe = @cache[fei.hash]
             return fe if fe
 
-            ldebug { "[] (reload) for #{fei.to_debug_s}" }
+            #ldebug { "[] (reload) for #{fei.to_debug_s}" }
 
             fe = get_real_storage[fei]
 
@@ -151,6 +151,9 @@ module OpenWFE
         end
 
         def []= (fei, fe)
+
+            #linfo { "[]= caching #{fei}" }
+
             @cache[fei.hash] = fe
         end
 
@@ -244,10 +247,8 @@ module OpenWFE
         #
         def each_of_kind (kind, &block)
 
-            return unless block
-
             self.each_value do |fexp|
-                block.call(fexp) if fexp.kind_of?(kind)
+                block.call(fexp.fei, fexp) if fexp.kind_of?(kind)
             end
         end
 

data/lib/openwfe/expool/threadedexpstorage.rb

@@ -0,0 +1,190 @@
+#
+#--
+# Copyright (c) 2007, John Mettraux, OpenWFE.org
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without 
+# modification, are permitted provided that the following conditions are met:
+# 
+# . Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.  
+# 
+# . Redistributions in binary form must reproduce the above copyright notice, 
+#   this list of conditions and the following disclaimer in the documentation 
+#   and/or other materials provided with the distribution.
+# 
+# . Neither the name of the "OpenWFE" nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#++
+#
+
+#
+# "made in Japan"
+#
+# John Mettraux at openwfe.org
+#
+
+#require 'openwfe/flowexpressionid'
+
+
+module OpenWFE
+
+    #
+    # This mixin gathers all the logic for a threaded expression storage,
+    # one that doesn't immediately stores workitems (removes overriding
+    # operations).
+    # Using this threaded storage brings a very important perf benefit.
+    #
+    module ThreadedStorageMixin
+
+        THREADED_FREQ = "427" # milliseconds
+            #
+            # the frequency at which the event queue should be processed
+
+        #
+        # Will take care of stopping the 'queue processing' thread.
+        #
+        def stop
+
+            get_scheduler.unschedule(@thread_id) if @thread_id
+
+            process_queue()
+                #
+                # flush every remaining events (especially the :delete ones)
+        end
+
+        #
+        # calls process_queue() before the call the super class each_of_kind()
+        # method
+        #
+        def each_of_kind (kind, &block)
+
+            #ldebug { "each_of_kind()" }
+
+            process_queue()
+            super
+        end
+
+        #
+        # calls process_queue() before the call the super class each()
+        # method
+        #
+        def each (wfid_prefix=nil, &block)
+
+            process_queue()
+            super
+        end
+
+        protected
+
+            #
+            # starts the thread that does the actual persistence.
+            #
+            def start_processing_thread
+
+                @events = {}
+                @op_count = 0
+
+                @thread_id = get_scheduler.schedule_every THREADED_FREQ do
+                    process_queue
+                end
+            end
+
+            #
+            # queues an event for later (well within a second) persistence
+            #
+            def queue (event, fei, fe=nil)
+                synchronize do
+
+                    old_size = @events.size
+                    @op_count += 1
+
+                    @events[fei] = [ event, fei, fe ]
+
+                    ldebug do 
+                        "queue() ops #{@op_count} "+
+                        "size #{old_size} -> #{@events.size}"
+                    end
+                end
+            end
+
+            #
+            # the actual "do persist" order
+            #
+            def process_queue
+
+                return unless @events.size > 0
+                    #
+                    # trying to exit as quickly as possible
+
+                ldebug do 
+                    "process_queue() #{@events.size} events #{@op_count} ops"
+                end
+
+                synchronize do
+                    @events.each_value do |v|
+                        event = v[0]
+                        begin
+                            if event == :update
+                                self[v[1]] = v[2]
+                            else
+                                safe_delete(v[1])
+                            end
+                        rescue Exception => e
+                            lwarn do
+                                "process_queue() ':#{event}' exception\n" + 
+                                OpenWFE::exception_to_s(e)
+                            end
+                        end
+                    end
+                    @op_count = 0
+                    @events.clear
+                end
+            end
+
+            #
+            # a call to delete that tolerates missing .yaml files
+            #
+            def safe_delete (fei)
+                begin
+                    self.delete(fei)
+                rescue Exception => e
+                #    lwarn do
+                #        "safe_delete() exception\n" + 
+                #        OpenWFE::exception_to_s(e)
+                #    end
+                end
+            end
+
+            #
+            # Adds the queue() method as an observer to the update and remove
+            # events of the expression pool.
+            # :update and :remove mean changes to expressions in the persistence
+            # that's why they are observed.
+            #
+            def observe_expool
+
+                get_expression_pool.add_observer(:update) do |event, fei, fe|
+                    ldebug { ":update  for #{fei.to_debug_s}" }
+                    queue(event, fei, fe)
+                end
+                get_expression_pool.add_observer(:remove) do |event, fei|
+                    ldebug { ":remove  for #{fei.to_debug_s}" }
+                    queue(event, fei)
+                end
+            end
+    end
+end