ruote 2.1.9 → 2.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/CHANGELOG.txt +32 -0
  2. data/CREDITS.txt +3 -0
  3. data/Rakefile +4 -4
  4. data/TODO.txt +55 -11
  5. data/examples/barley.rb +2 -1
  6. data/examples/flickr_report.rb +5 -6
  7. data/examples/web_first_page.rb +11 -0
  8. data/lib/ruote/context.rb +36 -13
  9. data/lib/ruote/engine.rb +88 -56
  10. data/lib/ruote/engine/process_error.rb +13 -0
  11. data/lib/ruote/engine/process_status.rb +33 -1
  12. data/lib/ruote/error_handler.rb +122 -0
  13. data/lib/ruote/evt/tracker.rb +27 -10
  14. data/lib/ruote/exp/fe_apply.rb +69 -0
  15. data/lib/ruote/exp/fe_participant.rb +33 -5
  16. data/lib/ruote/exp/flowexpression.rb +37 -5
  17. data/lib/ruote/exp/ro_persist.rb +8 -4
  18. data/lib/ruote/exp/ro_variables.rb +2 -2
  19. data/lib/ruote/fei.rb +59 -7
  20. data/lib/ruote/log/storage_history.rb +2 -0
  21. data/lib/ruote/log/test_logger.rb +28 -19
  22. data/lib/ruote/log/wait_logger.rb +4 -2
  23. data/lib/ruote/parser.rb +2 -1
  24. data/lib/ruote/part/dispatch_pool.rb +10 -10
  25. data/lib/ruote/part/engine_participant.rb +2 -2
  26. data/lib/ruote/part/local_participant.rb +99 -7
  27. data/lib/ruote/part/participant_list.rb +18 -7
  28. data/lib/ruote/part/storage_participant.rb +9 -6
  29. data/lib/ruote/receiver/base.rb +109 -10
  30. data/lib/ruote/storage/base.rb +118 -41
  31. data/lib/ruote/storage/fs_storage.rb +1 -0
  32. data/lib/ruote/storage/hash_storage.rb +2 -1
  33. data/lib/ruote/util/lookup.rb +22 -2
  34. data/lib/ruote/util/misc.rb +5 -5
  35. data/lib/ruote/version.rb +1 -1
  36. data/lib/ruote/worker.rb +50 -63
  37. data/lib/ruote/workitem.rb +64 -0
  38. data/ruote.gemspec +17 -12
  39. data/test/functional/base.rb +3 -1
  40. data/test/functional/concurrent_base.rb +35 -29
  41. data/test/functional/crunner.sh +19 -0
  42. data/test/functional/ct_0_concurrence.rb +17 -30
  43. data/test/functional/ct_1_iterator.rb +20 -17
  44. data/test/functional/ct_2_cancel.rb +32 -25
  45. data/test/functional/eft_12_listen.rb +2 -1
  46. data/test/functional/eft_23_apply.rb +23 -0
  47. data/test/functional/eft_3_participant.rb +27 -0
  48. data/test/functional/ft_11_recursion.rb +1 -1
  49. data/test/functional/ft_13_variables.rb +22 -0
  50. data/test/functional/ft_14_re_apply.rb +3 -0
  51. data/test/functional/ft_15_timeout.rb +1 -0
  52. data/test/functional/ft_20_storage_participant.rb +20 -2
  53. data/test/functional/ft_21_forget.rb +30 -0
  54. data/test/functional/ft_22_process_definitions.rb +2 -1
  55. data/test/functional/ft_24_block_participants.rb +1 -1
  56. data/test/functional/ft_25_receiver.rb +83 -1
  57. data/test/functional/ft_26_participant_timeout.rb +1 -1
  58. data/test/functional/ft_2_errors.rb +2 -5
  59. data/test/functional/ft_30_smtp_participant.rb +47 -45
  60. data/test/functional/ft_36_storage_history.rb +4 -4
  61. data/test/functional/ft_37_engine_participant.rb +14 -10
  62. data/test/functional/ft_38_participant_more.rb +178 -0
  63. data/test/functional/ft_39_wait_for.rb +100 -0
  64. data/test/functional/ft_40_participant_on_reply.rb +87 -0
  65. data/test/functional/ft_41_participants.rb +65 -0
  66. data/test/functional/ft_42_storage_copy.rb +67 -0
  67. data/test/functional/ft_5_on_error.rb +103 -0
  68. data/test/functional/ft_9_subprocesses.rb +2 -1
  69. data/test/functional/storage_helper.rb +5 -1
  70. data/test/functional/test.rb +4 -1
  71. data/test/functional/vertical.rb +46 -0
  72. data/test/unit/storage.rb +17 -1
  73. data/test/unit/storages.rb +27 -7
  74. data/test/unit/ut_11_lookup.rb +36 -0
  75. data/test/unit/ut_16_parser.rb +43 -0
  76. data/test/unit/ut_1_fei.rb +28 -1
  77. data/test/unit/ut_7_workitem.rb +23 -0
  78. metadata +67 -105
  79. data/lib/ruote/log/fs_history.rb +0 -182
  80. data/test/functional/ft_32_fs_history.rb +0 -188
  81. data/test/mpc_test.rb +0 -29
@@ -54,8 +54,9 @@ module Ruote::Exp
54
54
 
55
55
  r = @context.storage.put(@h)
56
56
 
57
- #puts "+ per #{h.fei['expid']} #{tree.first} #{h._rev} --> #{r.class}"
58
- #Ruote.p_caller('+ per') if r != nil || h.fei['expid'] == '0_0'
57
+ #t = Thread.current.object_id.to_s[-3..-1]
58
+ #puts "+ per #{h.fei['expid']} #{tree.first} #{h._rev} #{t} -> #{r.class}"
59
+ #Ruote.p_caller('+ per') #if r != nil || h.fei['expid'] == '0_0'
59
60
 
60
61
  r
61
62
  end
@@ -64,8 +65,9 @@ module Ruote::Exp
64
65
 
65
66
  r = @context.storage.delete(@h)
66
67
 
67
- #puts "- unp #{h.fei['expid']} #{tree.first} #{h._rev} --> #{r.class}"
68
- #Ruote.p_caller('- unp') if r != nil || h.fei['expid'] == '0_0'
68
+ #t = Thread.current.object_id.to_s[-3..-1]
69
+ #puts "- unp #{h.fei['expid']} #{tree.first} #{h._rev} #{t} -> #{r.class}"
70
+ #Ruote.p_caller('- unp') #if r != nil || h.fei['expid'] == '0_0'
69
71
 
70
72
  return r if r
71
73
 
@@ -73,6 +75,8 @@ module Ruote::Exp
73
75
  err = @context.storage.get('errors', "err_#{Ruote.to_storage_id(h.fei)}")
74
76
  @context.storage.delete(err) if err
75
77
  #end
78
+ # removes any error in the journal for this expression
79
+ # since it will now be gone, no need to keep track of its errors
76
80
 
77
81
  nil
78
82
  end
@@ -139,9 +139,9 @@ module Ruote::Exp
139
139
  def un_set_variable (op, var, val, should_persist)
140
140
 
141
141
  if op == :set
142
- h.variables[var] = val
142
+ Ruote.set(h.variables, var, val)
143
143
  else # op == :unset
144
- h.variables.delete(var)
144
+ Ruote.unset(h.variables, var)
145
145
  end
146
146
 
147
147
  return unless should_persist
@@ -23,6 +23,7 @@
23
23
  #++
24
24
 
25
25
  require 'ruote/version'
26
+ require 'ruote/workitem'
26
27
  require 'ruote/util/misc'
27
28
  require 'ruote/util/hashdot'
28
29
 
@@ -38,6 +39,15 @@ module Ruote
38
39
  Ruote::FlowExpressionId.to_storage_id(fei)
39
40
  end
40
41
 
42
+ # A shorter shortcut for
43
+ #
44
+ # Ruote::FlowExpressionId.to_storage_id(fei)
45
+ #
46
+ def self.sid (fei)
47
+
48
+ Ruote::FlowExpressionId.to_storage_id(fei)
49
+ end
50
+
41
51
  #
42
52
  # The FlowExpressionId (fei for short) is an process expression identifier.
43
53
  # Each expression when instantiated gets a unique fei.
@@ -78,21 +88,26 @@ module Ruote
78
88
  @h['sub_wfid']
79
89
  end
80
90
 
91
+ def engine_id
92
+ @h['engine_id']
93
+ end
94
+
81
95
  def to_storage_id
82
96
  "#{@h['expid']}!#{@h['sub_wfid']}!#{@h['wfid']}"
83
97
  end
84
98
 
85
99
  def self.to_storage_id (hfei)
86
- "#{hfei['expid']}!#{hfei['sub_wfid']}!#{hfei['wfid']}"
100
+
101
+ hfei.respond_to?(:to_storage_id) ?
102
+ hfei.to_storage_id :
103
+ "#{hfei['expid']}!#{hfei['sub_wfid']}!#{hfei['wfid']}"
87
104
  end
88
105
 
106
+ # Turns the result of to_storage_id back to a FlowExpressionId instance.
107
+ #
89
108
  def self.from_id (s, engine_id='engine')
90
109
 
91
- ss = s.split('!')
92
-
93
- FlowExpressionId.new(
94
- 'engine_id' => engine_id,
95
- 'expid' => ss[-3], 'sub_wfid' => ss[-2], 'wfid' => ss[-1])
110
+ extract("#{engine_id}!#{s}")
96
111
  end
97
112
 
98
113
  # Returns the last number in the expid. For instance, if the expid is
@@ -139,10 +154,47 @@ module Ruote
139
154
  return false if parent_fei[k] != other_fei[k]
140
155
  end
141
156
 
142
- pei = other_fei['expid'].split(CHILD_SEP)[0..-2].join('_')
157
+ pei = other_fei['expid'].split(CHILD_SEP)[0..-2].join(CHILD_SEP)
143
158
 
144
159
  (pei == parent_fei['expid'])
145
160
  end
161
+
162
+ # Attempts at extracting a FlowExpressionId from the given argument
163
+ # (workitem, string, ...)
164
+ #
165
+ # Uses .extract_h
166
+ #
167
+ def self.extract (arg)
168
+
169
+ FlowExpressionId.new(extract_h(arg))
170
+ end
171
+
172
+ # Attempts at extracting a FlowExpressionId (as a Hash instance) from the
173
+ # given argument (workitem, string, ...)
174
+ #
175
+ def self.extract_h (arg)
176
+
177
+ if arg.is_a?(Hash)
178
+ return arg if arg['expid']
179
+ return arg['fei'] if arg['fei']
180
+ end
181
+
182
+ return extract_h(arg.fei) if arg.respond_to?(:fei)
183
+ return arg.h if arg.is_a?(Ruote::FlowExpressionId)
184
+ return arg.h['fei'] if arg.is_a?(Ruote::Workitem)
185
+
186
+ if arg.is_a?(String)
187
+
188
+ ss = arg.split('!')
189
+
190
+ return {
191
+ 'engine_id' => ss[-4] || 'engine',
192
+ 'expid' => ss[-3], 'sub_wfid' => ss[-2], 'wfid' => ss[-1] }
193
+ end
194
+
195
+ raise ArgumentError.new(
196
+ "couldn't extract fei out of instance of #{arg.class}")
197
+ end
146
198
  end
147
199
  end
148
200
 
@@ -62,6 +62,8 @@ module Ruote
62
62
 
63
63
  ids = @context.storage.ids('history')
64
64
 
65
+ #p ids.sort == ids
66
+
65
67
  fm = DATE_REGEX.match(ids.first)[1]
66
68
  lm = DATE_REGEX.match(ids.last)[1]
67
69
 
@@ -74,7 +74,7 @@ module Ruote
74
74
  check_waiting
75
75
  end
76
76
 
77
- def wait_for (*interests)
77
+ def wait_for (interests)
78
78
 
79
79
  @waiting = [ Thread.current, interests ]
80
80
 
@@ -82,6 +82,8 @@ module Ruote
82
82
 
83
83
  Thread.stop if @waiting
84
84
 
85
+ # and when this thread gets woken up, go on and return __result__
86
+
85
87
  Thread.current['__result__']
86
88
  end
87
89
 
@@ -131,40 +133,45 @@ module Ruote
131
133
  end
132
134
  end
133
135
 
134
- def check_interest (msg)
136
+ FINAL_ACTIONS = %w[ terminated ceased error_intercepted ]
135
137
 
136
- over = false
138
+ def check_interest (msg)
137
139
 
138
140
  action = msg['action']
139
141
 
140
- Array(@waiting.last).each do |interest|
142
+ @waiting.last.each do |interest|
141
143
 
142
- over = if interest.is_a?(Symbol) # participant
144
+ satisfied = if interest == :inactive
143
145
 
144
- (action == 'dispatch' && msg['participant_name'] == interest.to_s)
146
+ (FINAL_ACTIONS.include?(action) && @context.worker.inactive?)
145
147
 
146
- elsif interest.is_a?(Fixnum)
148
+ elsif interest == :empty
147
149
 
148
- interest = interest - 1
149
- @waiting = [ @waiting.first, interest ]
150
+ (action == 'terminated' && @context.storage.empty?('expressions'))
150
151
 
151
- #@debug ||= {}
152
- #c = @debug[action] ||= 0
153
- #@debug[action] = c+1
154
- #p [ interest, @debug ]
152
+ elsif interest.is_a?(Symbol) # participant
155
153
 
156
- (interest < 1)
154
+ (action == 'dispatch' && msg['participant_name'] == interest.to_s)
155
+
156
+ elsif interest.is_a?(Fixnum)
157
+
158
+ @waiting[-1] = @waiting[-1] - [ interest ]
159
+ if (interest > 1)
160
+ @waiting[-1] << (interest - 1)
161
+ false
162
+ else
163
+ true
164
+ end
157
165
 
158
166
  else # wfid
159
167
 
160
- %w[ terminated ceased error_intercepted ].include?(action) &&
161
- msg['wfid'] == interest
168
+ (FINAL_ACTIONS.include?(action) && msg['wfid'] == interest)
162
169
  end
163
170
 
164
- break if over
171
+ @waiting[-1] = @waiting[-1] - [ interest ] if satisfied
165
172
  end
166
173
 
167
- over
174
+ @waiting.last.size < 1
168
175
  end
169
176
 
170
177
  # <ESC>[{attr1};...;{attrn}m
@@ -244,6 +251,7 @@ module Ruote
244
251
  action = msg['action'][0, 2]
245
252
  action = case msg['action']
246
253
  when 'receive' then 'rc'
254
+ when 'dispatched' then 'dd'
247
255
  when 'dispatch_cancel' then 'dc'
248
256
  else action
249
257
  end
@@ -253,8 +261,9 @@ module Ruote
253
261
  when 'ce' then color('31', action)
254
262
  when 'ca' then color('31', action)
255
263
  when 'rc' then color('4;33', action)
256
- when 'dc' then color('4;31', action)
257
264
  when 'di' then color('4;33', action)
265
+ when 'dd' then color('4;33', action)
266
+ when 'dc' then color('4;31', action)
258
267
  else action
259
268
  end
260
269
 
@@ -55,12 +55,14 @@ module Ruote
55
55
  check_msg(msg)
56
56
  end
57
57
 
58
- def wait_for (interest)
58
+ def wait_for (interests)
59
59
 
60
- @waiting = [ Thread.current, interest ]
60
+ @waiting = [ Thread.current, interests ]
61
61
 
62
62
  Thread.stop
63
63
 
64
+ # and when this thread gets woken up, go on and return __result__
65
+
64
66
  Thread.current['__result__']
65
67
  end
66
68
  end
@@ -104,7 +104,8 @@ module Ruote
104
104
  t = atts.find { |k, v| v == nil }
105
105
  if t
106
106
  atts.delete(t.first)
107
- atts['ref'] = t.first
107
+ key = tree[0] == 'if' ? 'test' : 'ref'
108
+ atts[key] = t.first
108
109
  end
109
110
 
110
111
  atts = atts.inject({}) { |h, (k, v)| h[k.to_s.gsub(/\_/, '-')] = v; h }
@@ -89,25 +89,25 @@ module Ruote
89
89
  workitem.fields['dispatched_at'] = Ruote.now_to_utc_s
90
90
 
91
91
  participant.consume(workitem)
92
+
93
+ @context.storage.put_msg('dispatched', 'fei' => msg['fei'])
94
+ # once the consume is done, asynchronously flag the
95
+ # participant expression as 'dispatched'
92
96
  end
93
97
 
94
98
  def do_threaded_dispatch (participant, msg)
95
99
 
100
+ # Maybe at some point a limit on the number of dispatch threads
101
+ # would be OK.
102
+ # Or maybe it's the job of an extension / subclass
103
+
96
104
  Thread.new do
97
105
  begin
98
106
 
99
107
  do_dispatch(participant, msg)
100
108
 
101
- rescue Exception => e
102
-
103
- #puts '/' * 80
104
- #p e
105
- #puts '/' * 80
106
-
107
- @context.worker.handle_exception(
108
- msg,
109
- Ruote::Exp::FlowExpression.fetch(@context, msg['workitem']['fei']),
110
- e)
109
+ rescue Exception => exception
110
+ @context.error_handler.msg_handle(msg, exception)
111
111
  end
112
112
  end
113
113
  end
@@ -63,7 +63,7 @@ module Ruote
63
63
  # detail the class and the arguments to the storage of the target engine.
64
64
  #
65
65
  # This example is a bit dry / flat. A real world example would perhaps detail
66
- # a 'master' engine connected to 'departemental' engines, something more
66
+ # a 'master' engine connected to 'departmental' engines, something more
67
67
  # hierarchical.
68
68
  #
69
69
  # The example also binds reciprocally engines. If the delegated processes
@@ -105,7 +105,7 @@ module Ruote
105
105
 
106
106
  include LocalParticipant
107
107
 
108
- def initialize (opts=nil)
108
+ def initialize (opts)
109
109
 
110
110
  if pa = opts['storage_path']
111
111
  require pa
@@ -22,6 +22,8 @@
22
22
  # Made in Japan.
23
23
  #++
24
24
 
25
+ require 'ruote/receiver/base'
26
+
25
27
 
26
28
  module Ruote
27
29
 
@@ -31,22 +33,112 @@ module Ruote
31
33
  # Assumes the class that includes this module has a #context method
32
34
  # that points to the worker or engine ruote context.
33
35
  #
36
+ # It's "local" because it has access to the ruote storage.
37
+ #
34
38
  module LocalParticipant
35
39
 
40
+ include ReceiverMixin
41
+ # the reply_to_engine method is there
42
+
36
43
  attr_accessor :context
37
44
 
38
- # Sends back the workitem to the ruote engine.
45
+ # Use this method to re_dispatch the workitem.
39
46
  #
40
- def reply_to_engine (workitem)
41
-
42
- # the local participant knows how to deal with the storage directly
47
+ # It takes two options :in and :at for "later re_dispatch".
48
+ #
49
+ # Look at the unschedule_re_dispatch method for an example of
50
+ # participant implementation that uses re_dispatch.
51
+ #
52
+ # Without one of those options, the method is a "reject".
53
+ #
54
+ def re_dispatch (workitem, opts={})
43
55
 
44
- @context.storage.put_msg(
45
- 'receive',
56
+ msg = {
57
+ 'action' => 'dispatch',
46
58
  'fei' => workitem.h.fei,
47
59
  'workitem' => workitem.h,
48
- 'participant_name' => workitem.participant_name)
60
+ 'participant_name' => workitem.participant_name,
61
+ 'rejected' => true
62
+ }
63
+
64
+ if t = opts[:in] || opts[:at]
65
+
66
+ sched_id = @context.storage.put_schedule('at', workitem.h.fei, t, msg)
67
+
68
+ fexp = fetch_flow_expression(workitem)
69
+ fexp.h['re_dispatch_sched_id'] = sched_id
70
+ fexp.try_persist
71
+
72
+ else
73
+
74
+ @context.storage.put_msg('dispatch', msg)
75
+ end
76
+ end
77
+
78
+ # Cancels the scheduled re_dispatch, if any.
79
+ #
80
+ # An example or 'retrying participant' :
81
+ #
82
+ # class RetryParticipant
83
+ # include Ruote::LocalParticipant
84
+ #
85
+ # def initialize (opts)
86
+ # @opts = opts
87
+ # end
88
+ #
89
+ # def consume (workitem)
90
+ # begin
91
+ # do_the_job
92
+ # reply(workitem)
93
+ # rescue
94
+ # re_dispatch(workitem, :in => @opts['delay'] || '1s')
95
+ # end
96
+ # end
97
+ #
98
+ # def cancel (fei, flavour)
99
+ # unschedule_re_dispatch(fei)
100
+ # end
101
+ # end
102
+ #
103
+ # Note how unschedule_re_dispatch is used in the cancel method. Warning,
104
+ # this example could loop forever...
105
+ #
106
+ def unschedule_re_dispatch (fei)
107
+
108
+ fexp = Ruote::Exp::FlowExpression.fetch(
109
+ @context, Ruote::FlowExpressionId.extract_h(fei))
110
+
111
+ if s = fexp.h['re_dispatch_sched_id']
112
+ @context.storage.delete_schedule(s)
113
+ end
49
114
  end
115
+
116
+ # WARNING : this method is only for 'stateless' participants, ie
117
+ # participants that are registered in the engine by passing their class
118
+ # and a set of options, like in
119
+ #
120
+ # engine.register_participant 'alpha', MyParticipant, 'info' => 'none'
121
+ #
122
+ # This reject method replaces the workitem in the [internal] message queue
123
+ # of the ruote engine (since it's a local participant, it has access to
124
+ # the storage and it's thus easy).
125
+ # The idea is that another worker will pick up the workitem and
126
+ # do the participant dispatching.
127
+ #
128
+ # This is an advanced technique. It was requested by people who
129
+ # want to have multiple workers and have only certain worker/participants
130
+ # do the handling.
131
+ # Using reject is not the best method, it's probably better to implement
132
+ # this by re-opening the Ruote::Worker class and changing the
133
+ # cannot_handle(msg) method.
134
+ #
135
+ # reject could be useful anyway, not sure now, but one could imagine
136
+ # scenarii where some participants reject workitems temporarily (while
137
+ # the same participant on another worker would accept it).
138
+ #
139
+ # Well, here it is, use with care.
140
+ #
141
+ alias :reject :re_dispatch
50
142
  end
51
143
  end
52
144