ruote 2.1.9 → 2.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/CHANGELOG.txt +32 -0
  2. data/CREDITS.txt +3 -0
  3. data/Rakefile +4 -4
  4. data/TODO.txt +55 -11
  5. data/examples/barley.rb +2 -1
  6. data/examples/flickr_report.rb +5 -6
  7. data/examples/web_first_page.rb +11 -0
  8. data/lib/ruote/context.rb +36 -13
  9. data/lib/ruote/engine.rb +88 -56
  10. data/lib/ruote/engine/process_error.rb +13 -0
  11. data/lib/ruote/engine/process_status.rb +33 -1
  12. data/lib/ruote/error_handler.rb +122 -0
  13. data/lib/ruote/evt/tracker.rb +27 -10
  14. data/lib/ruote/exp/fe_apply.rb +69 -0
  15. data/lib/ruote/exp/fe_participant.rb +33 -5
  16. data/lib/ruote/exp/flowexpression.rb +37 -5
  17. data/lib/ruote/exp/ro_persist.rb +8 -4
  18. data/lib/ruote/exp/ro_variables.rb +2 -2
  19. data/lib/ruote/fei.rb +59 -7
  20. data/lib/ruote/log/storage_history.rb +2 -0
  21. data/lib/ruote/log/test_logger.rb +28 -19
  22. data/lib/ruote/log/wait_logger.rb +4 -2
  23. data/lib/ruote/parser.rb +2 -1
  24. data/lib/ruote/part/dispatch_pool.rb +10 -10
  25. data/lib/ruote/part/engine_participant.rb +2 -2
  26. data/lib/ruote/part/local_participant.rb +99 -7
  27. data/lib/ruote/part/participant_list.rb +18 -7
  28. data/lib/ruote/part/storage_participant.rb +9 -6
  29. data/lib/ruote/receiver/base.rb +109 -10
  30. data/lib/ruote/storage/base.rb +118 -41
  31. data/lib/ruote/storage/fs_storage.rb +1 -0
  32. data/lib/ruote/storage/hash_storage.rb +2 -1
  33. data/lib/ruote/util/lookup.rb +22 -2
  34. data/lib/ruote/util/misc.rb +5 -5
  35. data/lib/ruote/version.rb +1 -1
  36. data/lib/ruote/worker.rb +50 -63
  37. data/lib/ruote/workitem.rb +64 -0
  38. data/ruote.gemspec +17 -12
  39. data/test/functional/base.rb +3 -1
  40. data/test/functional/concurrent_base.rb +35 -29
  41. data/test/functional/crunner.sh +19 -0
  42. data/test/functional/ct_0_concurrence.rb +17 -30
  43. data/test/functional/ct_1_iterator.rb +20 -17
  44. data/test/functional/ct_2_cancel.rb +32 -25
  45. data/test/functional/eft_12_listen.rb +2 -1
  46. data/test/functional/eft_23_apply.rb +23 -0
  47. data/test/functional/eft_3_participant.rb +27 -0
  48. data/test/functional/ft_11_recursion.rb +1 -1
  49. data/test/functional/ft_13_variables.rb +22 -0
  50. data/test/functional/ft_14_re_apply.rb +3 -0
  51. data/test/functional/ft_15_timeout.rb +1 -0
  52. data/test/functional/ft_20_storage_participant.rb +20 -2
  53. data/test/functional/ft_21_forget.rb +30 -0
  54. data/test/functional/ft_22_process_definitions.rb +2 -1
  55. data/test/functional/ft_24_block_participants.rb +1 -1
  56. data/test/functional/ft_25_receiver.rb +83 -1
  57. data/test/functional/ft_26_participant_timeout.rb +1 -1
  58. data/test/functional/ft_2_errors.rb +2 -5
  59. data/test/functional/ft_30_smtp_participant.rb +47 -45
  60. data/test/functional/ft_36_storage_history.rb +4 -4
  61. data/test/functional/ft_37_engine_participant.rb +14 -10
  62. data/test/functional/ft_38_participant_more.rb +178 -0
  63. data/test/functional/ft_39_wait_for.rb +100 -0
  64. data/test/functional/ft_40_participant_on_reply.rb +87 -0
  65. data/test/functional/ft_41_participants.rb +65 -0
  66. data/test/functional/ft_42_storage_copy.rb +67 -0
  67. data/test/functional/ft_5_on_error.rb +103 -0
  68. data/test/functional/ft_9_subprocesses.rb +2 -1
  69. data/test/functional/storage_helper.rb +5 -1
  70. data/test/functional/test.rb +4 -1
  71. data/test/functional/vertical.rb +46 -0
  72. data/test/unit/storage.rb +17 -1
  73. data/test/unit/storages.rb +27 -7
  74. data/test/unit/ut_11_lookup.rb +36 -0
  75. data/test/unit/ut_16_parser.rb +43 -0
  76. data/test/unit/ut_1_fei.rb +28 -1
  77. data/test/unit/ut_7_workitem.rb +23 -0
  78. metadata +67 -105
  79. data/lib/ruote/log/fs_history.rb +0 -182
  80. data/test/functional/ft_32_fs_history.rb +0 -188
  81. data/test/mpc_test.rb +0 -29
@@ -93,8 +93,9 @@ module Ruote
93
93
 
94
94
  else
95
95
 
96
- return Ruote::StorageParticipant.new(@context) \
97
- if entry.last.first == 'Ruote::StorageParticipant'
96
+ if entry.last.first == 'Ruote::StorageParticipant'
97
+ return Ruote::StorageParticipant.new(@context)
98
+ end
98
99
 
99
100
  nil
100
101
  end
@@ -110,8 +111,9 @@ module Ruote
110
111
  code = nil
111
112
  entry = nil
112
113
 
113
- name_or_participant = name_or_participant.to_s \
114
- if name_or_participant.is_a?(Symbol)
114
+ if name_or_participant.is_a?(Symbol)
115
+ name_or_participant = name_or_participant.to_s
116
+ end
115
117
 
116
118
  if name_or_participant.is_a?(String)
117
119
 
@@ -160,12 +162,12 @@ module Ruote
160
162
  end
161
163
  end
162
164
 
163
- def lookup (participant_name)
165
+ def lookup (participant_name, opts={})
164
166
 
165
167
  pi = lookup_info(participant_name)
166
168
 
167
169
  return nil unless pi
168
- return pi unless pi.is_a?(Array)
170
+ return opts[:on_reply] ? nil : pi unless pi.is_a?(Array)
169
171
 
170
172
  class_name, options = pi
171
173
 
@@ -173,8 +175,17 @@ module Ruote
173
175
  require(rp)
174
176
  end
175
177
 
176
- pa = Ruote.constantize(class_name).new(options)
178
+ pa_class = Ruote.constantize(class_name)
179
+ pa_m = pa_class.instance_methods
180
+
181
+ return nil if opts[:on_reply] && ! (
182
+ pa_m.include?(:on_reply) || pa_m.include?('on_reply'))
177
183
 
184
+ pa = if pa_class.instance_method(:initialize).arity > 0
185
+ pa_class.new(options)
186
+ else
187
+ pa_class.new
188
+ end
178
189
  pa.context = @context if pa.respond_to?(:context=)
179
190
 
180
191
  pa
@@ -96,7 +96,7 @@ module Ruote
96
96
  #
97
97
  def cancel (fei, flavour)
98
98
 
99
- doc = fetch(fei.to_h)
99
+ doc = fetch(fei)
100
100
 
101
101
  r = @context.storage.delete(doc)
102
102
 
@@ -112,16 +112,18 @@ module Ruote
112
112
 
113
113
  def fetch (fei)
114
114
 
115
- fei = fei.to_h if fei.respond_to?(:to_h)
115
+ hfei = Ruote::FlowExpressionId.extract_h(fei)
116
116
 
117
- @context.storage.get('workitems', to_id(fei))
117
+ @context.storage.get('workitems', to_id(hfei))
118
118
  end
119
119
 
120
120
  # Removes the workitem from the in-memory hash and replies to the engine.
121
121
  #
122
122
  def reply (workitem)
123
123
 
124
- doc = fetch(workitem.fei.to_h)
124
+ # TODO: change method name (receiver mess cleanup)
125
+
126
+ doc = fetch(Ruote::FlowExpressionId.extract_h(workitem))
125
127
 
126
128
  r = @context.storage.delete(doc)
127
129
 
@@ -237,8 +239,9 @@ module Ruote
237
239
 
238
240
  cr = criteria.inject({}) { |h, (k, v)| h[k.to_s] = v; h }
239
241
 
240
- return @context.storage.query_workitems(cr) \
241
- if @context.storage.respond_to?(:query_workitems)
242
+ return @context.storage.query_workitems(cr).collect { |h|
243
+ Ruote::Workitem.new(h)
244
+ } if @context.storage.respond_to?(:query_workitems)
242
245
 
243
246
  offset = cr.delete('offset')
244
247
  limit = cr.delete('limit')
@@ -29,20 +29,19 @@ module Ruote
29
29
  # The core methods for the Receiver class (sometimes a Mixin is easier
30
30
  # to integrate).
31
31
  #
32
- # (The engine itself includes this mixin)
32
+ # (The engine itself includes this mixin, the LocalParticipant module
33
+ # includes it as well).
33
34
  #
34
35
  module ReceiverMixin
35
36
 
36
- def receive (item)
37
-
38
- reply(item)
39
- end
40
-
41
- def reply (workitem)
37
+ # This method pipes back a workitem into the engine, letting it resume
38
+ # in its flow, hopefully.
39
+ #
40
+ def receive (workitem)
42
41
 
43
42
  workitem = workitem.to_h if workitem.respond_to?(:to_h)
44
43
 
45
- @storage.put_msg(
44
+ @context.storage.put_msg(
46
45
  'receive',
47
46
  'fei' => workitem['fei'],
48
47
  'workitem' => workitem,
@@ -50,10 +49,108 @@ module Ruote
50
49
  'receiver' => sign)
51
50
  end
52
51
 
52
+ # Given a process definitions and optional initial fields and variables,
53
+ # launches a new process instance.
54
+ #
55
+ # This method is mostly used from the Ruote::Engine class (which includes
56
+ # this mixin).
57
+ #
58
+ def launch (process_definition, fields={}, variables={})
59
+
60
+ wfid = @context.wfidgen.generate
61
+
62
+ @context.storage.put_msg(
63
+ 'launch',
64
+ 'wfid' => wfid,
65
+ 'tree' => @context.parser.parse(process_definition),
66
+ 'workitem' => { 'fields' => fields },
67
+ 'variables' => variables)
68
+
69
+ wfid
70
+ end
71
+
72
+ # Wraps a call to receive(workitem)
73
+ #
74
+ # Not aliasing so that if someone changes the receive implementation,
75
+ # reply is affected as well.
76
+ #
77
+ def reply (workitem)
78
+
79
+ receive (workitem)
80
+ end
81
+
82
+ # Wraps a call to receive(workitem)
83
+ #
84
+ # Not aliasing so that if someone changes the receive implementation,
85
+ # reply_to_engine is affected as well.
86
+ #
87
+ def reply_to_engine (workitem)
88
+
89
+ receive (workitem)
90
+ end
91
+
92
+ # A receiver signs a workitem when it comes back.
93
+ #
94
+ # Not used much as of now.
95
+ #
53
96
  def sign
54
97
 
55
98
  self.class.to_s
56
99
  end
100
+
101
+ protected
102
+
103
+ # Convenience method, fetches the flow expression (ParticipantExpression)
104
+ # that emitted that workitem.
105
+ #
106
+ def fetch_flow_expression (workitem)
107
+
108
+ Ruote::Exp::FlowExpression.fetch(@context, workitem.fei.to_h)
109
+ end
110
+
111
+ # Stashes values in the participant expression (in the storage).
112
+ #
113
+ # put(workitem.fei, 'key' => 'value', 'colour' => 'blue')
114
+ #
115
+ # Remember that keys/values must be serializable in JSON.
116
+ #
117
+ # put & get are useful for a participant that needs to communicate
118
+ # between its consume and its cancel.
119
+ #
120
+ # See the thread at
121
+ # http://groups.google.com/group/openwferu-users/t/2e6a95708c10847b for the
122
+ # justification.
123
+ #
124
+ def put (fei, hash)
125
+
126
+ fexp = Ruote::Exp::FlowExpression.fetch(@context, fei.to_h)
127
+
128
+ (fexp.h['stash'] ||= {}).merge!(hash)
129
+
130
+ fexp.persist_or_raise
131
+ end
132
+
133
+ # Fetches back a stashed value.
134
+ #
135
+ # get(fei, 'colour')
136
+ # # => 'blue'
137
+ #
138
+ # To return the whole stash
139
+ #
140
+ # get(fei)
141
+ # # => { 'colour' => 'blue' }
142
+ #
143
+ # put & get are useful for a participant that needs to communicate
144
+ # between its consume and its cancel.
145
+ #
146
+ def get (fei, key=nil)
147
+
148
+ fexp = Ruote::Exp::FlowExpression.fetch(@context, fei.to_h)
149
+
150
+ stash = fexp.h['stash'] rescue {}
151
+
152
+ key ? stash[key] : stash
153
+ end
57
154
  end
58
155
 
59
156
  #
@@ -63,9 +160,11 @@ module Ruote
63
160
  class Receiver
64
161
  include ReceiverMixin
65
162
 
66
- def initialize (storage, options={})
163
+ # Accepts context, worker, engine or storage as first argument.
164
+ #
165
+ def initialize (cwes, options={})
67
166
 
68
- @storage = storage
167
+ @context = cwes.context
69
168
  @options = options
70
169
  end
71
170
  end
@@ -32,6 +32,24 @@ module Ruote
32
32
  #
33
33
  module StorageBase
34
34
 
35
+ def context
36
+
37
+ @context ||= Ruote::Context.new(self)
38
+ end
39
+
40
+ def context= (c)
41
+
42
+ @context = c
43
+ end
44
+
45
+ # Attempts to delete a document, returns true if the deletion
46
+ # succeeded. This is used with msgs to reserve work on them.
47
+ #
48
+ def reserve (doc)
49
+
50
+ delete(doc).nil?
51
+ end
52
+
35
53
  #--
36
54
  # configurations
37
55
  #++
@@ -47,28 +65,16 @@ module Ruote
47
65
 
48
66
  def put_msg (action, options)
49
67
 
50
- # merge! is way faster than merge (no object creation probably)
51
-
52
- @counter ||= 0
53
-
54
- t = Time.now.utc
55
- ts = "#{t.strftime('%Y-%m-%d')}!#{t.to_i}.#{'%06d' % t.usec}"
56
- _id = "#{$$}!#{Thread.current.object_id}!#{ts}!#{'%03d' % @counter}"
57
-
58
- @counter = (@counter + 1) % 1000
59
- # some platforms (windows) have shallow usecs, so adding that counter...
60
-
61
- msg = options.merge!('type' => 'msgs', '_id' => _id, 'action' => action)
62
-
63
- msg.delete('_rev')
64
- # in case of message replay
68
+ msg = prepare_msg_doc(action, options)
65
69
 
66
70
  put(msg)
71
+
67
72
  #put(msg, :update_rev => true)
68
- #(@local_msgs ||= []) << options
73
+ #(@local_msgs ||= []) << Ruote.fulldup(msg)
69
74
  end
70
75
 
71
76
  #def get_local_msgs
77
+ # p @local_msgs
72
78
  # if @local_msgs
73
79
  # r = @local_msgs
74
80
  # @local_msgs = nil
@@ -87,6 +93,11 @@ module Ruote
87
93
  }
88
94
  end
89
95
 
96
+ def empty? (type)
97
+
98
+ (get_many(type) == [])
99
+ end
100
+
90
101
  #--
91
102
  # expressions
92
103
  #++
@@ -116,6 +127,9 @@ module Ruote
116
127
 
117
128
  def get_schedules (delta, now)
118
129
 
130
+ # TODO : bring that 'optimization' back in,
131
+ # maybe every minute, if min != last_min ...
132
+
119
133
  #if delta < 1.0
120
134
  # at = now.strftime('%Y%m%d%H%M%S')
121
135
  # get_many('schedules', /-#{at}$/)
@@ -133,33 +147,12 @@ module Ruote
133
147
 
134
148
  def put_schedule (flavour, owner_fei, s, msg)
135
149
 
136
- at = if s.is_a?(Time) # at or every
137
- s
138
- elsif Ruote.is_cron_string(s) # cron
139
- Rufus::CronLine.new(s).next_time(Time.now + 1)
140
- else # at or every
141
- Ruote.s_to_at(s)
142
- end
143
- at = at.utc
144
-
145
- if at <= Time.now.utc && flavour == 'at'
146
- put_msg(msg.delete('action'), msg)
147
- return
150
+ if doc = prepare_schedule_doc(flavour, owner_fei, s, msg)
151
+ put(doc)
152
+ return doc['_id']
148
153
  end
149
154
 
150
- sat = at.strftime('%Y%m%d%H%M%S')
151
- i = "#{flavour}-#{Ruote.to_storage_id(owner_fei)}-#{sat}"
152
-
153
- put(
154
- '_id' => i,
155
- 'type' => 'schedules',
156
- 'flavour' => flavour,
157
- 'original' => s,
158
- 'at' => Ruote.time_to_utc_s(at),
159
- 'owner' => owner_fei,
160
- 'msg' => msg)
161
-
162
- i
155
+ nil
163
156
  end
164
157
 
165
158
  def delete_schedule (schedule_id)
@@ -185,8 +178,92 @@ module Ruote
185
178
  put_engine_variable(k, v) unless put(vars).nil?
186
179
  end
187
180
 
181
+ #--
182
+ # migrations
183
+ #++
184
+
185
+ # Copies the content of this storage into a target storage.
186
+ #
187
+ # Of course, the target storage may be a different implementation.
188
+ #
189
+ def copy_to (target, opts={})
190
+
191
+ counter = 0
192
+
193
+ %w[
194
+ configurations errors expressions msgs schedules variables workitems
195
+ ].each do |type|
196
+
197
+ get_many(type).each do |item|
198
+
199
+ item.delete('_rev')
200
+ target.put(item)
201
+
202
+ counter += 1
203
+ puts(" #{type}/#{item['_id']}") if opts[:verbose]
204
+ end
205
+ end
206
+
207
+ counter
208
+ end
209
+
188
210
  protected
189
211
 
212
+ # Used by put_msg
213
+ #
214
+ def prepare_msg_doc (action, options)
215
+
216
+ # merge! is way faster than merge (no object creation probably)
217
+
218
+ @counter ||= 0
219
+
220
+ t = Time.now.utc
221
+ ts = "#{t.strftime('%Y-%m-%d')}!#{t.to_i}.#{'%06d' % t.usec}"
222
+ _id = "#{$$}!#{Thread.current.object_id}!#{ts}!#{'%03d' % @counter}"
223
+
224
+ @counter = (@counter + 1) % 1000
225
+ # some platforms (windows) have shallow usecs, so adding that counter...
226
+
227
+ msg = options.merge!('type' => 'msgs', '_id' => _id, 'action' => action)
228
+
229
+ msg.delete('_rev')
230
+ # in case of message replay
231
+
232
+ msg
233
+ end
234
+
235
+ # Used by put_schedule
236
+ #
237
+ def prepare_schedule_doc (flavour, owner_fei, s, msg)
238
+
239
+ at = if s.is_a?(Time) # at or every
240
+ s
241
+ elsif Ruote.is_cron_string(s) # cron
242
+ Rufus::CronLine.new(s).next_time(Time.now + 1)
243
+ else # at or every
244
+ Ruote.s_to_at(s)
245
+ end
246
+ at = at.utc
247
+
248
+ if at <= Time.now.utc && flavour == 'at'
249
+ put_msg(msg.delete('action'), msg)
250
+ return false
251
+ end
252
+
253
+ sat = at.strftime('%Y%m%d%H%M%S')
254
+ i = "#{flavour}-#{Ruote.to_storage_id(owner_fei)}-#{sat}"
255
+
256
+ {
257
+ '_id' => i,
258
+ 'type' => 'schedules',
259
+ 'flavour' => flavour,
260
+ 'original' => s,
261
+ 'at' => Ruote.time_to_utc_s(at),
262
+ 'owner' => owner_fei,
263
+ 'msg' => msg
264
+ }
265
+ end
266
+
190
267
  def get_engine_variables
191
268
 
192
269
  get('variables', 'variables') || {