ruote 2.1.9 → 2.1.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. data/CHANGELOG.txt +32 -0
  2. data/CREDITS.txt +3 -0
  3. data/Rakefile +4 -4
  4. data/TODO.txt +55 -11
  5. data/examples/barley.rb +2 -1
  6. data/examples/flickr_report.rb +5 -6
  7. data/examples/web_first_page.rb +11 -0
  8. data/lib/ruote/context.rb +36 -13
  9. data/lib/ruote/engine.rb +88 -56
  10. data/lib/ruote/engine/process_error.rb +13 -0
  11. data/lib/ruote/engine/process_status.rb +33 -1
  12. data/lib/ruote/error_handler.rb +122 -0
  13. data/lib/ruote/evt/tracker.rb +27 -10
  14. data/lib/ruote/exp/fe_apply.rb +69 -0
  15. data/lib/ruote/exp/fe_participant.rb +33 -5
  16. data/lib/ruote/exp/flowexpression.rb +37 -5
  17. data/lib/ruote/exp/ro_persist.rb +8 -4
  18. data/lib/ruote/exp/ro_variables.rb +2 -2
  19. data/lib/ruote/fei.rb +59 -7
  20. data/lib/ruote/log/storage_history.rb +2 -0
  21. data/lib/ruote/log/test_logger.rb +28 -19
  22. data/lib/ruote/log/wait_logger.rb +4 -2
  23. data/lib/ruote/parser.rb +2 -1
  24. data/lib/ruote/part/dispatch_pool.rb +10 -10
  25. data/lib/ruote/part/engine_participant.rb +2 -2
  26. data/lib/ruote/part/local_participant.rb +99 -7
  27. data/lib/ruote/part/participant_list.rb +18 -7
  28. data/lib/ruote/part/storage_participant.rb +9 -6
  29. data/lib/ruote/receiver/base.rb +109 -10
  30. data/lib/ruote/storage/base.rb +118 -41
  31. data/lib/ruote/storage/fs_storage.rb +1 -0
  32. data/lib/ruote/storage/hash_storage.rb +2 -1
  33. data/lib/ruote/util/lookup.rb +22 -2
  34. data/lib/ruote/util/misc.rb +5 -5
  35. data/lib/ruote/version.rb +1 -1
  36. data/lib/ruote/worker.rb +50 -63
  37. data/lib/ruote/workitem.rb +64 -0
  38. data/ruote.gemspec +17 -12
  39. data/test/functional/base.rb +3 -1
  40. data/test/functional/concurrent_base.rb +35 -29
  41. data/test/functional/crunner.sh +19 -0
  42. data/test/functional/ct_0_concurrence.rb +17 -30
  43. data/test/functional/ct_1_iterator.rb +20 -17
  44. data/test/functional/ct_2_cancel.rb +32 -25
  45. data/test/functional/eft_12_listen.rb +2 -1
  46. data/test/functional/eft_23_apply.rb +23 -0
  47. data/test/functional/eft_3_participant.rb +27 -0
  48. data/test/functional/ft_11_recursion.rb +1 -1
  49. data/test/functional/ft_13_variables.rb +22 -0
  50. data/test/functional/ft_14_re_apply.rb +3 -0
  51. data/test/functional/ft_15_timeout.rb +1 -0
  52. data/test/functional/ft_20_storage_participant.rb +20 -2
  53. data/test/functional/ft_21_forget.rb +30 -0
  54. data/test/functional/ft_22_process_definitions.rb +2 -1
  55. data/test/functional/ft_24_block_participants.rb +1 -1
  56. data/test/functional/ft_25_receiver.rb +83 -1
  57. data/test/functional/ft_26_participant_timeout.rb +1 -1
  58. data/test/functional/ft_2_errors.rb +2 -5
  59. data/test/functional/ft_30_smtp_participant.rb +47 -45
  60. data/test/functional/ft_36_storage_history.rb +4 -4
  61. data/test/functional/ft_37_engine_participant.rb +14 -10
  62. data/test/functional/ft_38_participant_more.rb +178 -0
  63. data/test/functional/ft_39_wait_for.rb +100 -0
  64. data/test/functional/ft_40_participant_on_reply.rb +87 -0
  65. data/test/functional/ft_41_participants.rb +65 -0
  66. data/test/functional/ft_42_storage_copy.rb +67 -0
  67. data/test/functional/ft_5_on_error.rb +103 -0
  68. data/test/functional/ft_9_subprocesses.rb +2 -1
  69. data/test/functional/storage_helper.rb +5 -1
  70. data/test/functional/test.rb +4 -1
  71. data/test/functional/vertical.rb +46 -0
  72. data/test/unit/storage.rb +17 -1
  73. data/test/unit/storages.rb +27 -7
  74. data/test/unit/ut_11_lookup.rb +36 -0
  75. data/test/unit/ut_16_parser.rb +43 -0
  76. data/test/unit/ut_1_fei.rb +28 -1
  77. data/test/unit/ut_7_workitem.rb +23 -0
  78. metadata +67 -105
  79. data/lib/ruote/log/fs_history.rb +0 -182
  80. data/test/functional/ft_32_fs_history.rb +0 -188
  81. data/test/mpc_test.rb +0 -29
@@ -93,8 +93,9 @@ module Ruote
93
93
 
94
94
  else
95
95
 
96
- return Ruote::StorageParticipant.new(@context) \
97
- if entry.last.first == 'Ruote::StorageParticipant'
96
+ if entry.last.first == 'Ruote::StorageParticipant'
97
+ return Ruote::StorageParticipant.new(@context)
98
+ end
98
99
 
99
100
  nil
100
101
  end
@@ -110,8 +111,9 @@ module Ruote
110
111
  code = nil
111
112
  entry = nil
112
113
 
113
- name_or_participant = name_or_participant.to_s \
114
- if name_or_participant.is_a?(Symbol)
114
+ if name_or_participant.is_a?(Symbol)
115
+ name_or_participant = name_or_participant.to_s
116
+ end
115
117
 
116
118
  if name_or_participant.is_a?(String)
117
119
 
@@ -160,12 +162,12 @@ module Ruote
160
162
  end
161
163
  end
162
164
 
163
- def lookup (participant_name)
165
+ def lookup (participant_name, opts={})
164
166
 
165
167
  pi = lookup_info(participant_name)
166
168
 
167
169
  return nil unless pi
168
- return pi unless pi.is_a?(Array)
170
+ return opts[:on_reply] ? nil : pi unless pi.is_a?(Array)
169
171
 
170
172
  class_name, options = pi
171
173
 
@@ -173,8 +175,17 @@ module Ruote
173
175
  require(rp)
174
176
  end
175
177
 
176
- pa = Ruote.constantize(class_name).new(options)
178
+ pa_class = Ruote.constantize(class_name)
179
+ pa_m = pa_class.instance_methods
180
+
181
+ return nil if opts[:on_reply] && ! (
182
+ pa_m.include?(:on_reply) || pa_m.include?('on_reply'))
177
183
 
184
+ pa = if pa_class.instance_method(:initialize).arity > 0
185
+ pa_class.new(options)
186
+ else
187
+ pa_class.new
188
+ end
178
189
  pa.context = @context if pa.respond_to?(:context=)
179
190
 
180
191
  pa
@@ -96,7 +96,7 @@ module Ruote
96
96
  #
97
97
  def cancel (fei, flavour)
98
98
 
99
- doc = fetch(fei.to_h)
99
+ doc = fetch(fei)
100
100
 
101
101
  r = @context.storage.delete(doc)
102
102
 
@@ -112,16 +112,18 @@ module Ruote
112
112
 
113
113
  def fetch (fei)
114
114
 
115
- fei = fei.to_h if fei.respond_to?(:to_h)
115
+ hfei = Ruote::FlowExpressionId.extract_h(fei)
116
116
 
117
- @context.storage.get('workitems', to_id(fei))
117
+ @context.storage.get('workitems', to_id(hfei))
118
118
  end
119
119
 
120
120
  # Removes the workitem from the in-memory hash and replies to the engine.
121
121
  #
122
122
  def reply (workitem)
123
123
 
124
- doc = fetch(workitem.fei.to_h)
124
+ # TODO: change method name (receiver mess cleanup)
125
+
126
+ doc = fetch(Ruote::FlowExpressionId.extract_h(workitem))
125
127
 
126
128
  r = @context.storage.delete(doc)
127
129
 
@@ -237,8 +239,9 @@ module Ruote
237
239
 
238
240
  cr = criteria.inject({}) { |h, (k, v)| h[k.to_s] = v; h }
239
241
 
240
- return @context.storage.query_workitems(cr) \
241
- if @context.storage.respond_to?(:query_workitems)
242
+ return @context.storage.query_workitems(cr).collect { |h|
243
+ Ruote::Workitem.new(h)
244
+ } if @context.storage.respond_to?(:query_workitems)
242
245
 
243
246
  offset = cr.delete('offset')
244
247
  limit = cr.delete('limit')
@@ -29,20 +29,19 @@ module Ruote
29
29
  # The core methods for the Receiver class (sometimes a Mixin is easier
30
30
  # to integrate).
31
31
  #
32
- # (The engine itself includes this mixin)
32
+ # (The engine itself includes this mixin, the LocalParticipant module
33
+ # includes it as well).
33
34
  #
34
35
  module ReceiverMixin
35
36
 
36
- def receive (item)
37
-
38
- reply(item)
39
- end
40
-
41
- def reply (workitem)
37
+ # This method pipes back a workitem into the engine, letting it resume
38
+ # in its flow, hopefully.
39
+ #
40
+ def receive (workitem)
42
41
 
43
42
  workitem = workitem.to_h if workitem.respond_to?(:to_h)
44
43
 
45
- @storage.put_msg(
44
+ @context.storage.put_msg(
46
45
  'receive',
47
46
  'fei' => workitem['fei'],
48
47
  'workitem' => workitem,
@@ -50,10 +49,108 @@ module Ruote
50
49
  'receiver' => sign)
51
50
  end
52
51
 
52
+ # Given a process definitions and optional initial fields and variables,
53
+ # launches a new process instance.
54
+ #
55
+ # This method is mostly used from the Ruote::Engine class (which includes
56
+ # this mixin).
57
+ #
58
+ def launch (process_definition, fields={}, variables={})
59
+
60
+ wfid = @context.wfidgen.generate
61
+
62
+ @context.storage.put_msg(
63
+ 'launch',
64
+ 'wfid' => wfid,
65
+ 'tree' => @context.parser.parse(process_definition),
66
+ 'workitem' => { 'fields' => fields },
67
+ 'variables' => variables)
68
+
69
+ wfid
70
+ end
71
+
72
+ # Wraps a call to receive(workitem)
73
+ #
74
+ # Not aliasing so that if someone changes the receive implementation,
75
+ # reply is affected as well.
76
+ #
77
+ def reply (workitem)
78
+
79
+ receive (workitem)
80
+ end
81
+
82
+ # Wraps a call to receive(workitem)
83
+ #
84
+ # Not aliasing so that if someone changes the receive implementation,
85
+ # reply_to_engine is affected as well.
86
+ #
87
+ def reply_to_engine (workitem)
88
+
89
+ receive (workitem)
90
+ end
91
+
92
+ # A receiver signs a workitem when it comes back.
93
+ #
94
+ # Not used much as of now.
95
+ #
53
96
  def sign
54
97
 
55
98
  self.class.to_s
56
99
  end
100
+
101
+ protected
102
+
103
+ # Convenience method, fetches the flow expression (ParticipantExpression)
104
+ # that emitted that workitem.
105
+ #
106
+ def fetch_flow_expression (workitem)
107
+
108
+ Ruote::Exp::FlowExpression.fetch(@context, workitem.fei.to_h)
109
+ end
110
+
111
+ # Stashes values in the participant expression (in the storage).
112
+ #
113
+ # put(workitem.fei, 'key' => 'value', 'colour' => 'blue')
114
+ #
115
+ # Remember that keys/values must be serializable in JSON.
116
+ #
117
+ # put & get are useful for a participant that needs to communicate
118
+ # between its consume and its cancel.
119
+ #
120
+ # See the thread at
121
+ # http://groups.google.com/group/openwferu-users/t/2e6a95708c10847b for the
122
+ # justification.
123
+ #
124
+ def put (fei, hash)
125
+
126
+ fexp = Ruote::Exp::FlowExpression.fetch(@context, fei.to_h)
127
+
128
+ (fexp.h['stash'] ||= {}).merge!(hash)
129
+
130
+ fexp.persist_or_raise
131
+ end
132
+
133
+ # Fetches back a stashed value.
134
+ #
135
+ # get(fei, 'colour')
136
+ # # => 'blue'
137
+ #
138
+ # To return the whole stash
139
+ #
140
+ # get(fei)
141
+ # # => { 'colour' => 'blue' }
142
+ #
143
+ # put & get are useful for a participant that needs to communicate
144
+ # between its consume and its cancel.
145
+ #
146
+ def get (fei, key=nil)
147
+
148
+ fexp = Ruote::Exp::FlowExpression.fetch(@context, fei.to_h)
149
+
150
+ stash = fexp.h['stash'] rescue {}
151
+
152
+ key ? stash[key] : stash
153
+ end
57
154
  end
58
155
 
59
156
  #
@@ -63,9 +160,11 @@ module Ruote
63
160
  class Receiver
64
161
  include ReceiverMixin
65
162
 
66
- def initialize (storage, options={})
163
+ # Accepts context, worker, engine or storage as first argument.
164
+ #
165
+ def initialize (cwes, options={})
67
166
 
68
- @storage = storage
167
+ @context = cwes.context
69
168
  @options = options
70
169
  end
71
170
  end
@@ -32,6 +32,24 @@ module Ruote
32
32
  #
33
33
  module StorageBase
34
34
 
35
+ def context
36
+
37
+ @context ||= Ruote::Context.new(self)
38
+ end
39
+
40
+ def context= (c)
41
+
42
+ @context = c
43
+ end
44
+
45
+ # Attempts to delete a document, returns true if the deletion
46
+ # succeeded. This is used with msgs to reserve work on them.
47
+ #
48
+ def reserve (doc)
49
+
50
+ delete(doc).nil?
51
+ end
52
+
35
53
  #--
36
54
  # configurations
37
55
  #++
@@ -47,28 +65,16 @@ module Ruote
47
65
 
48
66
  def put_msg (action, options)
49
67
 
50
- # merge! is way faster than merge (no object creation probably)
51
-
52
- @counter ||= 0
53
-
54
- t = Time.now.utc
55
- ts = "#{t.strftime('%Y-%m-%d')}!#{t.to_i}.#{'%06d' % t.usec}"
56
- _id = "#{$$}!#{Thread.current.object_id}!#{ts}!#{'%03d' % @counter}"
57
-
58
- @counter = (@counter + 1) % 1000
59
- # some platforms (windows) have shallow usecs, so adding that counter...
60
-
61
- msg = options.merge!('type' => 'msgs', '_id' => _id, 'action' => action)
62
-
63
- msg.delete('_rev')
64
- # in case of message replay
68
+ msg = prepare_msg_doc(action, options)
65
69
 
66
70
  put(msg)
71
+
67
72
  #put(msg, :update_rev => true)
68
- #(@local_msgs ||= []) << options
73
+ #(@local_msgs ||= []) << Ruote.fulldup(msg)
69
74
  end
70
75
 
71
76
  #def get_local_msgs
77
+ # p @local_msgs
72
78
  # if @local_msgs
73
79
  # r = @local_msgs
74
80
  # @local_msgs = nil
@@ -87,6 +93,11 @@ module Ruote
87
93
  }
88
94
  end
89
95
 
96
+ def empty? (type)
97
+
98
+ (get_many(type) == [])
99
+ end
100
+
90
101
  #--
91
102
  # expressions
92
103
  #++
@@ -116,6 +127,9 @@ module Ruote
116
127
 
117
128
  def get_schedules (delta, now)
118
129
 
130
+ # TODO : bring that 'optimization' back in,
131
+ # maybe every minute, if min != last_min ...
132
+
119
133
  #if delta < 1.0
120
134
  # at = now.strftime('%Y%m%d%H%M%S')
121
135
  # get_many('schedules', /-#{at}$/)
@@ -133,33 +147,12 @@ module Ruote
133
147
 
134
148
  def put_schedule (flavour, owner_fei, s, msg)
135
149
 
136
- at = if s.is_a?(Time) # at or every
137
- s
138
- elsif Ruote.is_cron_string(s) # cron
139
- Rufus::CronLine.new(s).next_time(Time.now + 1)
140
- else # at or every
141
- Ruote.s_to_at(s)
142
- end
143
- at = at.utc
144
-
145
- if at <= Time.now.utc && flavour == 'at'
146
- put_msg(msg.delete('action'), msg)
147
- return
150
+ if doc = prepare_schedule_doc(flavour, owner_fei, s, msg)
151
+ put(doc)
152
+ return doc['_id']
148
153
  end
149
154
 
150
- sat = at.strftime('%Y%m%d%H%M%S')
151
- i = "#{flavour}-#{Ruote.to_storage_id(owner_fei)}-#{sat}"
152
-
153
- put(
154
- '_id' => i,
155
- 'type' => 'schedules',
156
- 'flavour' => flavour,
157
- 'original' => s,
158
- 'at' => Ruote.time_to_utc_s(at),
159
- 'owner' => owner_fei,
160
- 'msg' => msg)
161
-
162
- i
155
+ nil
163
156
  end
164
157
 
165
158
  def delete_schedule (schedule_id)
@@ -185,8 +178,92 @@ module Ruote
185
178
  put_engine_variable(k, v) unless put(vars).nil?
186
179
  end
187
180
 
181
+ #--
182
+ # migrations
183
+ #++
184
+
185
+ # Copies the content of this storage into a target storage.
186
+ #
187
+ # Of course, the target storage may be a different implementation.
188
+ #
189
+ def copy_to (target, opts={})
190
+
191
+ counter = 0
192
+
193
+ %w[
194
+ configurations errors expressions msgs schedules variables workitems
195
+ ].each do |type|
196
+
197
+ get_many(type).each do |item|
198
+
199
+ item.delete('_rev')
200
+ target.put(item)
201
+
202
+ counter += 1
203
+ puts(" #{type}/#{item['_id']}") if opts[:verbose]
204
+ end
205
+ end
206
+
207
+ counter
208
+ end
209
+
188
210
  protected
189
211
 
212
+ # Used by put_msg
213
+ #
214
+ def prepare_msg_doc (action, options)
215
+
216
+ # merge! is way faster than merge (no object creation probably)
217
+
218
+ @counter ||= 0
219
+
220
+ t = Time.now.utc
221
+ ts = "#{t.strftime('%Y-%m-%d')}!#{t.to_i}.#{'%06d' % t.usec}"
222
+ _id = "#{$$}!#{Thread.current.object_id}!#{ts}!#{'%03d' % @counter}"
223
+
224
+ @counter = (@counter + 1) % 1000
225
+ # some platforms (windows) have shallow usecs, so adding that counter...
226
+
227
+ msg = options.merge!('type' => 'msgs', '_id' => _id, 'action' => action)
228
+
229
+ msg.delete('_rev')
230
+ # in case of message replay
231
+
232
+ msg
233
+ end
234
+
235
+ # Used by put_schedule
236
+ #
237
+ def prepare_schedule_doc (flavour, owner_fei, s, msg)
238
+
239
+ at = if s.is_a?(Time) # at or every
240
+ s
241
+ elsif Ruote.is_cron_string(s) # cron
242
+ Rufus::CronLine.new(s).next_time(Time.now + 1)
243
+ else # at or every
244
+ Ruote.s_to_at(s)
245
+ end
246
+ at = at.utc
247
+
248
+ if at <= Time.now.utc && flavour == 'at'
249
+ put_msg(msg.delete('action'), msg)
250
+ return false
251
+ end
252
+
253
+ sat = at.strftime('%Y%m%d%H%M%S')
254
+ i = "#{flavour}-#{Ruote.to_storage_id(owner_fei)}-#{sat}"
255
+
256
+ {
257
+ '_id' => i,
258
+ 'type' => 'schedules',
259
+ 'flavour' => flavour,
260
+ 'original' => s,
261
+ 'at' => Ruote.time_to_utc_s(at),
262
+ 'owner' => owner_fei,
263
+ 'msg' => msg
264
+ }
265
+ end
266
+
190
267
  def get_engine_variables
191
268
 
192
269
  get('variables', 'variables') || {