cpee 2.1.136 → 2.1.139

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -532,6 +532,7 @@ module CPEE
532
532
  id = @a[2]
533
533
  opts = @a[3]
534
534
  if val = CPEE::Persistence::extract_item(id,opts,@r.join('/'))
535
+ mime = 'text/plain' if mime == 'text/xml' && (XML::Smart::string(val) rescue nil).nil? # a description can be any model type, e.g. text
535
536
  Riddl::Parameter::Complex.new(item,mime,val)
536
537
  else
537
538
  @status = 404
@@ -826,6 +827,23 @@ module CPEE
826
827
  end
827
828
  end #}}}
828
829
 
830
+ def self::backtransform_async(id,opts,dslx,tdslx,tdslxtype) #{{{
831
+ change_uuid = Digest::SHA1.hexdigest(dslx)
832
+ Thread.new do
833
+ attrs = CPEE::Persistence::extract_list(id,opts,'attributes').to_h
834
+ begin
835
+ description = PutDescription::backtransform(dslx,dslx,tdslx,tdslxtype)
836
+ unless description.nil?
837
+ CPEE::Message::send(:event,'description/sync',File.join(opts[:url],'/'),id,attrs['uuid'],attrs['info'],{ :change_uuid => change_uuid, :description => description },opts[:redis])
838
+ end
839
+ rescue => e # description stays as it is
840
+ puts e.message
841
+ puts e.backtrace
842
+ CPEE::Message::send(:event,'description/error',File.join(opts[:url],'/'),id,attrs['uuid'],attrs['info'],{ :message => 'backtransformation failed: ' + e.message, :change_uuid => change_uuid, :attributes => attrs },opts[:redis])
843
+ end
844
+ end
845
+ end #}}}
846
+
829
847
  def self::set(id,opts,xml,exposition=[],copy=false)
830
848
  dslx, dsl, de, ep = if copy
831
849
  PutDescription::transform(
@@ -854,14 +872,13 @@ module CPEE
854
872
  opts
855
873
  )
856
874
  end
857
- description = if copy
875
+ tdslx = CPEE::Persistence::extract_item(id,opts,'transformation/dslx')
876
+ tdslxtype = CPEE::Persistence::extract_item(id,opts,'transformation/dslx/@type')
877
+ description = if copy && tdslxtype == 'rest'
878
+ nil # slow (e.g. bpmn layout), done after storing, see backtransform_async
879
+ elsif copy
858
880
  begin
859
- PutDescription::backtransform(
860
- xml,
861
- dslx,
862
- CPEE::Persistence::extract_item(id,opts,'transformation/dslx'),
863
- CPEE::Persistence::extract_item(id,opts,'transformation/dslx/@type')
864
- )
881
+ PutDescription::backtransform(xml,dslx,tdslx,tdslxtype)
865
882
  rescue => e # dslx is stored anyway, description stays as it is
866
883
  puts e.message
867
884
  puts e.backtrace
@@ -878,12 +895,14 @@ module CPEE
878
895
  :description => description,
879
896
  :dslx => dslx,
880
897
  :change_uuid => change_uuid,
898
+ :change_pending => copy && tdslxtype == 'rest', # description follows with description/sync
881
899
  :dsl => dsl,
882
900
  :dataelements => CPEE::Persistence::extract_list(id,opts,'dataelements').to_h,
883
901
  :documents => dos,
884
902
  :endpoints => CPEE::Persistence::extract_list(id,opts,'endpoints').to_h,
885
903
  :attributes => attrs
886
904
  )
905
+ PutDescription::backtransform_async(id,opts,dslx,tdslx,tdslxtype) if copy && tdslxtype == 'rest'
887
906
  PatchItems::set_hash('dataelements',id,opts,de) unless de.empty?
888
907
  PatchItems::set_hash('endpoints',id,opts,ep) unless ep.empty?
889
908
  exposition.each do |exp|
@@ -962,6 +981,8 @@ module CPEE
962
981
  tdslxtype = (t = doc.find('string(/p:transformation/p:dslx/@type)')).empty? ? 'copy' : t # optional, missing means copy
963
982
  changed = tdslx != (CPEE::Persistence::extract_item(id,opts,'transformation/dslx') || '') ||
964
983
  tdslxtype != (CPEE::Persistence::extract_item(id,opts,'transformation/dslx/@type') || 'copy')
984
+ dslx = CPEE::Persistence::extract_item(id,opts,'dslx')
985
+ dslx = nil if dslx&.empty?
965
986
  CPEE::Persistence::set_item(id,opts,'transformation',
966
987
  :description => doc.find('string(/p:transformation/p:description)'),
967
988
  :description_type => doc.find('string(/p:transformation/p:description/@type)'),
@@ -970,7 +991,9 @@ module CPEE
970
991
  :endpoints =>doc.find('string(/p:transformation/p:endpoints)'),
971
992
  :endpoints_type => doc.find('string(/p:transformation/p:endpoints/@type)'),
972
993
  :dslx => tdslx,
973
- :dslx_type => tdslxtype
994
+ :dslx_type => tdslxtype,
995
+ :change_uuid => dslx ? Digest::SHA1.hexdigest(dslx) : nil,
996
+ :change_pending => backtrans && changed && tdslxtype == 'rest' && !dslx.nil? # description follows with description/sync
974
997
  )
975
998
  PutTransformation::backtransform(id,opts,tdslx,tdslxtype) if backtrans && changed
976
999
  end
@@ -979,6 +1002,7 @@ module CPEE
979
1002
  # directly, as redis is only updated asynchronously by persist.rb
980
1003
  dslx = CPEE::Persistence::extract_item(id,opts,'dslx')
981
1004
  return if dslx.nil? || dslx.empty?
1005
+ return PutDescription::backtransform_async(id,opts,dslx,tdslx,tdslxtype) if tdslxtype == 'rest'
982
1006
  description = begin
983
1007
  PutDescription::backtransform(dslx,dslx,tdslx,tdslxtype)
984
1008
  rescue => e # description stays as it is
@@ -0,0 +1,110 @@
1
+ # This file is part of CPEE.
2
+ #
3
+ # CPEE is free software: you can redistribute it and/or modify it under the terms
4
+ # of the GNU Lesser General Public License as published by the Free Software Foundation,
5
+ # either version 3 of the License, or (at your option) any later version.
6
+ #
7
+ # CPEE is distributed in the hope that it will be useful, but WITHOUT ANY
8
+ # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
9
+ # PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
10
+ #
11
+ # You should have received a copy of the GNU Lesser General Public License along with
12
+ # CPEE (file LICENSE in the main directory). If not, see
13
+ # <http://www.gnu.org/licenses/>.
14
+
15
+ require 'ostruct'
16
+
17
+ module CPEE
18
+
19
+ module JSONPointer
20
+
21
+ def self::parse(pointer) #{{{
22
+ raise ArgumentError, "invalid JSON pointer #{pointer.inspect}" unless pointer.is_a?(String) && pointer.start_with?('/')
23
+ pointer[1..-1].split('/',-1).map { |s| s.gsub('~1','/').gsub('~0','~') }
24
+ end #}}}
25
+
26
+ def self::symbolize(value) #{{{
27
+ case value
28
+ when Hash; value.each_with_object({}) { |(k,v),h| h[k.to_sym] = symbolize(v) }
29
+ when Array; value.map { |v| symbolize(v) }
30
+ else value
31
+ end
32
+ end #}}}
33
+
34
+ def self::container?(node) #{{{
35
+ node.is_a?(Hash) || node.is_a?(Array) || node.is_a?(OpenStruct)
36
+ end #}}}
37
+
38
+ def self::array_index(node, seg) #{{{
39
+ return node.length if seg == '-'
40
+ raise ArgumentError, "invalid array index #{seg.inspect}" unless seg =~ /\A(0|[1-9]\d*)\z/
41
+ i = seg.to_i
42
+ raise ArgumentError, "array index #{i} out of range" if i > node.length
43
+ i
44
+ end #}}}
45
+
46
+ def self::get(node, seg) #{{{
47
+ case node
48
+ when Hash
49
+ node.key?(seg.to_sym) ? node[seg.to_sym] : node[seg]
50
+ when Array
51
+ node[array_index(node, seg)]
52
+ when OpenStruct
53
+ node[seg.to_sym]
54
+ end
55
+ end #}}}
56
+
57
+ def self::put(node, seg, value) #{{{
58
+ case node
59
+ when Hash
60
+ node[node.key?(seg) && !node.key?(seg.to_sym) ? seg : seg.to_sym] = value
61
+ when Array
62
+ node[array_index(node, seg)] = value
63
+ when OpenStruct
64
+ node[seg.to_sym] = value
65
+ end
66
+ end #}}}
67
+
68
+ def self::argument(value) #{{{
69
+ return value unless value.is_a?(Hash)
70
+ (defined?(::ParaStruct) ? ::ParaStruct : OpenStruct).new(:name => value[:name].to_s.to_sym, :value => value[:value])
71
+ end #}}}
72
+
73
+ # /arguments/<name>/... -> /arguments/<index>/value/... (argument is created if missing)
74
+ def self::argument_by_name(root, segs) #{{{
75
+ return segs unless segs.length >= 2 && segs[0] == 'arguments' && segs[1] !~ /\A(\d+|-)\z/
76
+ args = (get(root, 'arguments') || []).dup
77
+ put(root, 'arguments', args)
78
+ idx = args.index { |a| a.respond_to?(:name) && a.name.to_s == segs[1] }
79
+ if idx.nil?
80
+ args << argument(:name => segs[1], :value => nil)
81
+ idx = args.length - 1
82
+ end
83
+ ['arguments', idx.to_s, 'value'] + segs[2..-1]
84
+ end #}}}
85
+
86
+ # containers on the path are copied before writing, only root is modified in place
87
+ def self::set(root, pointer, value) #{{{
88
+ raise ArgumentError, 'root is not a Hash' unless root.is_a?(Hash)
89
+ segs = argument_by_name(root, parse(pointer))
90
+ value = symbolize(value)
91
+ value = value.to_s.sub(/^:/,'').to_sym if segs == ['method'] && !value.nil?
92
+ value = argument(value) if segs.length == 2 && segs[0] == 'arguments'
93
+ value = value.map { |a| argument(a) } if segs == ['arguments'] && value.is_a?(Array)
94
+
95
+ node = root
96
+ segs[0..-2].each do |seg|
97
+ child = get(node, seg)
98
+ child = {} if child.nil?
99
+ raise ArgumentError, "#{pointer}: #{seg.inspect} is not a container" unless container?(child)
100
+ child = child.dup
101
+ put(node, seg, child)
102
+ node = child
103
+ end
104
+ put(node, segs.last, value)
105
+ root
106
+ end #}}}
107
+
108
+ end
109
+
110
+ end
data/lib/cpee/message.rb CHANGED
@@ -12,9 +12,27 @@
12
12
  # CPEE (file LICENSE in the main directory). If not, see
13
13
  # <http://www.gnu.org/licenses/>.
14
14
 
15
+ require 'digest/sha1'
16
+ require_relative 'redis'
17
+
15
18
  module CPEE
16
19
 
17
20
  module Message
21
+ # INCR the per-instance sequence and PUBLISH in one round trip. The seq is
22
+ # appended to the prefix ("instance,uuid,seq {json}") so that routing
23
+ # workers can read it without parsing the JSON, and added to the JSON for
24
+ # consumers that only see the body. The payload is always a JSON object.
25
+ # Only count while the instance exists (KEYS[2] is its state), so late
26
+ # messages for a deleted instance do not recreate the counter (seq 0).
27
+ # INCR keeps the TTL that DeleteInstance set while the key expires.
28
+ SEQ_PUBLISH = <<~LUA.freeze
29
+ local s = 0
30
+ if redis.call('EXISTS', KEYS[2]) == 1 then s = redis.call('INCR', KEYS[1]) end
31
+ redis.call('PUBLISH', ARGV[1], ARGV[2] .. ',' .. s .. ' ' .. string.sub(ARGV[3], 1, -2) .. ',"seq":' .. s .. '}')
32
+ return s
33
+ LUA
34
+ SEQ_PUBLISH_SHA = Digest::SHA1.hexdigest(SEQ_PUBLISH).freeze
35
+
18
36
  @@mutex = Mutex.new
19
37
  @@who = 'cpee'
20
38
  @@type = 'instance'
@@ -35,7 +53,7 @@ module CPEE
35
53
  end #}}}
36
54
 
37
55
  def self::set_workers(workers)
38
- @@tworkers = (workers < 1 && workers > 99 ? 1 : workers).freeze
56
+ @@tworkers = (workers < 1 || workers > 99 ? 1 : workers).freeze
39
57
  @@last = -1
40
58
  end
41
59
 
@@ -77,7 +95,9 @@ module CPEE
77
95
  payload["#{@@type}-uuid"] = instance_uuid if instance_uuid
78
96
  payload["#{@@type}-name"] = instance_name if instance_name
79
97
 
80
- backend.publish("#{type}:#{target}:#{event}", "#{instance},#{instance_uuid} #{JSON::generate(payload)}")
98
+ keys = ["#{@@type}:#{instance}/seq", "#{@@type}:#{instance}/state"]
99
+ argv = ["#{type}:#{target}:#{event}", "#{instance},#{instance_uuid}", JSON::generate(payload)]
100
+ CPEE::redis_script(backend, SEQ_PUBLISH_SHA, SEQ_PUBLISH, keys, argv)
81
101
  end
82
102
 
83
103
  def self::send_url(type, event, cpee, content={}, backend)
@@ -183,6 +183,7 @@ module CPEE
183
183
  res += Persistence::keys_extract_name(opts,id,'handlers')
184
184
  hnd.each do |h|
185
185
  res << File.join(h,'url')
186
+ res << File.join(h,'seq')
186
187
  res += Persistence::keys_extract_set_raw(opts,h)
187
188
  end
188
189
  cbs = Persistence::keys_extract_set(opts,id,'callbacks')
@@ -197,12 +198,15 @@ module CPEE
197
198
  end
198
199
  res += Persistence::keys_extract_name(opts,id,'dsl')
199
200
  res += Persistence::keys_extract_name(opts,id,'dslx')
201
+ res += Persistence::keys_extract_name(opts,id,'dslx','@change_uuid')
200
202
  res += Persistence::keys_extract_name(opts,id,'status','message')
201
203
  res += Persistence::keys_extract_name(opts,id,'status','id')
202
204
  res += Persistence::keys_extract_name(opts,id,'executionhandler')
203
205
  res += Persistence::keys_extract_name(opts,id,'description')
206
+ res += Persistence::keys_extract_name(opts,id,'description','@change_uuid')
204
207
  res += Persistence::keys_extract_name(opts,id,'state')
205
208
  res += Persistence::keys_extract_name(opts,id,'state','@changed')
209
+ res += Persistence::keys_extract_name(opts,id,'seq')
206
210
  res += Persistence::keys_extract_name(opts,id,'transformation','endpoints')
207
211
  res += Persistence::keys_extract_name(opts,id,'transformation','endpoints','@type')
208
212
  res += Persistence::keys_extract_name(opts,id,'transformation','description')
data/lib/cpee/redis.rb CHANGED
@@ -16,6 +16,14 @@ require 'redis'
16
16
  require 'uri'
17
17
 
18
18
  module CPEE
19
+ # run a lua script by its sha (cached in redis), send the full script only if redis does not know it (yet)
20
+ def self::redis_script(backend,sha,script,keys,argv)
21
+ backend.evalsha(sha, keys: keys, argv: argv)
22
+ rescue Redis::CommandError => e
23
+ raise unless e.message.start_with?('NOSCRIPT')
24
+ backend.eval(script, keys: keys, argv: argv)
25
+ end
26
+
19
27
  def self::redis_connect(opts,name=nil)
20
28
  if opts[:redis_cmd].nil?
21
29
  begin
@@ -7,7 +7,17 @@
7
7
 
8
8
  <start>
9
9
  <element name="description">
10
- <ref name="dslx_attributes"/>
10
+ <optional> <!-- layout direction of the theme, e.g. for the bpmn export -->
11
+ <attribute name="dir">
12
+ <choice>
13
+ <value>td</value>
14
+ <value>lr</value>
15
+ </choice>
16
+ </attribute>
17
+ </optional>
18
+ <zeroOrMore>
19
+ <attribute><anyName><except><name ns="">dir</name></except></anyName></attribute>
20
+ </zeroOrMore>
11
21
  <ref name="dslx_block"/>
12
22
  </element>
13
23
  </start>
@@ -17,6 +17,7 @@ require 'mimemagic'
17
17
  require 'base64'
18
18
  require 'securerandom'
19
19
  require 'cpee-eval-ruby/translation'
20
+ require 'cpee/json_pointer'
20
21
 
21
22
  class ConnectionWrapper < WEEL::ConnectionWrapperBase
22
23
  def self::loop_guard(arguments,id,count) # {{{
@@ -288,7 +289,15 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
288
289
  @controller.vote("activity/syncing_after", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label)
289
290
  end # }}}
290
291
  def vote_sync_before(parameters=nil) # {{{
291
- @controller.vote("activity/syncing_before", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label, :parameters => parameters)
292
+ @label = parameters[:label] if parameters.is_a?(Hash) && parameters[:label]
293
+ @controller.vote("activity/syncing_before", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label, :parameters => parameters) do |changes|
294
+ next unless parameters.is_a?(Hash)
295
+ changes.each do |pointer,value|
296
+ CPEE::JSONPointer::set(parameters, pointer, value)
297
+ rescue => e
298
+ puts "vote change #{pointer} ignored: #{e.message}"
299
+ end
300
+ end
292
301
  end # }}}
293
302
 
294
303
  def callback(result=nil,options={}, dataelements) #{{{
@@ -388,13 +397,13 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
388
397
  end
389
398
  __params
390
399
  end #}}}
391
- def test_condition(__eid,__dataelements,__endpoints,__local,__additional,__code,__args={}) #{{{
400
+ def test_condition(__eid,__dataelements,__endpoints,__local,__additional,__code,__args={},__props={}) #{{{
392
401
  __struct = WEEL::ReadStructure.new(__dataelements,__endpoints,__local,__additional).instance_eval(__code,'Condition',1)
393
402
 
394
403
  __struct = 'false' unless __struct
395
404
  __struct = (__struct == 'false' || __struct == 'null' || __struct == 'nil' || __struct == false ? false : true)
396
405
 
397
- @controller.notify("gateway/decide", :eid => __eid, :instance_uuid => @controller.uuid, :code => __code, :condition => __struct)
406
+ @controller.notify("gateway/decide", :eid => __eid, :instance_uuid => @controller.uuid, :code => __code, :condition => __struct, **__props)
398
407
  @controller.notify("gateway/annotation", :eid => __eid, :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :annotations => {})
399
408
  __struct
400
409
  end #}}}
@@ -17,6 +17,7 @@ require 'mimemagic'
17
17
  require 'base64'
18
18
  require 'securerandom'
19
19
  require 'cpee-eval-ruby/translation'
20
+ require 'cpee/json_pointer'
20
21
 
21
22
  class ConnectionWrapper < WEEL::ConnectionWrapperBase
22
23
  def self::loop_guard(arguments,id,count) # {{{
@@ -288,7 +289,15 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
288
289
  @controller.vote("activity/syncing_after", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label)
289
290
  end # }}}
290
291
  def vote_sync_before(parameters=nil) # {{{
291
- @controller.vote("activity/syncing_before", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label, :parameters => parameters)
292
+ @label = parameters[:label] if parameters.is_a?(Hash) && parameters[:label]
293
+ @controller.vote("activity/syncing_before", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label, :parameters => parameters) do |changes|
294
+ next unless parameters.is_a?(Hash)
295
+ changes.each do |pointer,value|
296
+ CPEE::JSONPointer::set(parameters, pointer, value)
297
+ rescue => e
298
+ puts "vote change #{pointer} ignored: #{e.message}"
299
+ end
300
+ end
292
301
  end # }}}
293
302
 
294
303
  def callback(result=nil,options={}, dataelements) #{{{
@@ -422,7 +431,7 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
422
431
  end
423
432
  __params
424
433
  end #}}}
425
- def test_condition(__eid,__dataelements,__endpoints,__local,__additional,__code,__args={}) #{{{
434
+ def test_condition(__eid,__dataelements,__endpoints,__local,__additional,__code,__args={},__props={}) #{{{
426
435
  send = []
427
436
  send.push Riddl::Parameter::Simple::new('code',__code)
428
437
  send.push Riddl::Parameter::Complex::new('dataelements','application/json', JSON::generate(__dataelements))
@@ -439,7 +448,7 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
439
448
  __struct = 'false' unless __struct
440
449
  __struct = (__struct == 'false' || __struct == 'null' || __struct == 'nil' || __struct == false ? false : true)
441
450
 
442
- @controller.notify("gateway/decide", :eid => __eid, :instance_uuid => @controller.uuid, :code => __code, :condition => __struct)
451
+ @controller.notify("gateway/decide", :eid => __eid, :instance_uuid => @controller.uuid, :code => __code, :condition => __struct, **__props)
443
452
  @controller.notify("gateway/annotation", :eid => __eid, :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :annotations => {})
444
453
  __struct
445
454
  end #}}}
@@ -188,12 +188,15 @@ class Controller
188
188
  @votes += votes
189
189
  psredis = @opts[:redis_dyn].call "Instance #{@id} Vote"
190
190
  collect = []
191
+ changes = []
191
192
  psredis.subscribe(votes.map{|e| ['vote-response:00:' + e.to_s] }.flatten) do |on|
192
193
  on.message do |what, message|
193
194
  index = message.index(' ')
194
195
  mess = message[index+1..-1]
195
196
  m = JSON.parse(mess)
196
- collect << ((m['content'] == true || m['content'] == 'true') || false)
197
+ cont, chg = vote_parse(m['content'])
198
+ collect << cont
199
+ changes << chg if chg && chg.any?
197
200
  @votes.delete m['name']
198
201
  cancel_callback m['name']
199
202
  if collect.length >= votes.length
@@ -201,12 +204,24 @@ class Controller
201
204
  end
202
205
  end
203
206
  end
204
- !collect.include?(false)
207
+ ret = !collect.include?(false)
208
+ changes.each { |chg| yield chg } if ret && block_given?
209
+ ret
205
210
  else
206
211
  true
207
212
  end
208
213
  end
209
214
 
215
+ # single value: true / 'true' continues; hash: { 'continue' => true, '/json/pointer' => value, ... }
216
+ def vote_parse(content)
217
+ val = content.is_a?(String) ? (JSON::parse(content) rescue content) : content
218
+ if val.is_a?(Hash)
219
+ [val['continue'] == true || val['continue'] == 'true', val.select { |k,_| k.to_s.start_with?('/') }]
220
+ else
221
+ [val == true || val == 'true', nil]
222
+ end
223
+ end
224
+
210
225
  def callback(hw,key,content)
211
226
  CPEE::Message::send(:callback,'activity/content',base,@id,uuid,info,content.merge(:key => key),@redis)
212
227
  @callback_keys[key] = hw
@@ -1 +1 @@
1
- 1074986
1
+ 3635871
@@ -1 +1 @@
1
- 1075004
1
+ 3635891
@@ -18,10 +18,26 @@ require 'redis'
18
18
  require 'daemonite'
19
19
  require 'concurrent-ruby'
20
20
  require 'riddl/client'
21
+ require 'digest/sha1'
21
22
  require_relative '../../lib/cpee/redis'
22
23
 
23
24
  HTTP_POOL = Concurrent::FixedThreadPool.new(32)
24
25
 
26
+ # one round trip per event: KEYS[1] is the reverse index (subscription keys for exactly
27
+ # this topic/type/name), ARGV[1] the instance prefix. Returns key, url, sseq per match.
28
+ # GET needs the or '', a nil (false) would truncate the returned array.
29
+ MATCH = <<~LUA.freeze
30
+ local res = {}
31
+ for _, k in ipairs(redis.call('SMEMBERS', KEYS[1])) do
32
+ local b = ARGV[1] .. '/handlers/' .. k
33
+ res[#res+1] = k
34
+ res[#res+1] = redis.call('GET', b .. '/url') or ''
35
+ res[#res+1] = redis.call('INCR', b .. '/seq')
36
+ end
37
+ return res
38
+ LUA
39
+ MATCH_SHA = Digest::SHA1.hexdigest(MATCH).freeze
40
+
25
41
  Daemonite.new do |opts|
26
42
  opts[:runtime_opts] += [
27
43
  ["--url=URL", "-uURL", "Specify redis url", ->(p){ opts[:redis_url] = p }],
@@ -46,26 +62,27 @@ Daemonite.new do |opts|
46
62
  on.pmessage do |pat, what, message|
47
63
  index = message.index(' ')
48
64
  mess = message[index+1..-1]
49
- instance, uuid = message[0...index].split(',')
65
+ instance, uuid, seq = message[0...index].split(',')
50
66
  type, worker, event = what.split(':',3)
51
67
  topic = ::File::dirname(event)
52
68
  name = ::File::basename(event)
53
69
  long = File.join(topic,type,name)
54
70
 
55
- keys = opts[:redis].smembers("instance:#{instance}/handlers")
56
- checks = opts[:redis].pipelined { |p| keys.each { |k| p.sismember("instance:#{instance}/handlers/#{k}", long) } }
57
- matched = keys.zip(checks).select { |_, hit| hit }.map(&:first)
58
- urls = opts[:redis].pipelined { |p| matched.each { |k| p.get("instance:#{instance}/handlers/#{k}/url") } }
59
- matched.zip(urls).each do |key, url|
71
+ # per subscription sequence (sseq): gapless per subscription, so subscribers can detect missing messages; seq gives the order
72
+ res = CPEE::redis_script(opts[:redis], MATCH_SHA, MATCH, ["instance:#{instance}/handlers/#{long}"], ["instance:#{instance}"])
73
+ forwards = []
74
+ res.each_slice(3) do |key, url, sseq|
60
75
  if url.nil? || url == ""
61
- opts[:redis].publish("forward:#{instance}/#{key}",mess)
76
+ forwards << ["forward:#{instance}/#{key}","#{seq},#{sseq} #{mess}"]
62
77
  else
63
78
  # Ractor.new(url,type,topic,name,mess) do |url,type,topic,name,mess|
64
79
  # sadly typhoes does not support ractors
65
80
  HTTP_POOL.post do
66
- Riddl::Client.new(url).post [
81
+ Riddl::Client.new(url, nil, ignore_response: true).post [
67
82
  Riddl::Header.new("CPEE-INSTANCE",instance),
68
83
  Riddl::Header.new("CPEE-INSTANCE-UUID",uuid),
84
+ Riddl::Header.new("CPEE-SEQ",seq),
85
+ Riddl::Header.new("CPEE-SSEQ",sseq),
69
86
  Riddl::Parameter::Simple::new('type',type),
70
87
  Riddl::Parameter::Simple::new('topic',topic),
71
88
  Riddl::Parameter::Simple::new('event',name),
@@ -74,17 +91,11 @@ Daemonite.new do |opts|
74
91
  end
75
92
  end
76
93
  end
77
- unless opts[:redis].exists?("instance:#{instance}/state")
78
- ### delete everything under instance
79
- ### how can there be stuff under an instance (handlers, subscriptions)? Are they created later?
80
- empt = []
81
- opts[:redis].scan_each("instance:#{instance}/*") { |k| empt << k }
82
- opts[:redis].del(*empt) unless empt.empty?
83
- end
84
- rescue => e
85
- puts e.message
86
- puts e.backtrace
87
- p '-----------------'
94
+ # all sse subscribers in one round trip
95
+ opts[:redis].pipelined { |p| forwards.each { |c,m| p.publish(c,m) } } unless forwards.empty?
96
+ rescue
97
+ # deliberately silent: an exception must not end the psubscribe loop (and the
98
+ # worker), and subscribers detect missing messages through gaps in sseq
88
99
  end
89
100
  end
90
101
  end
@@ -1 +1 @@
1
- 1074998
1
+ 3635884