cpee 2.1.121 → 2.1.123

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80d909f1d5117cf3439add6bcfb46571778c8c6c482bc7cd308eb3c2a9c4431f
-  data.tar.gz: fc2d8deeb972a5a57b16b8ebdc2db982b284871386fedbd402dceb9429f8fc10
+  metadata.gz: 63c48292c70c40336ce8385bb86fbea2722ebb3becbfe091843f5d7f60268d0c
+  data.tar.gz: 8b53f0ea495c73ccc423ad1cbca5c764928fdee41ab7af9c853f725c3dc3ed34
 SHA512:
-  metadata.gz: a929aa173158993c2f96eacea72c1c5fd74302d820493e581f39a00f017303149357a3eedb19a1346d96b3050b29ddc3bacf7b2b873bac4c5fc3f1208dc5b99d
-  data.tar.gz: cde8af038d7911aa40825f71cca4cdd22e8e04767c9840992453bc4cc0d6e297f8d9de94f6e7a6921d66cf3826b3435791384cade92639f62532bf6973e91f0e
+  metadata.gz: 977f66a12107bd16c154d55320a3d1f42bf15713419e50dcf1456532dffd332b6d465ec97f2721b61b3ec0d50aab5768b3dabb77a8ecc87ae3e78f63aa63ac77
+  data.tar.gz: 6038b1ba120f2b8ca11e53f5b00987658c6789f33382c215be5bcf5b252f88d3e215c3a4281d614236a40bd93926e2bd4ab90ddb49f97195f67fdead80b5411d

data/cockpit/js/instance.js

@@ -1172,25 +1172,6 @@ async function set_testset(testset,exec) {// {{{
   await Promise.all(promises);
 
   document.dispatchEvent(model_loaded);
-
-  $.ajax({
-    type: "GET",
-    url: url + "/properties/state/",
-    dataType: "text",
-    success: function(res){
-      $.ajax({
-        type: "PUT",
-        url: url + "/properties/state/",
-        data: ({value: res}),
-        error: report_failure,
-        success: function(res){
-          if (exec) start_instance();
-        }
-      });
-    }
-  });
-
-
 }// }}}
 
 function load_testsetfile_after() { //{{{

data/cockpit/js/modifiers.js

@@ -143,10 +143,11 @@ function modifiers_select() {
         let top = $(r).attr('data-resource');
         let it = $(s).attr('value');
         $('select',r).val(it);
-        modifiers_display_ui(rep + 'modifiers/',top,it,save['modifiers_active'][top] == it);
-        if (save['modifiers_active'][top] != it) {
-           modifiers_update({"target": $('select',r)});
+        if (typeof(save['modifiers_active'][top]) == "undefined") {
+          // set for page load
+          save['modifiers_active'][top] = it;
         }
+        modifiers_display_ui(rep + 'modifiers/',top,it,save['modifiers_active'][top] == it);
         save['modifiers_active'][top] = it;
       }
     });

data/cpee.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name             = "cpee"
-  s.version          = "2.1.121"
+  s.version          = "2.1.123"
   s.platform         = Gem::Platform::RUBY
   s.license          = "LGPL-3.0-or-later"
   s.summary          = "The cloud process execution engine (cpee.org). If you just need workflow execution, without a rest service exposing it, then use WEEL."
@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'mimemagic', '~>0'
   s.add_runtime_dependency 'get_process_mem', '~>0.2'
   s.add_runtime_dependency 'webrick', '~>1.7'
+  s.add_runtime_dependency 'msgpack', '~>1.8'
   s.add_runtime_dependency 'by', '~> 1.1', '>= 1.1.0'
   s.add_runtime_dependency 'concurrent-ruby', '~> 1.3', '>= 1.3.5'
 end

data/lib/cpee/implementation_properties.rb

@@ -158,7 +158,11 @@ module CPEE
             CPEE::Properties::PutStatus::set id, opts, node.first.dump
           end
           if (node = doc.find('/p:properties/p:executionhandler')).any?
-            CPEE::Properties::PutExecutionHandler::set id, opts, node.first.text
+            if doc.find('/p:properties/p:description/*').any?
+              CPEE::Properties::PutExecutionHandler::set id, opts, node.first.text, false
+            else
+              CPEE::Properties::PutExecutionHandler::set id, opts, node.first.text
+            end
           end
 
           %w{dataelements endpoints attributes}.each do |item|
@@ -424,10 +428,7 @@ module CPEE
         doc.find("/*/*").each do |ele|
           val[ele.qname.name] = ele.text unless val.has_key?(ele.qname.name)
         end
-        oldkeys = CPEE::Persistence::extract_list(id,opts,item).to_h.keys
-        newkeys = val.keys
-        del = oldkeys - newkeys
-        CPEE::Persistence::set_list(id,opts,item,val,del)
+        CPEE::Persistence::set_list(id,opts,item,val,true,true)
       end
 
       def response
@@ -514,7 +515,7 @@ module CPEE
           @status = 410
         else
           if CPEE::Persistence::extract_item(id,opts,@r.join('/'))
-            CPEE::Persistence::set_list(id,opts,item,val,val.keys)
+            CPEE::Persistence::set_list(id,opts,item,val,false,true,val.keys)
           else
             @status = 404
           end

data/lib/cpee/persistence.rb

@@ -12,6 +12,7 @@
 # CPEE (file COPYING in the main directory).  If not, see
 # <http://www.gnu.org/licenses/>.
 
+require 'msgpack'
 require_relative 'attributes_helper'
 require_relative 'message'
 
@@ -30,26 +31,60 @@ module CPEE
       CPEE::Message::wait(opts[:redis],opts[:redis_dyn].call('Temporary Storage Transaction Subscriber'))
     end
 
-    def self::set_list(id,opts,item,values,deleted=[]) #{{{
+    def self::set_list(id,opts,item,values,diff=false,delete=false,deletions=[]) #{{{
       ah = AttributesHelper.new
       attributes = Persistence::extract_list(id,opts,'attributes').to_h
       dataelements = Persistence::extract_list(id,opts,'dataelements').to_h
       endpoints = Persistence::extract_list(id,opts,'endpoints').to_h
+      oldvalues = case item
+        when 'attributes' then attributes
+        when 'endpoints' then endpoints
+        when 'dataelements' then dataelements
+      end
+      sort = []
+      if diff
+        deletions = oldvalues.keys - values.keys
+        if oldvalues.length > 1 && oldvalues.length == values.length && deletions.length == 0 && oldvalues.keys != values.keys
+          sort = values.keys
+        end
+        values.delete_if do |k,v|
+          oldvalues[k] && oldvalues[k] == v
+        end
+      end
+      payload = {
+        :values => oldvalues.merge(values).transform_values{|val| JSON::parse(val) rescue val },
+        :attributes => ah.translate(attributes,dataelements,endpoints)
+      }
       CPEE::Message::send(
         :event,
-        File.join(item,'change'),
+        File.join(item,'modify'),
         opts[:url],
         id,
         Persistence::extract_item(id,opts,'attributes/uuid'),
         Persistence::extract_item(id,opts,'attributes/info'),
-        {
-          :changed => values.keys,
-          :deleted => deleted,
-          :values => values.transform_values{|val| JSON::parse(val) rescue val },
-          :attributes => ah.translate(attributes,dataelements,endpoints),
-        },
+        payload,
         opts[:redis]
       )
+      payload[:values] = values.transform_values{|val| JSON::parse(val) rescue val }
+      payload[:changed] = values.keys
+      if delete
+        payload[:deleted] = deletions
+      end
+      if sort.any?
+        payload[:sort] = sort
+      end
+      if values.any? || deletions.any? || sort.any?
+        CPEE::Message::send(
+          :event,
+          File.join(item,'change'),
+          opts[:url],
+          id,
+          Persistence::extract_item(id,opts,'attributes/uuid'),
+          Persistence::extract_item(id,opts,'attributes/info'),
+          payload,
+          opts[:redis]
+        )
+      end
     end #}}}
     def self::extract_set(id,opts,item) #{{{
       opts[:redis].smembers(@@obj + ":#{id}/#{item}").map do |e|

data/server/executionhandlers/eval/connection.rb

@@ -29,9 +29,12 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     tso + 2 > tsn && count > 100
   end # }}}
 
-  def self::inform_state_change(arguments,newstate) # {{{
+  def self::inform_state_change(arguments,newstate,data) # {{{
     controller = arguments[0]
-		controller.notify("state/change", :state => newstate)
+		controller.notify('state/change', :state => newstate)
+    if newstate == :stopped || newstate == :finished
+      controller.notify('dataelements/modify', :eid => 'ex-post', :values => data)
+    end
   end # }}}
   def self::inform_syntax_error(arguments,err,code)# {{{
     # TODO extract spot (code) where error happened for better error handling (ruby 3.1 only)

data/server/executionhandlers/eval/controller.rb

@@ -0,0 +1 @@
+../ruby/controller.rb

data/server/executionhandlers/eval/execution.rb

@@ -19,21 +19,22 @@ module CPEE
 
   module ExecutionHandler
 
-    module Eval
-      BACKEND_INSTANCE = 'instance.rb'
-      DSL_TO_DSLX_XSL  = File.expand_path(File.join(__dir__,'dsl_to_dslx.xsl'))
-      BACKEND_RUN      = File.expand_path(File.join(__dir__,'backend','run.rb'))
-      BACKEND_OPTS     = File.expand_path(File.join(__dir__,'backend','opts.yaml'))
-      BACKEND_TEMPLATE = File.expand_path(File.join(__dir__,'backend','instance.template'))
+    self.const_set File.basename(__dir__).capitalize, Module.new {
+      self::BACKEND_INSTANCE = 'instance.rb'
+      self::DSL_TO_DSLX_XSL  = File.expand_path(File.join(__dir__,'dsl_to_dslx.xsl'))
+      self::BACKEND_RUN      = File.expand_path(File.join(__dir__,'backend','run.rb'))
+      self::BACKEND_OPTS     = File.expand_path(File.join(__dir__,'backend','opts.yaml'))
+      self::BACKEND_TEMPLATE = File.expand_path(File.join(__dir__,'backend','instance.template'))
 
       def self::dslx_to_dsl(dslx,ep) # transpile
-        trans = XML::Smart::open_unprotected(ExecutionHandler::Eval::DSL_TO_DSLX_XSL)
+        trans = XML::Smart::open_unprotected(self::DSL_TO_DSLX_XSL)
         dslx.transform_with(trans).to_s
       end
 
       def self::prepare(id,opts) # write result to disk
+        FileUtils.rm_rf(Dir.glob(File.join(opts[:instances],id.to_s,'*')) + Dir.glob(File.join(opts[:instances],id.to_s,'.remote')))
         Dir.mkdir(File.join(opts[:instances],id.to_s)) rescue nil
-        FileUtils.copy(ExecutionHandler::Eval::BACKEND_RUN,File.join(opts[:instances],id.to_s))
+        FileUtils.copy(self::BACKEND_RUN,File.join(opts[:instances],id.to_s))
         dsl = CPEE::Persistence::extract_item(id,opts,'dsl')
         hw = CPEE::Persistence::extract_item(id,opts,'executionhandler')
         endpoints = CPEE::Persistence::extract_list(id,opts,'endpoints')
@@ -43,7 +44,8 @@ module CPEE
         positions.map! do |k, v|
           [ k, v, CPEE::Persistence::extract_item(id,opts,File.join('positions',k,'@passthrough')) ]
         end
-        iopts = YAML::load_file(ExecutionHandler::Eval::BACKEND_OPTS)
+        iopts = YAML::load_file(self::BACKEND_OPTS)
+        iopts[:instance_id] = id
         iopts[:host] = opts[:host]
         iopts[:url] = opts[:url]
         iopts[:redis_url] = opts[:redis_url]
@@ -57,13 +59,26 @@ module CPEE
           iopts[:executionhandlers] = opts[:executionhandlers]
           iopts[:global_executionhandlers] = opts[:global_executionhandlers]
         end
+        iopts[:attributes] = attributes
+        iopts[:votes] = {}
+        CPEE::Persistence::extract_handlers(id,opts).each do |de|
+          if de[1] && de[1]&.empty?.!
+            CPEE::Persistence::extract_handler(id,opts,de[0]).each do |ele|
+              topic, type, name = ele.split('/')
+              if type == 'vote'
+                iopts[:votes][ele] ||= []
+                iopts[:votes][ele] << de[0]
+              end
+            end
+          end
+        end
 
-        File.open(File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Eval::BACKEND_OPTS)),'w') do |f|
+        File.open(File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_OPTS)),'w') do |f|
           YAML::dump(iopts,f)
         end
-        template = ERB.new(File.read(ExecutionHandler::Eval::BACKEND_TEMPLATE), trim_mode: '-')
+        template = ERB.new(File.read(self::BACKEND_TEMPLATE), trim_mode: '-')
         res = template.result_with_hash(dsl: dsl, dataelements: dataelements, endpoints: endpoints, positions: positions)
-        File.write(File.join(opts[:instances],id.to_s,ExecutionHandler::Eval::BACKEND_INSTANCE),res)
+        File.write(File.join(opts[:instances],id.to_s,self::BACKEND_INSTANCE),res)
         if attributes.has_key?('remote')
           uri = URI::parse(attributes['remote'])
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
@@ -77,12 +92,12 @@ module CPEE
       def self::run(id,opts)
         if File.exist? File.join(opts[:instances],id.to_s,'.remote')
           uri = URI::parse(File.read(File.join(opts[:instances],id.to_s,'.remote')))
-          exe = File.join(uri.path,id.to_s,File.basename(BACKEND_RUN))
+          exe = File.join(uri.path,id.to_s,File.basename(self::BACKEND_RUN))
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
             ssh.exec!("ruby #{exe} >#{exe}.out 2>#{exe}.err &")
           end
         else
-          exe = File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Eval::BACKEND_RUN))
+          exe = File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_RUN))
           pid = Kernel.spawn(opts[:libs_preloaderrun] + ' ' + exe , :pgroup => true, :in => '/dev/null', :out => exe + '.out', :err => exe + '.err')
           Process.detach pid
           File.write(exe + '.pid',pid)
@@ -92,7 +107,7 @@ module CPEE
       def self::stop(id,opts) ### return: bool to tell if manually changing redis is necessary
         if File.exist? File.join(opts[:instances],id.to_s,'.remote')
           uri = URI::parse(File.read(File.join(opts[:instances],id.to_s,'.remote')))
-          exe = File.join(uri.path,id.to_s,File.basename(BACKEND_RUN))
+          exe = File.join(uri.path,id.to_s,File.basename(self::BACKEND_RUN))
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
             pid = ssh.exec!("cat #{exe}.pid 2>/dev/null")
             if pid != '' && ssh.exec!("kill -0 #{pid} >/dev/null 2>&1; echo $?").strip == '0'
@@ -104,7 +119,7 @@ module CPEE
             end
           end
         else
-          exe = File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Eval::BACKEND_RUN))
+          exe = File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_RUN))
           pid = File.read(exe + '.pid') rescue nil
           if pid && (Process.kill(0, pid.to_i) rescue false)
             Process.kill('HUP', pid.to_i) rescue nil
@@ -115,7 +130,7 @@ module CPEE
           end
         end
       end
-    end
+    }
 
   end
 

data/server/executionhandlers/ruby/connection.rb

@@ -29,9 +29,12 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     tso + 2 > tsn && count > 100
   end # }}}
 
-  def self::inform_state_change(arguments,newstate) # {{{
+  def self::inform_state_change(arguments,newstate,data) # {{{
     controller = arguments[0]
-		controller.notify("state/change", :state => newstate)
+		controller.notify('state/change', :state => newstate)
+    if newstate == :stopped || newstate == :finished
+      controller.notify('dataelements/modify', :eid => 'ex-post', :values => data)
+    end
   end # }}}
   def self::inform_syntax_error(arguments,err,code)# {{{
     # TODO extract spot (code) where error happened for better error handling (ruby 3.1 only)
@@ -96,7 +99,7 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     }
   end #}}}
 
-  def proto_curl(parameters) #{{{
+  def proto_curl(parameters, dataelements) #{{{
     params = []
     callback = SecureRandom.hex(16)
     (parameters[:arguments] || []).each do |s|
@@ -200,11 +203,11 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
       headers['CPEE_SALVAGE'] = true
       c = result[0]&.value
       c = c.read if c.respond_to? :read
-      callback([ Riddl::Parameter::Complex.new('error','application/json',StringIO.new(JSON::generate({ 'status' => status, 'error' => c }))) ], headers)
+      callback([ Riddl::Parameter::Complex.new('error','application/json',StringIO.new(JSON::generate({ 'status' => status, 'error' => c }))) ], headers, dataelements)
     else
       if headers['CPEE_CALLBACK'] && headers['CPEE_CALLBACK'] == 'true' && result.any?
         headers['CPEE_UPDATE'] = true
-        callback result, headers
+        callback result, headers, dataelements
       elsif headers['CPEE_CALLBACK'] && headers['CPEE_CALLBACK'] == 'true' && result.empty?
         if headers['CPEE_INSTANTIATION']
           @controller.notify("task/instantiation", :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :endpoint => @handler_endpoint, :received => CPEE::ValueHelper.parse(headers['CPEE_INSTANTIATION']))
@@ -214,19 +217,19 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
         end
         # do nothing, later on things will happend
       else
-        callback result, headers
+        callback result, headers, dataelements
       end
     end
   end #}}}
 
-  def activity_handle(passthrough, parameters) # {{{
+  def activity_handle(passthrough, parameters, dataelements) # {{{
     raise "Wrong endpoint" if @handler_endpoint.nil? || @handler_endpoint.empty?
     @label = parameters[:label]
     @anno = parameters.delete(:annotations) rescue nil
     @controller.notify("activity/calling", :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :passthrough => passthrough, :endpoint => @handler_endpoint, :parameters => parameters)
     @controller.notify("activity/annotation", :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :annotations => @anno)
     if passthrough.to_s.empty?
-      proto_curl parameters
+      proto_curl parameters, dataelements
     else
       @controller.callback(self,passthrough,:'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position)
       @handler_passthrough = passthrough
@@ -271,6 +274,9 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     unless changed_dataelements.nil? || changed_dataelements.empty?
       de = dataelements.slice(*changed_dataelements).transform_values { |v| enc = CPEE::EvalRuby::Translation::detect_encoding(v); (enc == 'OTHER' ? v : (v.encode('UTF-8',enc) rescue CPEE::EvalRuby::Translation::convert_to_base64(v))) }
       @controller.notify("dataelements/change", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :label => @label, :activity => @handler_position, :changed => changed_dataelements, :values => de)
+      if @anno && @anno[:_context_data_analysis] && @anno[:_context_data_analysis][:probes] && @anno[:_context_data_analysis][:probes].find{ |p| p.dig(:probe,:extractor_type) == 'intrinsic' }
+        @controller.notify("task/probe", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :label => @label, :activity => @handler_position, :data => dataelements, :probes => @anno[:_context_data_analysis][:probes])
+      end
     end
     unless changed_endpoints.nil? || changed_endpoints.empty?
       @controller.notify("endpoints/change", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :label => @label, :activity => @handler_position, :changed => changed_endpoints, :values => endpoints.slice(*changed_endpoints))
@@ -284,7 +290,7 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     @controller.vote("activity/syncing_before", :'activity-uuid' => @handler_activity_uuid, :endpoint => @handler_endpoint, :activity => @handler_position, :label => @label, :parameters => parameters)
   end # }}}
 
-  def callback(result=nil,options={}) #{{{
+  def callback(result=nil,options={}, dataelements) #{{{
     status, ret, headers = Riddl::Client.new(@controller.url_result_transformation).request 'put' => result
     recv = if status >= 200 && status < 300
       JSON::parse(ret[0].value.read)
@@ -293,6 +299,9 @@ class ConnectionWrapper < WEEL::ConnectionWrapperBase
     end
 
     @controller.notify("activity/receiving", :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :endpoint => @handler_endpoint, :received => recv)
+    if @anno[:_context_data_analysis] && @anno[:_context_data_analysis][:probes] && @anno[:_context_data_analysis][:probes].find{ |p| p.dig(:probe,:extractor_type) == 'extrinsic' }
+      @controller.notify("task/probe", :'activity-uuid' => @handler_activity_uuid, :label => @label, :activity => @handler_position, :endpoint => @handler_endpoint, :received => recv, :data => dataelements, :probes => @anno[:_context_data_analysis]&.[](:probes))
+    end
 
     @guard_files += result
     @guard_files += ret

data/server/executionhandlers/ruby/controller.rb

@@ -21,7 +21,6 @@ require 'cpee/value_helper'
 require 'cpee/attributes_helper'
 require 'cpee/message'
 require 'cpee/redis'
-require 'cpee/persistence'
 require 'get_process_mem'
 
 require 'ostruct'
@@ -42,12 +41,9 @@ class Controller
 
     @id = id
 
-    @attributes = {}
-    CPEE::Persistence::extract_list(id,opts,'attributes').each do |de|
-      @attributes[de[0]] = de[1]
-    end
-
+    @attributes = opts[:attributes]
     @attributes_helper = AttributesHelper.new
+
     @thread = nil
     @opts = opts
     @instance = nil
@@ -74,7 +70,7 @@ class Controller
                   resp << Riddl::Parameter::Complex.new(e[0],e[1][1],e[1][2])
                 end
               end
-              @callback_keys[identifier].send(:callback,resp,m['content']['headers'])
+              @callback_keys[identifier].send(:callback,resp,m['content']['headers'],dataelements)
            end
           end
           if pat == 'callback-end:*'
@@ -183,7 +179,7 @@ class Controller
     handler = File.join(topic,'vote',name)
     votes = []
 
-    CPEE::Persistence::extract_handler(id,@opts,handler).each do |client|
+    @opts[:votes][handler]&.each do |client|
       voteid = SecureRandom.hex(16)
       content[:key] = voteid
       content[:attributes] = attributes_translated

data/server/executionhandlers/ruby/dsl_to_dslx.xsl

@@ -91,9 +91,12 @@
     </xsl:if>
     <xsl:if test="name()='manipulate'">
       <xsl:if test="@label">
-        <xsl:text>, { :label => "</xsl:text>
-          <xsl:value-of select="@label"/>
-        <xsl:text>" }</xsl:text>
+        <xsl:text>, { </xsl:text>
+          <xsl:text>:label => "</xsl:text>
+            <xsl:value-of select="@label"/>
+          <xsl:text>", </xsl:text>
+          <xsl:apply-templates select="d:annotations" mode="annotations"/>
+        <xsl:text> }</xsl:text>
       </xsl:if>
       <xsl:call-template name="print-mcontent">
         <xsl:with-param name="myspace">

data/server/executionhandlers/ruby/execution.rb

@@ -19,22 +19,22 @@ module CPEE
 
   module ExecutionHandler
 
-    module Ruby
-      BACKEND_INSTANCE = 'instance.rb'
-      DSL_TO_DSLX_XSL  = File.expand_path(File.join(__dir__,'dsl_to_dslx.xsl'))
-      BACKEND_RUN      = File.expand_path(File.join(__dir__,'backend','run.rb'))
-      BACKEND_OPTS     = File.expand_path(File.join(__dir__,'backend','opts.yaml'))
-      BACKEND_TEMPLATE = File.expand_path(File.join(__dir__,'backend','instance.template'))
+    self.const_set File.basename(__dir__).capitalize, Module.new {
+      self::BACKEND_INSTANCE = 'instance.rb'
+      self::DSL_TO_DSLX_XSL  = File.expand_path(File.join(__dir__,'dsl_to_dslx.xsl'))
+      self::BACKEND_RUN      = File.expand_path(File.join(__dir__,'backend','run.rb'))
+      self::BACKEND_OPTS     = File.expand_path(File.join(__dir__,'backend','opts.yaml'))
+      self::BACKEND_TEMPLATE = File.expand_path(File.join(__dir__,'backend','instance.template'))
 
       def self::dslx_to_dsl(dslx,ep) # transpile
-        trans = XML::Smart::open_unprotected(ExecutionHandler::Ruby::DSL_TO_DSLX_XSL)
+        trans = XML::Smart::open_unprotected(self::DSL_TO_DSLX_XSL)
         dslx.transform_with(trans).to_s
       end
 
       def self::prepare(id,opts) # write result to disk
         FileUtils.rm_rf(Dir.glob(File.join(opts[:instances],id.to_s,'*')) + Dir.glob(File.join(opts[:instances],id.to_s,'.remote')))
         Dir.mkdir(File.join(opts[:instances],id.to_s)) rescue nil
-        FileUtils.copy(ExecutionHandler::Ruby::BACKEND_RUN,File.join(opts[:instances],id.to_s))
+        FileUtils.copy(self::BACKEND_RUN,File.join(opts[:instances],id.to_s))
         dsl = CPEE::Persistence::extract_item(id,opts,'dsl')
         hw = CPEE::Persistence::extract_item(id,opts,'executionhandler')
         endpoints = CPEE::Persistence::extract_list(id,opts,'endpoints')
@@ -44,7 +44,8 @@ module CPEE
         positions.map! do |k, v|
           [ k, v, CPEE::Persistence::extract_item(id,opts,File.join('positions',k,'@passthrough')) ]
         end
-        iopts = YAML::load_file(ExecutionHandler::Ruby::BACKEND_OPTS)
+        iopts = YAML::load_file(self::BACKEND_OPTS)
+        iopts[:instance_id] = id
         iopts[:host] = opts[:host]
         iopts[:url] = opts[:url]
         iopts[:redis_url] = opts[:redis_url]
@@ -58,13 +59,26 @@ module CPEE
           iopts[:executionhandlers] = opts[:executionhandlers]
           iopts[:global_executionhandlers] = opts[:global_executionhandlers]
         end
+        iopts[:attributes] = attributes
+        iopts[:votes] = {}
+        CPEE::Persistence::extract_handlers(id,opts).each do |de|
+          if de[1] && de[1]&.empty?.!
+            CPEE::Persistence::extract_handler(id,opts,de[0]).each do |ele|
+              topic, type, name = ele.split('/')
+              if type == 'vote'
+                iopts[:votes][ele] ||= []
+                iopts[:votes][ele] << de[0]
+              end
+            end
+          end
+        end
 
-        File.open(File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Ruby::BACKEND_OPTS)),'w') do |f|
+        File.open(File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_OPTS)),'w') do |f|
           YAML::dump(iopts,f)
         end
-        template = ERB.new(File.read(ExecutionHandler::Ruby::BACKEND_TEMPLATE), trim_mode: '-')
+        template = ERB.new(File.read(self::BACKEND_TEMPLATE), trim_mode: '-')
         res = template.result_with_hash(dsl: dsl, dataelements: dataelements, endpoints: endpoints, positions: positions)
-        File.write(File.join(opts[:instances],id.to_s,ExecutionHandler::Ruby::BACKEND_INSTANCE),res)
+        File.write(File.join(opts[:instances],id.to_s,self::BACKEND_INSTANCE),res)
         if attributes.has_key?('remote')
           uri = URI::parse(attributes['remote'])
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
@@ -78,12 +92,12 @@ module CPEE
       def self::run(id,opts)
         if File.exist? File.join(opts[:instances],id.to_s,'.remote')
           uri = URI::parse(File.read(File.join(opts[:instances],id.to_s,'.remote')))
-          exe = File.join(uri.path,id.to_s,File.basename(BACKEND_RUN))
+          exe = File.join(uri.path,id.to_s,File.basename(self::BACKEND_RUN))
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
             ssh.exec!("ruby #{exe} >#{exe}.out 2>#{exe}.err &")
           end
         else
-          exe = File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Ruby::BACKEND_RUN))
+          exe = File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_RUN))
           pid = Kernel.spawn(opts[:libs_preloaderrun] + ' ' + exe , :pgroup => true, :in => '/dev/null', :out => exe + '.out', :err => exe + '.err')
           Process.detach pid
           File.write(exe + '.pid',pid)
@@ -93,7 +107,7 @@ module CPEE
       def self::stop(id,opts) ### return: bool to tell if manually changing redis is necessary
         if File.exist? File.join(opts[:instances],id.to_s,'.remote')
           uri = URI::parse(File.read(File.join(opts[:instances],id.to_s,'.remote')))
-          exe = File.join(uri.path,id.to_s,File.basename(BACKEND_RUN))
+          exe = File.join(uri.path,id.to_s,File.basename(self::BACKEND_RUN))
           Net::SSH.start(uri.host,uri.user,:keys => [ opts[:ssh_key] ] ) do |ssh|
             pid = ssh.exec!("cat #{exe}.pid 2>/dev/null")
             if pid != '' && ssh.exec!("kill -0 #{pid} >/dev/null 2>&1; echo $?").strip == '0'
@@ -105,7 +119,7 @@ module CPEE
             end
           end
         else
-          exe = File.join(opts[:instances],id.to_s,File.basename(ExecutionHandler::Ruby::BACKEND_RUN))
+          exe = File.join(opts[:instances],id.to_s,File.basename(self::BACKEND_RUN))
           pid = File.read(exe + '.pid') rescue nil
           if pid && (Process.kill(0, pid.to_i) rescue false)
             Process.kill('HUP', pid.to_i) rescue nil
@@ -116,7 +130,7 @@ module CPEE
           end
         end
       end
-    end
+    }
 
   end
 

data/server/resources/topics.xml

@@ -50,6 +50,7 @@
   </topic>
   <topic id='dataelements'>
     <event>change</event>
+    <event>modify</event>
   </topic>
   <topic id='endpoints'>
     <event>change</event>

data/server/routing/end.pid

@@ -1 +1 @@
-26491
+1283631

data/server/routing/forward-events-00.pid

@@ -1 +1 @@
-26511
+1283649

data/server/routing/forward-votes.pid

@@ -1 +1 @@
-26505
+1283643

data/server/routing/persist.pid

@@ -1 +1 @@
-26499
+1283637

data/server/routing/persist.rb

@@ -88,10 +88,18 @@ Daemonite.new do |opts|
             end
           when /event:\d+:dataelements\/change/, /event:\d+:endpoints\/change/, /event:\d+:attributes\/change/
             topic = mess.dig('topic')
+            max = opts[:redis].zrange("instance:#{instance}/#{topic}", -1, -1, :withscores => true)&.dig(0,1).to_i
+            if (eles = mess.dig('content','sort'))&.any?
+              opts[:redis].multi do |multi|
+                eles.each_with_index do |e,i|
+                  multi.zadd("instance:#{instance}/#{topic}",i,e,xx: true)
+                end
+              end
+            end
             opts[:redis].multi do |multi|
               mess.dig('content','changed')&.each_with_index do |c,i|
                 unless what =~ /event:\d+:attributes\/change/ && c == 'uuid'
-                  multi.zadd("instance:#{instance}/#{topic}",i,c)
+                  multi.zadd("instance:#{instance}/#{topic}",max + 1 + i,c,nx: true)
                   if what =~ /event:\d+:dataelements\/change/
                     multi.set("instance:#{instance}/#{topic}/#{c}",CPEE::ValueHelper::generate(mess.dig('content','values',c)))
                   else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cpee
 version: !ruby/object:Gem::Version
-  version: 2.1.121
+  version: 2.1.123
 platform: ruby
 authors:
 - Juergen eTM Mangler
@@ -171,6 +171,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: by
   requirement: !ruby/object:Gem::Requirement

data/server/executionhandlers/eval/controller.rb

@@ -1,226 +0,0 @@
-# This file is part of CPEE.
-#
-# CPEE is free software: you can redistribute it and/or modify it under the terms
-# of the GNU General Public License as published by the Free Software Foundation,
-# either version 3 of the License, or (at your option) any later version.
-#
-# CPEE is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# CPEE (file COPYING in the main directory).  If not, see
-# <http://www.gnu.org/licenses/>.
-
-require 'weel'
-require 'json'
-require 'redis'
-require 'securerandom'
-require 'riddl/client'
-require 'cpee/value_helper'
-require 'cpee/attributes_helper'
-require 'cpee/message'
-require 'cpee/redis'
-require 'cpee/persistence'
-require 'get_process_mem'
-
-require 'ostruct'
-class ParaStruct < OpenStruct
-  def to_json(*a)
-    table.to_json
-  end
-end
-def ⭐(a); ParaStruct.new(a); end
-
-class Controller
-  def initialize(id,dir,opts)
-    CPEE::redis_connect(opts,"Instance #{id}")
-    CPEE::Message::set_workers(opts[:workers])
-
-    @redis = opts[:redis]
-    @votes = []
-
-    @id = id
-
-    @attributes = {}
-    CPEE::Persistence::extract_list(id,opts,'attributes').each do |de|
-      @attributes[de[0]] = de[1]
-    end
-
-    @attributes_helper = AttributesHelper.new
-    @thread = nil
-    @opts = opts
-    @instance = nil
-    @loop_guard = {}
-
-    @callback_keys = {}
-
-    @subs = Thread.new do
-      @psredis = @opts[:redis_dyn].call "Instance #{@id} Callback Response"
-      @psredis.psubscribe('callback-response:*','callback-end:*') do |on|
-        on.pmessage do |pat, what, message|
-          if pat == 'callback-response:*'
-            _, worker, identifier = what.split(':')
-            if @callback_keys.has_key?(identifier)
-              index = message.index(' ')
-              mess = message[index+1..-1]
-              instance, uuid = message[0...index].split(',')
-              m = JSON.parse(mess)
-              resp = []
-              m['content']['values'].each do |e|
-                if e[1][0] == 'simple'
-                  resp << Riddl::Parameter::Simple.new(e[0],e[1][1])
-                elsif e[1][0] == 'complex'
-                  resp << Riddl::Parameter::Complex.new(e[0],e[1][1],e[1][2])
-                end
-              end
-              @callback_keys[identifier].send(:callback,resp,m['content']['headers'])
-           end
-          end
-          if pat == 'callback-end:*'
-            _, worker, identifier = what.split(':')
-            @callback_keys.delete(identifier)
-          end
-        end
-      end
-      @psredis.close
-    rescue =>  e
-      sleep 1
-      retry
-    end
-    Thread.new do
-      while true
-        notify("status/resource_utilization", :mib => GetProcessMem.new.mb, **Process.times.to_h)
-        sleep 3
-      end
-    end
-  end
-
-  attr_reader :id
-  attr_reader :attributes
-  attr_reader :loop_guard
-
-  def attributes_translated
-    @attributes_translated || @attributes_translated = @attributes_helper.translate(attributes,dataelements,endpoints)
-  end
-
-  def uuid
-    @attributes['uuid']
-  end
-  def host
-    @opts[:host]
-  end
-  def base_url
-    File.join(@opts[:url],'/')
-  end
-  def instance_url
-    File.join(@opts[:url].to_s,@id.to_s,'/')
-  end
-  def instance_id
-    @id
-  end
-  def base
-    base_url
-  end
-  def instance=(inst)
-    @instance = inst
-  end
-  def endpoints
-    @instance.endpoints
-  end
-  def dataelements
-    @instance.data
-  end
-  def url_result_transformation
-    @opts[:url_result_transformation]
-  end
-  def url_code
-   @opts[:url_code]
-  end
-
-  def start
-    if vote("state/change", :state => 'running')
-      @thread = @instance.start
-      @thread.join
-    else
-      @thread = @instance.stop
-      @thread.join
-    end
-  end
-
-  def sim
-    if vote("state/change", :state => 'simulating')
-      @thread = @instance.sim
-      @thread.join
-    else
-      @thread = @instance.stop
-      @thread.join
-    end
-  end
-
-  def stop
-    ### tell the instance to stop
-    @instance.stop
-    ### end all votes or it will not work
-    Thread.new do # doing stuff in trap context is a nono. but in a thread its fine :-)
-      @votes.each do |key|
-        CPEE::Message::send(:'vote-response',key,base,@id,uuid,info,true,@redis)
-      end
-    end
-  end
-
-  def info
-    @attributes['info']
-  end
-
-  def notify(what,content={})
-    content[:attributes] = attributes_translated
-    CPEE::Message::send(:event,what,base,@id,uuid,info,content,@redis)
-  end
-
-  def vote(what,content={})
-    topic, name = what.split('/')
-    handler = File.join(topic,'vote',name)
-    votes = []
-
-    CPEE::Persistence::extract_handler(id,@opts,handler).each do |client|
-      voteid = SecureRandom.hex(16)
-      content[:key] = voteid
-      content[:attributes] = attributes_translated
-      content[:subscription] = client
-      votes << voteid
-      CPEE::Message::send(:vote,what,base,@id,uuid,info,content,@redis)
-    end
-
-    if votes.length > 0
-      @votes += votes
-      psredis = @opts[:redis_dyn].call "Instance #{@id} Vote"
-      collect = []
-      psredis.subscribe(votes.map{|e| ['vote-response:00:' + e.to_s] }.flatten) do |on|
-        on.message do |what, message|
-          index = message.index(' ')
-          mess = message[index+1..-1]
-          m = JSON.parse(mess)
-          collect << ((m['content'] == true || m['content'] == 'true') || false)
-          @votes.delete m['name']
-          cancel_callback m['name']
-          if collect.length >= votes.length
-            psredis.unsubscribe
-          end
-        end
-      end
-      !collect.include?(false)
-    else
-      true
-    end
-  end
-
-  def callback(hw,key,content)
-    CPEE::Message::send(:callback,'activity/content',base,@id,uuid,info,content.merge(:key => key),@redis)
-    @callback_keys[key] = hw
-  end
-
-  def cancel_callback(key)
-    CPEE::Message::send(:'callback-end',key,base,@id,uuid,info,{},@redis)
-  end
-end