cpee 2.0.5 → 2.0.11

data/cockpit/themes/preset/rngs/callmanipulate.rng

@@ -89,6 +89,11 @@
         </zeroOrMore>
       </element>
     </element>
+    <element name="report" rngui:header="Reporting Annotation" rngui:fold="closed">
+      <element name="url" rngui:label='HTML Snippet'>
+        <data type="string" rngui:label="url to report snippet"/>
+      </element>
+    </element>
   </element>
   <element name="code" rngui:header="Implementation">
     <element name="prepare" rngui:header="Prepare" rngui:label="Code" rngui:fold="closed_conditional">
@@ -137,4 +142,4 @@
       </element>
     </element>
   </element>
-</element>
+</element>

data/cockpit/track.html

@@ -62,12 +62,41 @@
     <div class='hidden' id='relaxngworker'></div>
     <div id="trackfull">
       <div id='graphcolumn'>
-        <div id="usage">
-          <span id="title"></span> | <span id='state'>
-            <span id="state_text"></span>
+        <div id="trackusage">
+          <div id='tracktitle'>
+            <span>
+              <a class="x-ui-button" name="glob_reload" title='reload' href=''>
+                <svg viewBox="0 -1.5 7 12" width="10" height="16">
+                  <path
+										style="fill:#000000;stroke-width:0;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stop-color:#000000"
+     								d="M 3.5,1.5 C 1.431441,1.5001975 -0.24540803,3.1770466 -0.24560554,5.2456055 -0.2454082,7.3141647 1.4314408,8.991014 3.5,8.9912113 5.5685591,8.991014 7.2454081,7.3141647 7.2456054,5.2456055 7.2454079,3.1770466 5.5685589,1.5001975 3.5,1.5 Z M 3.437195,4.0616198 c 0.020931,-5.551e-4 0.041873,-5.551e-4 0.062805,0 0.6537674,3.133e-4 1.1836723,0.5302183 1.1839856,1.1839857 C 4.6836723,5.8993729 4.1537674,6.4292778 3.5,6.4295912 2.8462325,6.4292778 2.3163276,5.8993729 2.3160142,5.2456055 2.315414,4.6155841 2.8080758,4.0953252 3.437195,4.0616198 Z"/>
+                </svg>
+              </a>
+              <a class="x-ui-button" name="glob_edit" title='edit' target='_blank' href='' id='current-monitor'><svg viewBox="0 -1.5 7 12" width="10" height="16">
+                  <path
+                     style="fill:#000000;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+                     d="M -0.25,7.8194404 0.57057194,10.599984 3.1882886,9.3540573 Z"/>
+                  <path
+                     style="fill:#000000;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+                     d="M 3.5215262,8.6074434 6.0527848,2.9361926 2.6144961,1.4015756 0.08323736,7.0728263 Z"/>
+                  <path
+                     style="fill:#000000;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
+                     d="M 6.4439758,2.0597335 6.9719316,0.87685566 3.533643,-0.6577613 3.005687,0.52511656 Z"/>
+                </svg></a>
+              <a class="x-ui-button" name="glob_unshow" title='unshow'>
+                <svg viewBox="0 -1.5 7 12" width="10" height="16">
+                    <path
+                      style="fill:#000000;stroke-width:0;stop-color:#000000"
+                      d="M 5.75,1.4999999 3.5,3.75 1.25,1.4999999 -0.25000013,3 2,5.25 -0.25000013,7.5 1.25,9.0000002 3.5,6.75 5.75,9.0000002 7.2500001,7.5 5,5.25 7.2500001,3 Z"/>
+                </svg>
+              </a>
+            </span>
+            <span> - </span>
+            <span id="title">Loading ...</span>
+          </div>
+          <div id='state'>
             <span id="state_any">
-              <span> ⇒ </span>
-              <button name="state_start" title='start'>
+              <button name="state_start" title='start' style='display:none'>
                 <svg viewBox="0 -1.5 7 12" width="10" height="16">
                   <path
                     style="fill:#000000;fill-rule:evenodd;stroke:#000000;stroke-width:0;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
@@ -82,7 +111,6 @@
                 </svg>
               </button>
               <span id="state_extended">
-                <span> / </span>
                 <button name="state_replay" title='replay'>
                   <svg viewBox="0 -1.5 7 12" width="10" height="16">
                     <path
@@ -90,7 +118,6 @@
                       d="m -0.24999978,8.9999993 c 0,-2.7488167 0,-5.4976332 0,-8.24644997 C 1.1183278,0.76955385 2.4884861,0.71666052 3.8555376,0.79065567 4.5083287,0.83810218 5.2238942,1.0790874 5.5755039,1.7307256 6.1661398,2.7080902 6.0299484,4.2180754 5.1108772,4.9019222 4.7622397,5.1617467 4.3480204,5.2956156 3.9336064,5.3605366 4.9287868,5.9394132 5.4254071,7.0996814 6.0167729,8.0858876 6.1549588,8.3769029 6.4657247,8.7723933 6.5,8.9999999 c -0.5944185,0 -1.1888371,0 -1.7832559,0 C 4.0758679,7.9893475 3.4946311,6.9294648 2.7923497,5.9678488 2.4230355,5.5319748 1.8508715,5.5458842 1.3513114,5.5576027 c -0.1507526,-0.00412 -0.044576,0.272902 -0.076255,0.3990664 0,1.0144436 0,2.028887 0,3.0433307 -0.50835205,-3e-7 -1.01670555,8e-7 -1.52505618,-5e-7 z M 1.2750578,4.2411336 C 2.111059,4.2189781 2.9581852,4.3060525 3.7849326,4.1387394 4.4863715,3.8915585 4.5732952,2.6432813 3.9026802,2.2941453 3.5098807,2.068434 3.050719,2.1857044 2.6267379,2.1489138 c -0.4505578,-0.00242 -0.90112,0.00147 -1.3516801,-3.456e-4 0,0.6975176 0,1.3950347 0,2.0925522 z"
                   </svg>
                 </button>
-                <span> / </span>
                 <button name="state_abandon" title='abandon'>
                   <svg viewBox="0 -1.5 7 12" width="10" height="16">
                     <path
@@ -99,8 +126,10 @@
                   </svg>
                 </button>
               </span>
+              <span> - Status: </span>
             </span>
-          </span>
+            <span id="state_text">loading ...</span>
+          </div>
         </div>
         <div id='graphgrid'>
           <svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:x="http://www.w3.org/1999/xlink" id='graphcanvas' width='1' height='1'></svg>

data/cpee.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
   s.name             = "cpee"
-  s.version          = "2.0.5"
+  s.version          = "2.0.11"
   s.platform         = Gem::Platform::RUBY
   s.license          = "LGPL-3.0"
   s.summary          = "Preliminary release of cloud process execution engine (cpee.org). If you just need workflow execution, without a rest service exposing it, then use WEEL."
 
   s.description      = "see http://cpee.org"
 
-  s.files            = Dir['{example/**/*,server/**/*,tools/**/*,lib/**/*,cockpit/**/*,cockpit/themes/*/*/*,contrib/logo*,contrib/Screen*}'] - Dir['{server/instances/**/*,cockpit/js_libs/**/*}'] + %w(COPYING FEATURES.md INSTALL.md Rakefile cpee.gemspec README.md AUTHORS)
+  s.files            = Dir['{example/**/*,server/**/*,tools/**/*,tools/archive,lib/**/*,cockpit/**/*,cockpit/themes/*/*/*,contrib/logo*,contrib/Screen*}'] - Dir['{server/instances/**/*,cockpit/js_libs/**/*}'] + %w(COPYING FEATURES.md INSTALL.md Rakefile cpee.gemspec README.md AUTHORS)
   s.require_path     = 'lib'
   s.extra_rdoc_files = ['README.md']
   s.bindir           = 'tools'
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   s.email            = 'juergen.mangler@gmail.com'
   s.homepage         = 'http://cpee.org/'
 
-  s.add_runtime_dependency 'riddl', '~> 0.105'
+  s.add_runtime_dependency 'riddl', '~> 0.108'
   s.add_runtime_dependency 'weel', '~> 1.99', '>= 1.99.90'
   s.add_runtime_dependency 'highline', '~> 2.0'
   s.add_runtime_dependency 'json', '~>2.1'

data/lib/cpee/implementation.rb

@@ -72,7 +72,9 @@ module CPEE
     opts[:infinite_loop_stop]         ||= 10000
     opts[:redis_path]                 ||= '/tmp/redis.sock'
     opts[:redis_db]                   ||= 3
+    opts[:sse_keepalive_frequency]    ||= 10
 
+    opts[:sse_connections]            = {}
     opts[:redis]                      = Redis.new(path: opts[:redis_path], db: opts[:redis_db])
     opts[:statemachine]               = CPEE::StateMachine.new opts[:states], %w{running simulating replaying finishing stopping abandoned finished} do |id|
       opts[:redis].get("instance:#{id}/state")
@@ -88,13 +90,20 @@ module CPEE
 
     Proc.new do
       parallel do
-        CPEE::watch_services(@riddl_opts[:watchdog_start_off])
-        EM.add_periodic_timer(@riddl_opts[:watchdog_frequency]) do
-          CPEE::watch_services(@riddl_opts[:watchdog_start_off])
+        CPEE::watch_services(opts[:watchdog_start_off])
+        EM.add_periodic_timer(opts[:watchdog_frequency]) do ### start services
+          CPEE::watch_services(opts[:watchdog_start_off])
+        end
+        EM.defer do ### catch all sse connections
+          CPEE::Notifications::sse_distributor(opts)
+        end
+        EM.add_periodic_timer(opts[:sse_keepalive_frequency]) do
+          CPEE::Notifications::sse_heartbeat(opts)
         end
       end
+
       cleanup do
-        CPEE::cleanup_services(@riddl_opts[:watchdog_start_off])
+        CPEE::cleanup_services(opts[:watchdog_start_off])
       end
 
       interface 'main' do
@@ -183,7 +192,7 @@ module CPEE
       doc = XML::Smart::open_unprotected(opts[:properties_init])
       doc.register_namespace 'p', 'http://cpee.org/ns/properties/2.0'
       name     = @p[0].value
-      id       = redis.zcount('instances','-inf','+inf').to_i + 1
+      id       = redis.zrevrange('instances', 0, 0).first.to_i + 1
       uuid     = SecureRandom.uuid
       instance = 'instance:' + id.to_s
       redis.multi do |multi|

data/lib/cpee/implementation_notifications.rb

@@ -73,28 +73,32 @@ module CPEE
         id = @a[0]
         opts = @a[1]
         key = @r[-1]
-        Riddl::Parameter::Complex.new("subscriptions","text/xml") do
-          ret = XML::Smart::string <<-END
-            <subscription xmlns='http://riddl.org/ns/common-patterns/notifications-producer/2.0'/>
-          END
-          url = CPEE::Persistence::extract_item(id,opts,File.join('handler',key,'url'))
-          ret.root.attributes['url'] = url if url && !url.empty?
-          items = {}
-          CPEE::Persistence::extract_handler(id,opts,key).each do |h|
-            t, i, v = h.split('/')
-            items[t] ||= []
-            items[t] << [i,v]
-          end
-          items.each do |k,v|
-            ret.root.add('topic').tap do |n|
-              n.attributes['id'] = k
-              v.each do |e|
-                n.add *e
+        if CPEE::Persistence::exists_handler?(id,opts,key)
+          Riddl::Parameter::Complex.new("subscriptions","text/xml") do
+            ret = XML::Smart::string <<-END
+              <subscription xmlns='http://riddl.org/ns/common-patterns/notifications-producer/2.0'/>
+            END
+            url = CPEE::Persistence::extract_item(id,opts,File.join('handler',key,'url'))
+            ret.root.attributes['url'] = url if url && !url.empty?
+            items = {}
+            CPEE::Persistence::extract_handler(id,opts,key).each do |h|
+              t, i, v = h.split('/')
+              items[t] ||= []
+              items[t] << [i,v]
+            end
+            items.each do |k,v|
+              ret.root.add('topic').tap do |n|
+                n.attributes['id'] = k
+                v.each do |e|
+                  n.add *e
+                end
               end
             end
+            ret.to_s
           end
-          ret.to_s
-        end
+        else
+          @status = 404
+       end
       end
     end #}}}
 
@@ -102,19 +106,24 @@ module CPEE
       def response
         id = @a[0]
         opts = @a[1]
-        key = Digest::MD5.hexdigest(Kernel::rand().to_s)
-
-        url = @p[0].name == 'url' ? @p.shift.value : nil
-        values = []
-        while @p.length > 0
-          topic = @p.shift.value
-          base = @p.shift
-          type = base.name
-          values += base.value.split(',').map { |i| File.join(topic,type[0..-2],i) }
-        end
-        @header = CPEE::Persistence::set_handler(id,opts,key,url,values)
 
-        Riddl::Parameter::Simple.new('key',key)
+        if opts[:statemachine].readonly? id
+          @status = 423
+        else
+          key = Digest::MD5.hexdigest(Kernel::rand().to_s)
+
+          url = @p[0].name == 'url' ? @p.shift.value : nil
+          values = []
+          while @p.length > 0
+            topic = @p.shift.value
+            base = @p.shift
+            type = base.name
+            values += base.value.split(',').map { |i| File.join(topic,type[0..-2],i) }
+          end
+          @header = CPEE::Persistence::set_handler(id,opts,key,url,values)
+
+          Riddl::Parameter::Simple.new('key',key)
+        end
       end
     end #}}}
 
@@ -124,15 +133,19 @@ module CPEE
         opts = @a[1]
         key = @r.last
 
-        url = @p[0].name == 'url' ? @p.shift.value : nil
-        values = []
-        while @p.length > 0
-          topic = @p.shift.value
-          base = @p.shift
-          type = base.name
-          values += base.value.split(',').map { |i| File.join(topic,type[0..-2],i) }
+        if CPEE::Persistence::exists_handler?(id,opts,key)
+          url = @p[0].name == 'url' ? @p.shift.value : nil
+          values = []
+          while @p.length > 0
+            topic = @p.shift.value
+            base = @p.shift
+            type = base.name
+            values += base.value.split(',').map { |i| File.join(topic,type[0..-2],i) }
+          end
+          @header = CPEE::Persistence::set_handler(id,opts,key,url,values,true)
+        else
+          @status = 404
         end
-        @header = CPEE::Persistence::set_handler(id,opts,key,url,values,true)
       end
     end #}}}
 
@@ -146,41 +159,61 @@ module CPEE
         opts = @a[1]
         key = @r.last
 
-        DeleteSubscription::set(id,opts,key)
+        if CPEE::Persistence::exists_handler?(id,opts,key)
+          DeleteSubscription::set(id,opts,key)
+        else
+          @status = 404
+        end
         nil
       end
     end #}}}
 
-    class SSE < Riddl::SSEImplementation #{{{
-      def onopen
-        @id = @a[0]
-        @opts = @a[1]
-        @key = @r[-2]
-        @conn = Redis.new(path: @opts[:redis_path], db: @opts[:redis_db])
-        EM.defer do
-          @conn.subscribe("forward:#{@id}/#{@key}", "forward-end:#{@id}/#{@key}") do |on|
-            on.message do |what, message|
-              if what == "forward-end:#{@id}/#{@key}"
-                @conn.unsubscribe
-              else
-                send message
+    def self::sse_distributor(opts) #{{{
+      conn = Redis.new(path: opts[:redis_path], db: opts[:redis_db])
+      conn.psubscribe('forward:*','event:state/change') do |on|
+        on.pmessage do |pat, what, message|
+          if pat == 'forward:*'
+            _, id, key = what.match(/forward(-end)?:([^\/]+)\/(.+)/).captures
+            opts.dig(:sse_connections,id.to_i,key)&.send message
+          elsif pat == 'event:state/change'
+            mess = JSON.parse(message[message.index(' ')+1..-1])
+            state = mess.dig('content','state')
+            if state == 'finished' || state == 'abandoned'
+              opts.dig(:sse_connections,mess.dig('instance').to_i)&.each do |key,sse|
+                EM.add_timer(2) do # just to be sure that all messages arrived
+                  sse.close
+                end
               end
             end
           end
-          @conn.close
         end
-        EM.defer do
-          until closed?
-            send_with_id 'hearbeat', '42'
-            sleep 10
-          end
+      end
+      conn.close
+    end #}}}
+    def self::sse_heartbeat(opts) #{{{
+      opts.dig(:sse_connections).each do |id,keys|
+        keys.each do |key,sse|
+          sse.send_with_id('heartbeat', '42') unless sse&.closed?
+        end
+      end
+    end #}}}
+    class SSE < Riddl::SSEImplementation #{{{
+      def onopen
+        @opts = @a[1]
+        @id = @a[0]
+        @key = @r[-2]
+        if !@opts[:statemachine].readonly?(@id) && CPEE::Persistence::exists_handler?(@id,@opts,@key)
+          @opts[:sse_connections][@id] ||= {}
+          @opts[:sse_connections][@id][@key] = self
+          true
+        else
+          false
         end
       end
 
       def onclose
-        tredis = Redis.new(path: @opts[:redis_path], db: @opts[:redis_db])
-        tredis.publish("forward-end:#{@id}/#{@key}",true)
-        tredis.close
+        @opts.dig(:sse_connections,@id)&.delete(@key)
+        @opts.dig(:sse_connections)&.delete(@id) if @opts.dig(:sse_connections,@id)&.length == 0
         DeleteSubscription::set(@id,@opts,@key)
       end
     end #}}}

data/lib/cpee/persistence.rb

@@ -125,6 +125,9 @@ module CPEE
     def self::extract_handler(id,opts,key) #{{{
       opts[:redis].smembers("instance:#{id}/handlers/#{key}")
     end #}}}
+    def self::exists_handler?(id,opts,key) #{{{
+      opts[:redis].exists?("instance:#{id}/handlers/#{key}")
+    end #}}}
     def self::extract_handlers(id,opts) #{{{
       opts[:redis].smembers("instance:#{id}/handlers").map do |e|
         [e, opts[:redis].get("instance:#{id}/handlers/#{e}/url")]

data/server/routing/end.pid

@@ -0,0 +1 @@
+189089

data/server/routing/forward-events.pid

@@ -0,0 +1 @@
+189071

data/server/routing/forward-events.rb

@@ -39,7 +39,8 @@ Daemonite.new do |opts|
             if url.nil? || url == ""
               redis.publish("forward:#{instance}/#{key}",mess)
             else
-              client = Riddl::Client.new(url,'http://riddl.org/ns/common-patterns/notifications-consumer/2.0/consumer.xml')
+              p "#{type}/#{topic}/#{event}-#{url}"
+              client = Riddl::Client.new(url)
               client.post [
                 Riddl::Parameter::Simple::new('type',type),
                 Riddl::Parameter::Simple::new('topic',topic),

data/server/routing/forward-votes.pid

@@ -0,0 +1 @@
+189081

data/server/routing/forward-votes.rb

@@ -70,7 +70,7 @@ Daemonite.new do |opts|
               persist_handler instance, callback_key, m, redis
               redis.publish("forward:#{instance}/#{subscription_key}",mess)
             else
-              client = Riddl::Client.new(url,'http://riddl.org/ns/common-patterns/notifications-consumer/2.0/consumer.xml')
+              client = Riddl::Client.new(url)
               callback = m['instance-url'] + '/callbacks/' + subscription_key
               status, result, headers = (client.post [
                 Riddl::Header.new("CPEE-BASE",m['cpee']),

data/server/routing/persist.pid

@@ -0,0 +1 @@
+189085

data/server/server.conf

@@ -0,0 +1,4 @@
+:host:   tango.wst.univie.ac.at
+:port:   8298
+:url:    http://tango.wst.univie.ac.at:8298
+:secure: false

data/server/server.pid

@@ -0,0 +1 @@
+189185

data/tools/cpee

@@ -117,6 +117,16 @@ elsif command == 'convert'
         doc.find('//x:start_git').each do |e|
           e.text = 'https://centurio.work/flow-test/start/git/'
         end rescue nil
+        doc.find('//d:finalize | //d:update | //d:prepare | //d:rescue').each do |e|
+          if e.parent.qname.name != 'code'
+            n = e.parent
+            if (x = n.find('d:code')).any?
+              x.first.add(e)
+            else
+              n.add('d:code').add(e)
+            end
+          end
+        end rescue nil
 
         doc.find('//p:*').each do |e|
           e.namespaces.delete_all!
@@ -152,7 +162,7 @@ elsif command == 'archive'
         XML::Smart.string(res.response_body) do |doc|
           if doc.root.qname.to_s == "instances"
             doc.root.children.each do |i|
-              if ["finished","abandoned"].include?(i.attributes['state'])
+              if ["finished","abandoned"].include?(i.attributes['state']) || (["ready"].include?(i.attributes['state']) && Time.parse(i.attributes['changed']).to_i < Time.now-(60*60*24))
                 prop = Typhoeus.get(File.join(base,name.to_s,i.attributes['id'],'properties','/'))
                 if prop.success?
                   File.write(File.join(p1,i.attributes['uuid'] + '.xml'),prop.response_body) if prop.headers['Content-Type'] =~ /^(text|application)\/xml/
@@ -165,11 +175,10 @@ elsif command == 'archive'
             if prop.success?
               xprop = XML::Smart::string(prop.response_body)
               xprop.register_namespace 'p', 'http://cpee.org/ns/properties/2.0'
-              if ["finished","abandoned"].include?(xprop.find("string(/p:properties/p:state)"))
+              if ["finished","abandoned"].include?(xprop.find("string(/p:properties/p:state)")) || (["ready"].include?(xprop.find("string(/p:properties/p:state)")) && Time.parse(xprop.find("string(/p:properties/p:state/@changed)")) < Time.now-(60*60*12))
                 uuid = xprop.find("string(/p:properties/p:attributes/p:uuid)")
                 id = name.to_s
                 File.write(File.join(p1,uuid + '.xml'),prop.response_body) if prop.headers['Content-Type'] =~ /^(text|application)\/xml/
-                p File.join(base,name.to_s,'/')
                 Typhoeus.delete(File.join(base,name.to_s,'/'))
               end
             end
@@ -200,11 +209,10 @@ elsif command == 'abandon'
     names << File.basename(p1)
   end
   names.each do |name|
-    res = Typhoeus.get(File.join(base,name.to_s,'properties','state','/'))
-    if res.success?
-      case res.response_body
-        when "ready", "stopped" then
-          Typhoeus.put(File.join(base,name.to_s,'properties','state','/'), headers: {'Content-Type' => 'application/x-www-form-urlencoded'}, body: "value=abandoned")
+    res1 = Typhoeus.get(File.join(base,name.to_s,'properties','state','/'))
+    if res1.success?
+      if res1.response_body == 'ready' || res1.response_body == 'stopped'
+        Typhoeus.put(File.join(base,name.to_s,'properties','state','/'), headers: {'Content-Type' => 'application/x-www-form-urlencoded'}, body: "value=abandoned")
       end
     end
   end
@@ -246,6 +254,7 @@ elsif command == 'delete!'
 elsif command == 'new'
   if !File.exists?(p1)
     FileUtils.cp_r("#{curpath}/server/",p1)
+    FileUtils.mkdir("#{p1}/archive") rescue nil
     FileUtils.mkdir("#{p1}/instances") rescue nil
     FileUtils.mkdir("#{p1}/resources") rescue nil
     FileUtils.mkdir("#{p1}/handlerwrappers") rescue nil
@@ -256,9 +265,6 @@ elsif command == 'new'
       cont.gsub!(/{CPEEWORKINGDIR}/,"#{File.realpath(p1)}")
       cont.gsub!(/{CPEESERVER}/,"#{File.realpath(p1)}/server")
       File.write("#{p1}/#{nam}",cont)
-      p f
-      p nam
-      p "#{p1}/#{nam}"
     end
   else
     puts 'Directory already exists.'