cpee-logging-xes-yaml 1.3.30 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2475b45c7eb85a9d59ad80640699cbf91cfc14b67e4ac7b3d64954b559d7a34
-  data.tar.gz: a80f1b2d4e65322df40bd4e7a0234d2721cc44b9d5f1fa4adde78f84b1d8c7d4
+  metadata.gz: '093abf131db31464c118276deec2c33110a1eada1249b40451fca9eaeee3c38d'
+  data.tar.gz: 8ba44738e83d6d5487f33026012d9ee813bd4dada97d0e4d4ffeefa0b04494b5
 SHA512:
-  metadata.gz: b0ddbf8911d7131428e8eb2a70457f22de9dc81f6a4fad430f894442bc1d62e236ff4a883f4460b9875a56b1b2734a1731169a92752faa6a59b97f221c18aa62
-  data.tar.gz: c86ee1ba144e1d557505e405efc06d27ae180996add76376968f1e093220b95c4d030e85a3aed987e44a81679fa6f8be8d639a537cfe51b4b3574a718ba7a905
+  metadata.gz: 2b20ebd080a36d9e71ea5e1bf1556d26c2e709fe28e9dd783058111dc4472bd52ecac7562d4e97a1c756a7768e29aec4f7aed4ec9cfeaf1df9e53a40d47e25a8
+  data.tar.gz: 958888e149af9efc14c16429830a3c54db0e97ac681826f08072a0e902ec036ae9369df8ae5b33599a8968f75fa70f774520e53f2e4429b8aca0923e90f65911

data/cpee-logging-xes-yaml.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name             = "cpee-logging-xes-yaml"
-  s.version          = "1.3.30"
+  s.version          = "1.4.1"
   s.platform         = Gem::Platform::RUBY
   s.license          = "LGPL-3.0-or-later"
   s.summary          = "Logging for the cloud process execution engine (cpee.org)"

data/lib/cpee-logging-xes-yaml/logging.rb

@@ -36,19 +36,20 @@ module CPEE
         event_name = @p[2].value
         payload    = @p[3].value.read
 
-        unless File.exist? File.join(opts[:log_dir],@h['CPEE_INSTANCE_UUID'] + '.xes.yaml')
-          notification = JSON.parse(payload)
-          log = YAML::load(File.read(opts[:template]))
-          log['log']['trace']['concept:name']                    ||= notification['instance']
-          log['log']['trace']['cpee:name']                       ||= notification['instance-name'] if notification['instance-name']
-          log['log']['trace']['cpee:instance']                   ||= notification['instance-uuid']
-          log['log']['trace']['cpee:parent_instance']            ||= notification.dig('content','attributes','parent_instance').to_i       if notification.dig('content','attributes','parent_instance')
-          log['log']['trace']['cpee:parent_instance_uuid']       ||= notification.dig('content','attributes','parent_instance_uuid')       if notification.dig('content','attributes','parent_instance_uuid')
-          log['log']['trace']['cpee:parent_instance_model']      ||= notification.dig('content','attributes','parent_instance_model')      if notification.dig('content','attributes','parent_instance_model')
-          log['log']['trace']['cpee:parent_instance_task_id']    ||= notification.dig('content','attributes','parent_instance_task_id')    if notification.dig('content','attributes','parent_instance_task_id')
-          log['log']['trace']['cpee:parent_instance_task_label'] ||= notification.dig('content','attributes','parent_instance_task_label') if notification.dig('content','attributes','parent_instance_task_label')
-          File.open(File.join(opts[:log_dir],@h['CPEE_INSTANCE_UUID']+'.xes.yaml'),'w'){|f| f.puts log.to_yaml}
-        end
+        ### we do not write headers for now. Or else we can only do per instance sharding.
+        # unless File.exist? File.join(opts[:log_dir],@h['CPEE_INSTANCE_UUID'] + '.xes.yaml')
+        #   notification = JSON.parse(payload)
+        #   log = YAML::load(File.read(opts[:template]))
+        #   log['log']['trace']['concept:name']                    ||= notification['instance']
+        #   log['log']['trace']['cpee:name']                       ||= notification['instance-name'] if notification['instance-name']
+        #   log['log']['trace']['cpee:instance']                   ||= notification['instance-uuid']
+        #   log['log']['trace']['cpee:parent_instance']            ||= notification.dig('content','attributes','parent_instance').to_i       if notification.dig('content','attributes','parent_instance')
+        #   log['log']['trace']['cpee:parent_instance_uuid']       ||= notification.dig('content','attributes','parent_instance_uuid')       if notification.dig('content','attributes','parent_instance_uuid')
+        #   log['log']['trace']['cpee:parent_instance_model']      ||= notification.dig('content','attributes','parent_instance_model')      if notification.dig('content','attributes','parent_instance_model')
+        #   log['log']['trace']['cpee:parent_instance_task_id']    ||= notification.dig('content','attributes','parent_instance_task_id')    if notification.dig('content','attributes','parent_instance_task_id')
+        #   log['log']['trace']['cpee:parent_instance_task_label'] ||= notification.dig('content','attributes','parent_instance_task_label') if notification.dig('content','attributes','parent_instance_task_label')
+        #   File.open(File.join(opts[:log_dir],@h['CPEE_INSTANCE_UUID']+'.xes.yaml'),'w'){|f| f.puts log.to_yaml}
+        # end
 
         EM.defer do
           CPEE::Logging::forward opts, topic, event_name, payload

data/lib/cpee-logging-xes-yaml/tools.rb

@@ -81,13 +81,15 @@ module CPEE
       opts[:subscriptions].each do |e,urls|
         if e == topic + '/' + event_name
           urls.each do |url|
-            client = Riddl::Client.new(url)
-            client.post [
-              Riddl::Parameter::Simple::new('type','event'),
-              Riddl::Parameter::Simple::new('topic',topic),
-              Riddl::Parameter::Simple::new('event',event_name),
-              Riddl::Parameter::Complex::new('notification','application/json',payload)
-            ]
+            EM.defer do
+              client = Riddl::Client.new(url)
+              client.post [
+                Riddl::Parameter::Simple::new('type','event'),
+                Riddl::Parameter::Simple::new('topic',topic),
+                Riddl::Parameter::Simple::new('event',event_name),
+                Riddl::Parameter::Complex::new('notification','application/json',payload)
+              ]
+            end
           end
         end
       end
@@ -150,7 +152,7 @@ module CPEE
               ret[tid] = ret[tid][0]
             end
             hash = Digest::SHA1.hexdigest(ret[tid])
-            if !File.exist?(fname) || (File.exist?(fname) && File.read(fname) !=  hash)
+            if !File.exist?(fname) || File.read(fname) !=  hash
               File.write(fname,hash)
             end
           end
@@ -213,7 +215,7 @@ module CPEE
     def self::load_values(where)
       ret = nil
       File.open(where,'r') do |f|
-        f.flock(File::LOCK_EX)
+        f.flock(File::LOCK_SH)
         ret = JSON::load(f)
         f.flock(File::LOCK_UN)
       end
@@ -223,8 +225,6 @@ module CPEE
     def self::forward(opts,topic,event_name,payload)
       if topic == 'state' && event_name == 'change'
         self::notify(opts,topic,event_name,payload)
-      elsif topic == 'state' && event_name == 'change'
-        self::notify(opts,topic,event_name,payload)
       elsif topic == 'gateway' && event_name == 'join'
         self::notify(opts,topic,event_name,payload)
       end
@@ -236,7 +236,6 @@ module CPEE
       return unless instance
 
       log_dir = opts[:log_dir]
-      template = opts[:template]
 
       instancenr = notification['instance']
       content = notification['content']
@@ -250,7 +249,7 @@ module CPEE
       if content['dslx']
         CPEE::Logging::extract_probes(File.join(log_dir,instance),content['dslx'])
         CPEE::Logging::extract_annotations(File.join(log_dir,instance),content['dslx']).each do |k,v|
-          so = Marshal.load(Marshal.dump(notification))
+          so = JSON.parse(notification.to_json)
           so['content'].delete('dslx')
           so['content'].delete('dsl')
           so['content'].delete('description')
@@ -324,11 +323,17 @@ module CPEE
             end
           end
           if event['stream:datastream'] && event['stream:datastream'].any?
-            notification['datastream'] = event['stream:datastream']
             EM.defer do
-              notification['topic'] = 'stream'
-              notification['name'] = 'extraction'
-              self::notify(opts,'stream','extraction',notification.to_json)
+              self::notify(
+                opts,
+                'stream',
+                'extraction',
+                notification.merge(
+                  'topic'=>'stream',
+                  'name'=>'extraction',
+                  'datastream'=>event['stream:datastream']
+                ).to_json
+              )
             end
           end
         end
@@ -362,11 +367,17 @@ module CPEE
             File.open(File.join(log_dir,instance+'.xes.yaml'),'a') do |f|
               f << {'event' => te}.to_yaml
             end
-            notification['datastream'] = te['stream:datastream']
             EM.defer do
-              notification['topic'] = 'stream'
-              notification['name'] = 'extraction'
-              self::notify(opts,'stream','extraction',notification.to_json)
+              self::notify(
+                opts,
+                'stream',
+                'extraction',
+                notification.merge(
+                  'topic'=>'stream',
+                  'name'=>'extraction',
+                  'datastream'=>te['stream:datastream']
+                ).to_json
+              )
             end
           end
         end

data/tools/cpee-logging-xes-yaml

@@ -48,7 +48,7 @@ def wrap(s, width=78, indent=19, extra_indent=2)
 	return lines.join "\n"
 end
 
-TEMPLATE_XES_XML = <<-END
+TEMPLATE_XES_XML_START = <<-END
   <log xmlns="http://www.xes-standard.org/" xes.version="2.0" xes.features="nested-attributes">
     <string key="creator" value="cpee.org"/>
     <extension name="Time" prefix="time" uri="http://www.xes-standard.org/time.xesext"/>
@@ -70,7 +70,18 @@ TEMPLATE_XES_XML = <<-END
       <string key="cpee:lifecycle:transition" value="activity/calling"/>
       <date key="time:timestamp" value="__NOTSPECIFIED__"/>
     </global>
-    <trace/>
+END
+TEMPLATE_XES_XML_TRC = <<-END
+  <trace xmlns="http://www.xes-standard.org/"/>
+END
+TEMPLATE_XES_XML_EVT = <<-END
+  <event xmlns="http://www.xes-standard.org/"/>
+END
+TEMPLATE_XES_XML_MID = <<-END
+    <trace>
+END
+TEMPLATE_XES_XML_END = <<-END
+    </trace>
   </log>
 END
 
@@ -86,16 +97,27 @@ def rec_type(it)
   end
 end
 
+def format_secs(s)
+  return 'long' if s.infinite?
+  s = s.to_i
+  m = s / 60
+  m < 0 ? "#{s}s" : "#{'%02d' % m}m #{'%02d' % (s%60)}s"
+end
+
 def rec_a_insert(event,node,level=0)
   event.each do |i|
     tnode = node
     case i
       when Hash
+        tnode = node.add('x:list', 'key' => 'element')
         rec_insert(i,tnode,level+1)
       when Array
-        rec_a_insert(i,tnode,level+1)
+        tnode = node.add('x:list', 'key' => 'element')
+        rec_insert(i,tnode,level+1)
       when String
-        node.add(rec_type(i), 'key' => i, 'value' => (i.empty? ? "__UNSPECIFIED__" : i))
+        node.add(rec_type(i), 'key' => 'element', 'value' => (i.empty? ? "__UNSPECIFIED__" : i))
+      when Integer, Float
+        node.add(rec_type(i), 'key' => 'element', 'value' => i)
     end
   end
 end
@@ -240,7 +262,7 @@ ARGV.options { |opt|
   opt.on("--help", "-h", "This text") { puts opt; exit }
   opt.on("--add [STRING]", "-a [STRING]", "Add attribute for extract-all and extract-last. Example: -a 'filter=me'") { |a| x,y = a.split('='); attr[x] = y }
 	opt.on("")
-  opt.on(wrap("\"#{exname}\" will be call \"c\" in the examples for each command."))
+  opt.on(wrap("\"#{exname}\" will be called \"c\" in the examples for each command."))
   exname = 'c'
 	opt.on("")
   opt.on(wrap("new [DIR]          scaffolds a sample logging service. Add a handler to a cpee instance to experience the pleasure."))
@@ -403,26 +425,41 @@ elsif command == 'to-xes-xml'
   end
 
   path.each do |f|
-    xml = XML::Smart.string(TEMPLATE_XES_XML)
-    xml.register_namespace 'x', 'http://www.xes-standard.org/'
+    fname = File.basename(f,'.xes.yaml') + '.xes.xml'
+    File.write(fname,TEMPLATE_XES_XML_START)
 
     io = File.open(f)
+    count = 1
+    start = Time.now
+    total = `grep -c -e '---' #{f}`.to_f
+    percent = 0
+    duration = 0
     YAML.load_stream(io) do |e|
+      percent = count/total
+      diff = Time.now - start
+      duration = diff/percent
+      print "#{"%0#{Math.log10(total).to_i + 1}d" % count}/#{total.to_i} #{'%05.2f' % (percent*100)}% (#{format_secs(duration-diff)} left)          \r" if (count % 1000) == 0
+      count += 1
       if trace = e.dig('log','trace')
+        xml = XML::Smart.string(TEMPLATE_XES_XML_TRC)
+        xml.register_namespace 'x', 'http://www.xes-standard.org/'
         trace.each do |t,tv|
           xml.find('//x:trace').each do |ele|
             ele.add('x:string', 'key' => t, 'value' => tv)
           end
         end
+        File.write(fname, '    ' + xml.root.dump.gsub(/\n/,"\n    ") + "\n", mode: 'a')
+        File.write(fname, TEMPLATE_XES_XML_MID, mode: 'a')
       end
       if e.dig('event')
-        xml.find('//x:trace').each do |node|
-          rec_insert(e.dig('event'),node.add('x:event'))
-        end
+        xml = XML::Smart.string(TEMPLATE_XES_XML_EVT)
+        xml.register_namespace 'x', 'http://www.xes-standard.org/'
+        rec_insert(e.dig('event'),xml.root)
+        File.write(fname, '      ' + xml.root.dump.gsub(/\n/,"\n      ") + "\n", mode: 'a')
       end
     end
-
-    File.write(File.basename(f,'.xes.yaml') + '.xes.xml', xml.to_s)
+    File.write(fname, TEMPLATE_XES_XML_END, mode: 'a')
+    puts "#{count}/#{total.to_i} 100% (took #{format_secs(duration)})                                         \n"
   end
 else
   puts ARGV.options

data/tools/cpee-logging-xes-yaml-finder

@@ -0,0 +1,56 @@
+#!/usr/bin/ruby
+#
+# This file is part of CPEE-LOGGING-XES-YAML.
+#
+# CPEE-LOGGING-XES-YAML is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# CPEE-LOGGING-XES-YAML is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Lesser General Public License along with
+# CPEE-LOGGING-XES-YAML (file LICENSE in the main directory).  If not, see
+# <http://www.gnu.org/licenses/>.
+
+path = ARGV.shift()
+lookup_lines = ARGV.shift().to_i()
+regexes = ARGV
+regexes.map!() { |el| el.gsub("\\n","\n") }
+
+dir = Dir.open(File.join(path))
+children = dir.children()
+#pp children.length()
+#pp children.filter() { |el| el.include?('.xes.yaml') }.length()
+#pp children.filter() { |el| el.include?('.xes.shift.yaml') }.length()
+#pp children.filter() { |el| el.include?('data.json') }.length()
+#pp children.filter() { |el| el.include?('.anno') }.length()
+#pp children.filter() { |el| el.include?('.probe') }.length()
+#pp children.filter() { |el| el.include?('.branches.json') }.length()
+#pp children.filter() { |el| el.include?('.shift.json') }.length()
+children.filter!() { |el| el.end_with?('.xes.yaml') }
+children.each() { |filename|
+  begin
+    file = File.open(File.join(path,filename))
+    lookup_lines.times() {
+      line = file.readline()
+      regexes.each() { |regex|
+        #pp regex
+        if(line.include?(regex)) then
+          puts "#{filename} matches '#{regex.gsub("\n","\\n")}' in line '#{line.gsub("\n","\\n")}' (file last modified: #{file.stat().mtime()})"                                             
+          break
+        end
+      }
+    }
+  rescue Exception => e
+    #pp "Problem opening or reading file: #{e} -> go to next file"
+    #pp filename
+    next
+  ensure
+    file.close()
+  end
+}
+dir.close()

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cpee-logging-xes-yaml
 version: !ruby/object:Gem::Version
-  version: 1.3.30
+  version: 1.4.1
 platform: ruby
 authors:
 - Juergen eTM Mangler
@@ -112,6 +112,7 @@ files:
 - lib/cpee-logging-xes-yaml/topics.xml
 - server/log
 - tools/cpee-logging-xes-yaml
+- tools/cpee-logging-xes-yaml-finder
 homepage: http://cpee.org/
 licenses:
 - LGPL-3.0-or-later