cpee-logging-xes-yaml 1.3.4 → 1.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aab20ce260e56f4571add36cef8c7fe6f638fad4a723d28f8a13e6e93e99a32c
-  data.tar.gz: 122128fd3e956198fc7096409ff1f64bf47d0e17f5f2e72dd2f4cf3d138ea173
+  metadata.gz: 899e0d2f04869dfa68eb7c7922222c825f87a1bd90d3b73f0cbb528bd95779d3
+  data.tar.gz: 64ef69bebaad740bb8ef4944fb2a5fa3a37d26ab1e0156bf58b75392c9fe8d39
 SHA512:
-  metadata.gz: f36ec97639c73e0735250eee61c39febc1b43e2bab34912780be9a393ccedc45ef3cffdcfd97cd432145a73fcb0935e10dfc405bdd9500117de9afb53522b9de
-  data.tar.gz: 242461d18564a0c2fb9b2b5992e45a99e7506e79aab3eaa08be4c4a1bc4846733b39bc49b7b6e5c7714f719e74ce6afd6fa8225a91642be3188f61c597dfa4f2
+  metadata.gz: 0b0f53c72122468d2f3c4e6a17cd69a7bb7ca0c9088bb843e82ebe3700cb0e6b78a20b453d761648acba181afcb3f9122eb4e4a65c0da166f07249c7b3b48341
+  data.tar.gz: 0557b1e7f8cb2e1538b4984238530d713db7b7927b6ccba5abc949c1b2f180d307370c9d2d6781f689cfce156f1b19339e624dab2eae1a8dfc0478706329ed61

data/cpee-logging-xes-yaml.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name             = "cpee-logging-xes-yaml"
-  s.version          = "1.3.4"
+  s.version          = "1.3.6"
   s.platform         = Gem::Platform::RUBY
   s.license          = "LGPL-3.0"
   s.summary          = "Logging for the cloud process execution engine (cpee.org)"

data/tools/cpee-logging-xes-yaml

@@ -25,6 +25,8 @@ require 'typhoeus'
 require 'stringio'
 require 'typhoeus'
 require 'date'
+require 'msgpack'
+require 'csv'
 
 def wrap(s, width=78, indent=19, extra_indent=2)
 	lines = []
@@ -46,7 +48,75 @@ def wrap(s, width=78, indent=19, extra_indent=2)
 	return lines.join "\n"
 end
 
-def follow(fname,io,copy,deep=0)
+TEMPLATE_XES_XML = <<-END
+  <log xmlns="http://www.xes-standard.org/" xes.version="2.0" xes.features="nested-attributes">
+    <string key="creator" value="cpee.org"/>
+    <extension name="Time" prefix="time" uri="http://www.xes-standard.org/time.xesext"/>
+    <extension name="Concept" prefix="concept" uri="http://www.xes-standard.org/concept.xesext"/>
+    <extension name="ID" prefix="id" uri="http://www.xes-standard.org/identity.xesext"/>
+    <extension name="Lifecycle" prefix="lifecycle" uri="http://www.xes-standard.org/lifecycle.xesext"/>
+    <extension name="CPEE" prefix="cpee" uri="http://cpee.org/cpee.xesext"/>
+    <extension name="stream" prefix="stream" uri="https://cpee.org/datastream/datastream.xesext"/>
+    <global scope="trace">
+      <string key="concept:name" value="__INVALID__"/>
+      <string key="cpee:name" value="__INVALID__"/>
+    </global>
+    <global scope="event">
+      <string key="concept:name" value="__INVALID__"/>
+      <string key="concept:instance" value="-1"/>
+      <string key="concept:endpoint" value=""/>
+      <string key="id:id" value=""/>
+      <string key="lifecycle:transition" value="complete" />
+      <string key="cpee:lifecycle:transition" value="activity/calling"/>
+      <date key="time:timestamp" value=""/>
+    </global>
+    <trace/>
+  </log>
+END
+
+def rec_type(it)
+  if (Time.parse(it) rescue nil)
+    'x:date'
+  elsif it.is_a? Float
+    'x:float'
+  elsif it.is_a? Integer
+    'x:int'
+  elsif it.is_a? String
+    'x:string'
+  end
+end
+
+def rec_a_insert(event,node,level=0)
+  event.each do |i|
+    tnode = node
+    # tnode = node.add('x:string', 'key' => 'item')
+    case i
+      when Hash
+        rec_insert(i,tnode,level+1)
+      when Array
+        rec_a_insert(i,tnode,level+1)
+      when String
+        node.add(rec_type(i), 'key' => i, 'value' => i)
+    end
+  end
+end
+
+def rec_insert(event,node,level=0)
+  event.each do |k,v|
+    case v
+      when String
+        node.add(rec_type(v), 'key' => k, 'value' => v)
+      when Array
+        tnode = node.add('x:list', 'key' => k)
+        rec_a_insert(v,tnode,level+1)
+      when Hash
+        tnode = node.add('x:string', 'key' => k)
+        rec_insert(v,tnode)
+    end
+  end
+end
+
+def follow(fname,io,copy,deep=0,index=nil)
   tname = if fname =~ /\.xes\.shift\.yaml/
     File.basename(fname,'.xes.shift.yaml')
   elsif fname =~ /\.xes\.yaml/
@@ -58,7 +128,7 @@ def follow(fname,io,copy,deep=0)
   end
   YAML.load_stream(io) do |e|
     if name = e.dig('log','trace','cpee:name')
-      puts " " * deep + name + " (#{tname}) - #{e.dig('log','trace','concept:name')}"
+      index.write " " * deep + name + " (#{tname}) - #{e.dig('log','trace','concept:name')}\n"
     end
     if e.dig('event','cpee:lifecycle:transition') == 'task/instantiation'
       base = e.dig('event','raw')
@@ -68,13 +138,14 @@ def follow(fname,io,copy,deep=0)
       end
       uuid = base.dig('CPEE-INSTANCE-UUID') rescue nil
       if uuid
-        react File.dirname(fname) + "/#{uuid}.xes.yaml",copy,deep + 2
+        react File.dirname(fname) + "/#{uuid}.xes.yaml", copy, deep + 2, index
       end
     end
   end
 end
 
-def react(name,copy=false,deep=0)
+def react(name,copy=false,deep=0,index=nil)
+  index ||= File.open('index.txt','w')
   if name.nil?
     help
   elsif name =~ /^https?:\/\//
@@ -83,59 +154,72 @@ def react(name,copy=false,deep=0)
       file = Tempfile.new('sic')
       file.write(res.body)
       file.rewind
-      follow name, file, copy, deep
+      follow name, file, copy, deep, index
       file.close
       file.unlink
     end
   elsif File.exist? name
-    follow name, File.open(name), copy, deep
+    follow name, File.open(name), copy, deep, index
   else
     help
   end
 end
 
 def extract(path)
+  unlink = false
   if path =~ /^http.*/
-    response = Typhoeus.get(path)
-    if response.success?
-      text = response.response_body
+    unlink = true
+    text = Tempfile.new('extract-model-download')
+    request = Typhoeus::Request.new("www.example.com/huge.iso")
+    request.on_headers do |response|
+      if response.code != 200
+        raise "Request failed"
+      end
+    end
+    request.on_body do |chunk|
+      text.write(chunk)
+    end
+    request.on_complete do |response|
+      text.rewind
     end
+    request.run
   else
-    text = File.read(path)
+    text = File.open(path)
   end
   yaml = Psych.load_stream(text)
   changes = []
 
-  info = yaml.shift()
+  info = yaml.shift
   uuid = info.dig('log','trace','cpee:instance')
-  yaml.each() do |el|
-    if(el['event']['id:id'] == 'external' && (el.dig('event','cpee:lifecycle:transition') == 'endpoints/change' || el.dig('event','cpee:lifecycle:transition') == 'dataelements/change'|| el.dig('event','cpee:lifecycle:transition') == 'description/change'))
+  yaml.each do |el|
+    if el['event']['id:id'] == 'external' && (el.dig('event','cpee:lifecycle:transition') == 'endpoints/change' || el.dig('event','cpee:lifecycle:transition') == 'dataelements/change'|| el.dig('event','cpee:lifecycle:transition') == 'description/change')
       changes.push(el)
     end
   end
-  #puts "found #{changes.size()} changes"
 
-  changes.sort!() {|a,b| DateTime.strptime(a.dig('event','time:timestamp'),'%Y-%m-%dT%H:%M:%S.%L%:z') <=> DateTime.strptime(b.dig('event','time:timestamp'),'%Y-%m-%dT%H:%M:%S.%L%:z')}
-  #p changes.map() {|el| el.dig('event','time:timestamp')}
+  changes.sort! { |a,b| DateTime.strptime(a.dig('event','time:timestamp'),'%Y-%m-%dT%H:%M:%S.%L%:z') <=> DateTime.strptime(b.dig('event','time:timestamp'),'%Y-%m-%dT%H:%M:%S.%L%:z') }
 
   de = ep = desc = nil
   counter = 0
-  changes.each() do |change|
-    if(change.dig('event','cpee:lifecycle:transition') == 'dataelements/change') then
+  changes.each do |change|
+    if change.dig('event','cpee:lifecycle:transition') == 'dataelements/change'
       de = change.dig('event','data')
     end
-    if(change.dig('event','cpee:lifecycle:transition') == 'endpoints/change') then
+    if change.dig('event','cpee:lifecycle:transition') == 'endpoints/change'
       ep = change.dig('event','data')
     end
-    if(change.dig('event','cpee:lifecycle:transition') == 'description/change') then
+    if change.dig('event','cpee:lifecycle:transition') == 'description/change'
       desc = change.dig('event','cpee:description')
     end
-    if(change.dig('event','cpee:lifecycle:transition') == 'description/change' || change.dig('event','cpee:lifecycle:transition') == 'endpoints/change') then
+    if change.dig('event','cpee:lifecycle:transition') == 'description/change' || change.dig('event','cpee:lifecycle:transition') == 'endpoints/change'
       yield uuid, de, ep, desc, counter if block_given?
       counter += 1
     end
   end
 
+  text.close
+  text.unlink if unlink
+
   [de, ep, desc]
 end
 
@@ -153,21 +237,28 @@ ARGV.options { |opt|
 	opt.on("")
   opt.on(wrap("new [DIR]          scaffolds a sample logging service. Add a handler to a cpee instance to experience the pleasure."))
 	opt.on("")
-  opt.on(wrap("view [DIR]         view the dependencies between processes and subprocesses. Works for local and remote logs. Examples:\n#{exname} view https://cpee.org/log/123.xes.yaml\n#{exname} view https://cpee.org/log/a.xes.yaml > index.txt\n#{exname} view ~/log/logs/456.xes.yaml"))
+  opt.on(wrap("view [LOG]         view the dependencies between processes and subprocesses. Works for local and remote logs. Examples:\n#{exname} view https://cpee.org/log/123.xes.yaml\n#{exname} view https://cpee.org/log/a.xes.yaml > index.txt\n#{exname} view ~/log/logs/456.xes.yaml"))
 	opt.on("")
-  opt.on(wrap("copy [DIR]         copy dependent processes and subprocesses to the current directory. Works for local and remote logs. Examples: \n#{exname} copy https://cpee.org/log/123.xes.yaml\n#{exname} copy ~/log/logs/456.xes.yaml"))
+  opt.on(wrap("copy [LOG]         copy dependent processes and subprocesses to the current directory. Works for local and remote logs. Examples: \n#{exname} copy https://cpee.org/log/123.xes.yaml\n#{exname} copy ~/log/logs/456.xes.yaml"))
 	opt.on("")
   opt.on(wrap("extract-all [LOG]  extract cpee testset from cpee xes-yaml log. Works for local and remote logs. Write logs to files in folder named like the instance uuid contained in the log. Examples: \n#{exname} extract https://cpee.org/log/123.xes.yaml\n#{exname} extract ~/log/logs/456.xes.yaml"))
 	opt.on("")
-  opt.on(wrap("extract-last [LOG] extract cpee testset from cpee xes-yaml log. Works for local and remote logs. Write last revision of the model to stdout. Examples:\n#{exname} extract https://cpee.org/log/123.xes.yaml\n#{exname} extract ~/log/logs/456.xes.yaml"))
+  opt.on(wrap("extract-last [LOG] extract cpee testset from cpee xes-yaml log. Works for local and remote logs. When called with out [LOG], models for all log files in the current directory are extracted. Examples:\n#{exname} extract https://cpee.org/log/123.xes.yaml\n#{exname} extract ~/log/logs/456.xes.yaml"))
+	opt.on("")
+  opt.on(wrap("index [LOG]        creates an index for a log file, for more efficient parsing. When called without [LOG], indexes all log files in the current directory. Examples:\n#{exname} index https://cpee.org/log/123.xes.yaml\n#{exname} index ~/log/logs/456.xes.yaml"))
+	opt.on("")
+  opt.on(wrap("to-xes-xml [LOG]   convert cpee xes-yaml to xes-xml. Works for local and remote logs. When called with out [LOG], all log files in the current directory are converted. Examples:\n#{exname} to-xes-xml https://cpee.org/log/123.xes.yaml\n#{exname} to-xes-xml ~/log/logs/456.xes.yaml"))
   opt.parse!
 }
-if (ARGV.length != 2)
+
+if (ARGV.length < 1 || ARGV.length > 2)
   puts ARGV.options
   exit
-else
+elsif ARGV.length == 2
   command = ARGV[0]
   path = ARGV[1]
+elsif ARGV.length == 1 && %w{index extract-last to-xes-xml}.include?(ARGV[0])
+  command = ARGV[0]
 end
 
 if command == 'new'
@@ -206,27 +297,108 @@ elsif command == 'extract-all'
     File.write(filename, xml.to_s())
   end
 elsif command == 'extract-last'
-  de, ep, desc = extract(path)
+  path = if path
+    [path]
+  else
+    Dir.glob('*.xes.yaml')
+  end
+
+  path.each do |f|
+    de, ep, desc = extract(f)
 
-  xml = XML::Smart.string('<testset xmlns="http://cpee.org/ns/properties/2.0"><executionhandler>ruby</executionhandler></testset>')
-  dataelements = xml.root().add('dataelements')
-  endpoints = xml.root().add('endpoints')
-  description = xml.root().add('description').add(XML::Smart.string('<description xmlns="http://cpee.org/ns/description/1.0"/>').root())
-  unless de.nil?
-    de.each do |d|
-      dataelements.add(d['name'],d['value'])
+    xml = XML::Smart.string('<testset xmlns="http://cpee.org/ns/properties/2.0"><executionhandler>ruby</executionhandler></testset>')
+    dataelements = xml.root().add('dataelements')
+    endpoints = xml.root().add('endpoints')
+    description = xml.root().add('description').add(XML::Smart.string('<description xmlns="http://cpee.org/ns/description/1.0"/>').root())
+    unless de.nil?
+      de.each do |d|
+        dataelements.add(d['name'],d['value'])
+      end
+    end
+    unless ep.nil?
+      ep.each do |e|
+        endpoints.add(e['name'],e['value'])
+      end
+    end
+    unless desc.nil?
+      description.replace_by(XML::Smart.string(desc).root())
     end
+
+    File.write(f + '.model', xml.to_s)
+  end
+elsif command == 'index'
+  path = if path
+    [path]
+  else
+    Dir.glob('*.xes.yaml')
   end
-  unless ep.nil?
-    ep.each do |e|
-      endpoints.add(e['name'],e['value'])
+
+  path.each do |f|
+    index = []
+    io = File.open(f)
+    while not io.eof?
+      start = io.pos
+      docs = io.readline("---\n",chomp: true)
+      doc = YAML::load(docs, permitted_classes: [Time])
+      if doc
+        transition = doc.dig('event','cpee:lifecycle:transition')
+        if transition =~ /^(activity\/calling|activity\/receiving|task\/instantiation)/
+          endpoint = doc.dig('event','concept:endpoint')
+          uuid = doc.dig('event','cpee:activity_uuid')
+          transition = case transition
+            when 'activity/calling'
+              'c'
+            when 'activity/receiving'
+              'r'
+            when 'task/instantiation'
+              'i'
+          end
+          index << { :e => endpoint.to_s, :u => uuid.to_s, :t => transition.to_s, :s => start.to_i, :l => docs.length.to_i }
+        end
+      end
+    end
+    io.close
+    CSV.open(f + '.index.csv', 'w') do |csv|
+      index.each do |e|
+        csv << e.values
+      end
     end
+
+    nindex = index.group_by{ |a| a[:u] }.collect do |k,v|
+      [v[0][:e], v.collect{ |a| [ a[:t], {:s => a[:s], :l => a[:l]} ] } ]
+    end
+
+    File.write(f + '.index', MessagePack.pack(nindex))
   end
-  unless desc.nil?
-    description.replace_by(XML::Smart.string(desc).root())
+elsif command == 'to-xes-xml'
+  path = if path
+    [path]
+  else
+    Dir.glob('*.xes.yaml')
   end
 
-  puts xml.to_s
+  path.each do |f|
+    xml = XML::Smart.string(TEMPLATE_XES_XML)
+    xml.register_namespace 'x', 'http://www.xes-standard.org/'
+
+    io = File.open(f)
+    YAML.load_stream(io) do |e|
+      if trace = e.dig('log','trace')
+        trace.each do |t,tv|
+          xml.find('//x:trace').each do |ele|
+            ele.add('x:string', 'key' => t, 'value' => tv)
+          end
+        end
+      end
+      if e.dig('event')
+        xml.find('//x:trace').each do |node|
+          rec_insert(e.dig('event'),node.add('x:event'))
+        end
+      end
+    end
+
+    File.write(File.basename(f,'.xes.yaml') + '.xes.xml', xml.to_s)
+  end
 else
   puts ARGV.options
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cpee-logging-xes-yaml
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.3.6
 platform: ruby
 authors:
 - Juergen eTM Mangler
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: tools
 cert_chain: []
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: riddl