cpee 1.4.8 → 1.4.10

data/lib/engine/instances.rng

@@ -19,7 +19,7 @@
     <element name="instances">
       <zeroOrMore>
         <ref name="instance"/>
-      </zeroOrMore>  
+      </zeroOrMore>
    	</element>
   </start>
 
@@ -28,11 +28,14 @@
       <attribute name='id'>
         <data type="integer"/>
       </attribute>
+      <attribute name='uuid'>
+        <data type="string"/>
+      </attribute>
       <attribute name='state'>
         <data type="string"/>
       </attribute>
       <data type="string"/>
     </element>
-  </define>  
+  </define>
 
 </grammar>

data/lib/instantiation.xml

@@ -1,5 +1,4 @@
 <description datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes" xmlns="http://riddl.org/ns/description/1.0" xmlns:xi="http://www.w3.org/2001/XInclude">
-
   <message name="xmlsimple">
     <parameter name="xml" mimetype="*/xml"/>
   </message>
@@ -64,6 +63,11 @@
     <resource relative="url">
       <post in="url" out="result"/>
     </resource>
+    <resource relative="callback">
+      <resource>
+        <post/>
+      </resource>
+    </resource>
   </resource>
 
 </description>

data/log/elasticsearch.rb

@@ -1,111 +1,6 @@
 #!/usr/bin/ruby
-require 'pp'
-require 'json'
-require 'yaml'
-require 'rubygems'
-require 'riddl/server'
-require 'riddl/utils/fileserve'
-require 'json'
-require 'time'
+require_relative 'elasticsearch_logging'
 
-require 'faraday'
-require 'elasticsearch'
-require 'logger'
-
-class Logging < Riddl::Implementation #{{{
-  def doc(topic,event_name,esc,template,instancenr,notification)
-    uuid = notification['instance_uuid']
-    return unless uuid
-
-    activity = notification['activity']
-    parameters = notification['parameters']
-    receiving = notification['received']
-
-    log = YAML::load(File.read(template))
-    log["log"]["trace"]["concept:name"] ||= instancenr.to_i
-    log["log"]["trace"]["cpee:name"] ||= notification['instance_name'] if notification["instance_name"]
-    log["log"]["trace"]["cpee:uuid"] ||= notification['instance_uuid'] if notification["instance_uuid"]
-    unless esc.indices.exists? index: 'trace'
-      esc.indices.create index: 'trace', body: {
-        "mappings" => {
-          "entry" => {
-            "properties" => {
-              "concept:name" => {
-                "type" => "integer"
-              },
-              "cpee:name" => {
-                "type" => "text"
-              },
-              "cpee:uuid": {
-                "type" => "text"
-              }
-            }
-          }
-        }
-      }
-    end
-
-    esc.index  index: 'trace', type: 'entry', id: log["log"]["trace"]["cpee:uuid"], body: log["log"]["trace"]
-    p notification['attributes']
-
-    event = {}
-    event["trace:id"] = instancenr
-    event["concept:name"] = notification["label"] if notification["label"]
-    if notification["endpoint"]
-      event["concept:endpoint"] = notification["endpoint"]
-    end
-    event["id:id"] = (activity.nil? || activity == "") ? 'external' : activity
-    event["cpee:uuid"] = notification['instance_uuid'] if notification["instance_uuid"]
-    case event_name
-      when 'receiving', 'change'
-        event["lifecycle:transition"] = "unknown"
-      when 'done'
-        event["lifecycle:transition"] = "complete"
-      else
-        event["lifecycle:transition"] = "start"
-    end
-    event["cpee:lifecycle:transition"] = "#{topic}/#{event_name}"
-    data_send = ((parameters["arguments"].nil? ? [] : parameters["arguments"]) rescue [])
-    event["list"] = {"data_send" => data_send} unless data_send.empty?
-    if notification['changed']&.any?
-      if event.has_key? "list"
-        event["list"]["data_changed"] ||= notification['changed']
-      else
-        event["list"] = {"data_changer" => notification['changed']}
-      end
-    end
-    if notification['values']&.any?
-      if event.has_key? "list"
-        event["list"]["data_values"] ||= notification['values']
-      else
-        event["list"] = {"data_values" => notification['values']}
-      end
-    end
-    if receiving&.any?
-      if event.has_key? "list"
-        event["list"]["data_received"] ||= receiving
-      else
-        event["list"] = {"data_receiver" => receiving}
-      end
-    end
-    event["time:timestamp"]= Time.now.iso8601
-
-    iname = "instance_" + notification['instance_name'].downcase.gsub(/[^a-z]/,'_')
-    esc.indices.create index: iname rescue nil
-    esc.index  index: iname, type: 'entry', body: event
-    nil
-  end
-
-  def response
-    topic        = @p[1].value
-    event_name   = @p[2].value
-    esc          = @a[0]
-    template     = @a[1]
-    instancenr   = @h['CPEE_INSTANCE'].split('/').last
-    notification = JSON.parse(@p[3].value)
-    doc topic, event_name, esc, template, instancenr, notification
-  end
-end #}}}
 
 Riddl::Server.new(File.join(__dir__,'/log.xml'), :host => 'localhost', :port => 9307) do
   accessible_description true

data/log/elasticsearch_logging.rb

@@ -0,0 +1,237 @@
+require 'pp'
+require 'time'
+require 'digest'
+
+require 'json'
+require 'yaml'
+require 'riddl/server'
+require 'faraday'
+require 'elasticsearch'
+
+class Logging < Riddl::Implementation
+
+  def doc(topic,event_name,esc,template,instancenr,notification)
+    uuid = notification['instance_uuid']
+    return unless uuid
+
+    # activity   = notification['activity']
+    # parameters = notification['parameters']
+    # receiving  = notification['received']
+    # attributes = notification['attributes']
+
+    unless esc.indices.exists? index: 'artefacts' #{{{
+      esc.indices.create index: 'artefacts', body: {
+        "mappings" => {
+          "entry" => {
+            "properties" => {
+              "group" => {
+                "type" => "keyword"
+
+              },
+              "name" => {
+                "type" => "keyword"
+              }
+            }
+          }
+        }
+      }
+    end #}}}
+    unless esc.indices.exists? index: 'instances' #{{{
+      esc.indices.create index: 'instances', body: {
+        "mappings" => {
+          "entry" => {
+            "properties" => {
+              "uuid" => {
+                "type" => "text"
+              },
+              "group" => {
+                "type" => "keyword"
+              },
+              "name" => {
+                "type" => "keyword"
+              },
+              "date": {
+                "type": "date",
+                "format": "date_time_no_millis"
+              },
+              "info": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      }
+    end #}}}
+    unless esc.indices.exists? index: 'spawned' #{{{
+      esc.indices.create index: 'spawned', body: {
+        "mappings" => {
+          "entry" => {
+            "properties" => {
+              "uuid" => {
+                "type" => "text"
+              },
+              "spawned_uuid" => {
+                "type" => "text"
+              },
+              "date": {
+                "type": "date",
+                "format": "date_time_no_millis"
+              },
+            }
+          }
+        }
+      }
+    end #}}}
+    unless esc.indices.exists? index: 'sensors' #{{{
+      esc.indices.create index: 'sensors', body: {
+        "mappings" => {
+          "entry" => {
+            "properties" => {
+              "uuid" => {
+                "type" => "text"
+              },
+              "sensor" => {
+                "type" => "keyword"
+              },
+              "task" => {
+                "type" => "keyword"
+              },
+              "visualizer_url" => {
+                "type" => "text"
+              },
+              "visualizer_params" => {
+                "type" => "nested"
+              }
+            }
+          }
+        }
+      }
+    end #}}}
+    unless esc.indices.exists? index: 'values' #{{{
+      esc.indices.create index: 'values', body: {
+        "mappings" => {
+          "entry" => {
+            "properties" => {
+              "uuid" => {
+                "type" => "keyword"
+              },
+              "sensor" => {
+                "type" => "keyword"
+              },
+              "task" => {
+                "type" => "keyword"
+              },
+              "timestamp": {
+                "type": "date",
+                "format": "date_time"
+              },
+              "value" => {
+                "type" => "text",
+                "fields" => {
+                  "value_f" => {
+                    "type" => "float",
+                    "store" => true
+                  },
+                  "value_l" => {
+                    "type" => "long",
+                    "store" => true
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    end #}}}
+
+    uuid = notification.dig('instance_uuid')
+
+    if notification.dig('attributes','artefacts') #{{{
+      artefacts = JSON.parse(notification.dig('attributes','artefacts'))
+
+      artefacts.each do |a|
+        aid = Digest::MD5.hexdigest(a['group'] + '_' + a['name'])
+        iid = Digest::MD5.hexdigest(a['group'] + '_' + a['name'] + '_' + uuid)
+        esc.index  index: 'artefacts', type: 'entry', id: aid, body: a
+        esc.index  index: 'instances', type: 'entry', id: iid, body: {
+          'uuid':  uuid,
+          'group': a['group'],
+          'name':  a['name'],
+          'date':  Time.now.iso8601,
+          'info':  notification.dig('attributes','info')
+        }
+
+      end
+    end #}}}
+
+    case "#{topic}/#{event_name}"
+      when "dataelements/change", "endpoints/change"
+        sensors = JSON.parse(notification.dig('attributes','sensors') || '[]')
+        sensors.each do |s|
+          sid = Digest::MD5.hexdigest(uuid + '_' + s['name'])
+          esc.index  index: 'sensors', type: 'entry', id: sid, body: {
+            'uuid': uuid,
+            'sensor': s['name'],
+            'visualizer_url': s['visualizer_url'],
+            'visualizer_params': [s['visualizer_params']]
+          }
+          esc.index  index: 'values', type: 'entry', body: {
+            'uuid': uuid,
+            'sensor': s['name'],
+            'timestamp': notification.dig('timestamp'),
+            'value': s['value']
+          }
+        end
+      when "activity/receiving"
+        sensors = JSON.parse(notification.dig('sensors') || '[]')
+        tdoc = notification.dig('received')
+        sensors.each do |s|
+          sid = Digest::MD5.hexdigest(uuid + '_' + s['name'])
+          esc.index  index: 'sensors', type: 'entry', id: sid, body: {
+            'uuid': uuid,
+            'sensor': s['name'],
+            'task': notification.dig('activity'),
+            'visualizer_url': s['visualizer_url'],
+            'visualizer_params': (s['visualizer_params'].nil? || s['visualizer_params'].empty? ? [] : [s['visualizer_params']])
+          }
+          status, result = Riddl::Client.new(s['extractor_url']).post [
+            Riddl::Parameter::Simple.new('data',JSON.pretty_generate(tdoc)),
+            Riddl::Parameter::Simple.new('what',s['extractor_arg'])
+          ]
+          if status >= 200 && status < 300
+            ret = JSON::parse(result[0]&.value.read) rescue []
+            ret.each do |v,t|
+              esc.index  index: 'values', type: 'entry', body: {
+                'uuid': uuid,
+                'sensor': s['name'],
+                'timestamp': t,
+                'value': v
+              }
+            end
+          end
+
+        end
+    end
+    nil
+  end
+
+  def response #{{{
+    ### save events for later replay
+    # a = {
+    #   :topic        => @p[1].value,
+    #   :event_name   => @p[2].value,
+    #   :instancenr   => @h['CPEE_INSTANCE'].split('/').last,
+    #   :notification => JSON.parse(@p[3].value)
+    # }.to_yaml
+    # File.open('events.yaml','a') do |f|
+    #   f << a
+    # end
+    topic        = @p[1].value
+    event_name   = @p[2].value
+    esc          = @a[0]
+    template     = @a[1]
+    instancenr   = @h['CPEE_INSTANCE'].split('/').last
+    notification = JSON.parse(@p[3].value)
+    doc topic, event_name, esc, template, instancenr, notification
+  end #}}}
+end

data/log/legacy/test_es.rb

@@ -0,0 +1,26 @@
+require 'faraday'
+require 'elasticsearch'
+require 'logger'
+
+client = Elasticsearch::Client.new hosts: ['localhost:8400']
+unless client.indices.exists? index: 'trace'
+  client.indices.create index: 'trace', body: {
+    "mappings" => {
+      "entry" => {
+        "properties" => {
+          "concept:name" => {
+            "type" => "integer"
+          },
+          "cpee:name" => {
+            "type" => "text"
+          },
+          "cpee:uuid": {
+            "type" => "text"
+          }
+        }
+      }
+    }
+  }
+end
+
+client.index  index: 'trace', type: 'entry', id: , body: log["log"]["trace"]

data/log/legacy/test_split.rb

@@ -0,0 +1,211 @@
+require 'yaml'
+require 'pp'
+
+#{{{
+yaml1 = <<-END
+event:
+  cpee:lifecycle:transition: activity/receiving
+  list:
+    data_receiver:
+    - message:
+        mimetype: application/json
+        content:
+        - ID: ns=2;s=/Channel/ProgramInfo/actBlock
+          meta:
+            StatusCodea: Good
+        - ID: ns=2;s=/Channel/Spindle/driveLoad
+          meta:
+            StatusCodeb: Good
+    - message:
+        mimetype: application/xml
+        content:
+        - ID: ns=2;s=/Channel/ProgramInfo/actBlock
+          meta:
+            StatusCodec: Good
+        - ID: ns=2;s=/Channel/Spindle/driveLoad
+          meta:
+            StatusCoded: Good
+  time:timestamp: '2018-05-03T14:08:14+02:00'
+END
+#}}}
+#{{{
+yaml2 = <<-END
+event:
+  trace:id: '160'
+  concept:name: Fetch
+  concept:endpoint: https://centurio.work/data/mt45/queue/48623a67-7b67-4902-b5d4-7243f1d090e2/push
+  id:id: a1
+  lifecycle:transition: unknown
+  cpee:lifecycle:transition: activity/receiving
+  list:
+    data_receiver:
+    - message:
+        mimetype: application/json
+        content:
+        - ID: ns=2;s=/Channel/ProgramInfo/actBlock
+          source: opcua
+          name: Program/actBlock
+          description: Current part program block.
+          path: "/Object/Sinumerik/Channel/ProgramInfo/actBlock"
+          value:
+          timestamp: '2018-05-03 14:16:55.241000'
+          meta:
+            StatusCode: Good
+            ServerTimestamp: '2018-05-03 12:16:55.318853'
+            VariantType: VariantType.String
+            ClientHandle: '212'
+        - ID: ns=2;s=/Channel/Spindle/driveLoad
+          source: opcua
+          name: Spindle/driveLoad
+          description: Load
+          path: "/Object/Sinumerik/Channel/Spindle/driveLoad"
+          value: 0.030517578125
+          timestamp: '2018-05-03 14:16:55.241000'
+          meta:
+            StatusCode: Good
+            ServerTimestamp: '2018-05-03 12:16:55.318853'
+            VariantType: VariantType.Double
+            ClientHandle: '217'
+        - ID: ns=2;s=/Channel/MachineAxis/aaLeadP[u1,3]
+          source: opcua
+          name: Axis/Z/aaLeadP
+          description: ''
+          path: "/Object/Sinumerik/Channel/MachineAxis/aaLeadP[u1,3]"
+          value: 162.51611
+          timestamp: '2018-05-03 14:16:55.241000'
+          meta:
+            StatusCode: Good
+            ServerTimestamp: '2018-05-03 12:16:55.318853'
+            VariantType: VariantType.Double
+            ClientHandle: '222'
+        - ID: ns=2;s=/Channel/MachineAxis/aaTorque[u1,1]
+          source: opcua
+          name: Axis/X/aaTorque
+          description: ''
+          path: "/Object/Sinumerik/Channel/MachineAxis/aaTorque[u1,1]"
+          value: -2.072
+          timestamp: '2018-05-03 14:16:55.241000'
+          meta:
+            StatusCode: Good
+            ServerTimestamp: '2018-05-03 12:16:55.318853'
+            VariantType: VariantType.Double
+            ClientHandle: '223'
+        - ID: ns=2;s=/Channel/MachineAxis/aaTorque[u1,2]
+          source: opcua
+          name: Axis/Y/aaTorque
+          description: ''
+          path: "/Object/Sinumerik/Channel/MachineAxis/aaTorque[u1,2]"
+          value: 0.107
+          timestamp: '2018-05-03 14:16:55.241000'
+          meta:
+            StatusCode: Good
+            ServerTimestamp: '2018-05-03 12:16:55.318853'
+            VariantType: VariantType.Double
+            ClientHandle: '224'
+  time:timestamp: '2018-05-03T14:08:14+02:00'
+END
+#}}}
+
+def traverse(node,paths=[[]],anal=[],depth=0)
+  cpath = paths.last.dup
+  case node
+    when Hash
+      node.each do |k,v|
+        unless cpath.empty?
+          paths.last << [] unless paths.last.last.class == Array
+          paths << cpath.dup
+        end
+        paths.last << k
+        traverse(v,paths,anal,depth+1)
+      end
+    when Array
+      node.each_with_index do |e,i|
+        posanal = [depth,paths.length,nil,[]]
+        anal << posanal
+
+        unless cpath.empty?
+          paths.last << [] unless paths.last.last.class == Array
+          paths << cpath.dup
+        end
+        paths.last << i
+        traverse(e,paths,posanal.last,depth+1)
+        dp = cpath.dup
+        dp << [] unless dp.last.class == Array
+        paths << dp unless paths.include?(dp)
+
+        posanal[2] = paths.length - 1
+      end
+    else
+      paths.last << [] unless paths.last.last.class == Array
+  end
+end
+
+def duplicate(doc,paths,anal)
+  res = []
+  deep_cloned = Marshal::load(Marshal.dump(paths))
+  anal.each_with_index do |e,ei|
+    local_cloned = Marshal::load(Marshal.dump(deep_cloned))
+    anal.select{ |a| a == e }.each do |a|
+      (a[1]).upto(a[2]) do |i|
+        local_cloned[i].last << a[0]
+      end
+    end
+    anal.reject{ |a| a == e }.each do |a|
+      (a[1]).upto(a[2]) do |i|
+        local_cloned[i] = nil
+      end
+    end
+    if !e[3]&.empty?
+      e[3..-1].each_with_index do |ee,eei|
+        ret = duplicate(doc,local_cloned,ee)
+        res.concat ret
+      end
+    else
+      res << extract_from_doc(doc,local_cloned.compact)
+    end
+  end
+  res
+end
+
+def extract_from_doc(doc,paths)
+  ret = {}
+  paths.each do |p|
+    next if p.nil?
+    a = doc.dig(*p[0..-2])
+
+    py = p.dup
+    p[-1].each_with_index do |px,i|
+      py.delete_at(px-i)
+    end
+
+    x1 = py[-2]
+    x2 = py[0..-3]
+    where = ret
+    if x2.any?
+      where = ret.dig(*x2)
+    end
+    where[x1] = {}
+
+    unless a.class == Hash || a.class == Array
+      where[x1] = a
+    end
+  end
+  ret
+end
+
+doc = YAML.load(yaml2)
+
+paths = [[]]
+anal  = []
+traverse(doc,paths,anal)
+anal.uniq!
+
+paths.each do |p|
+  p p
+end
+
+res = duplicate(doc,paths,anal)
+
+res.each do |r|
+  pp r
+end