camtool 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3f150b39a20695aa69a3bde1a84e1f22f78e1fc007920b10a8a49c332c3d0512
+  data.tar.gz: 94f865410c2a203d77e4ca8fcbc3db3422b12e79412a9caf8cbed2ce06c86f2e
+SHA512:
+  metadata.gz: 2c7c82e135f38b92b6fb70468c9fe42c4647e0fc3af17fd2208d38e4326e6f2e43d6d03965abc769341697181fbe6ebf498e1442a0013a7a33de9c809614a9ba
+  data.tar.gz: 187a61ec87799ac4c7e9546b624263a252948396d2a96fabd967313028fc6bd01629dd62893f0c126a13bb73acccf5e84d889754a1e71b7432cf46ce9e53f693

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2015-2019 University of Bristol, University of Cambridge,
+Harvard University, University of British Columbia http://camflow.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,17 @@
+# camflow-ruby
+
+```
+sudo dnf install ruby ruby-devel redhat-rpm-config graphviz
+```
+
+```
+gem install json rgl mqtt rake bundler
+```
+
+## install
+
+```
+git clone https://github.com/tfjmp/camtool
+cd camtool
+rake install
+```

data/bin/camtool

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'camtool.rb'
+require 'rgl/dot'
+require 'rgl/topsort'
+require 'rgl/transitivity'
+
+instruction = ARGV[0]
+target = ARGV[1]
+
+if instruction == '--version' || instruction == '-v'
+   CamTool.version
+elsif instruction == '--info'
+  print CamTool::ProvJSONtoRGL.new.read_log_file(target).information
+  CamTool::EdgeCounter.new.read_log_file(target).show
+  CamTool::VertexCounter.new.read_log_file(target).show
+elsif instruction == '--publish'
+  CamTool::MQTTBreaker.new.break_log_file target
+elsif instruction == '--break'
+  CamTool::LogBreaker.new.break_log_file target
+elsif instruction == '--validate'
+  CamTool::ProvJSONtoRGL.new.read_log_file(target).validate
+  puts 'Graph structure verified...'
+  CamTool::EdgeValidator.new.read_log_file(target)
+  puts 'Relations verified...'
+elsif instruction == '--packet'
+  CamTool::CheckPacket.new.read_log_file(target)
+elsif instruction == '--png'
+  print CamTool::ProvJSONtoRGL.new.read_log_file(target).png
+elsif instruction == '--list'
+    types = ARGV[2]
+    CamTool::ProvJSONtoRGL.new.read_log_file(target).list(types)
+else
+  puts '--version'
+  puts '--info <path to log file>'
+  puts '--publish <path to log file>'
+  puts '--break <path to log file>'
+  puts '--png <path to log file>'
+  puts '--packet <path to log file>'
+  puts '--validate <path to log file>'
+  puts '--list <path to log file> [optional comma-separated list of element type e.g. "path,argv"]'
+end

data/lib/camtool.rb

@@ -0,0 +1,9 @@
+require 'camtool/provjson_parser'
+require 'camtool/provjson_to_rgl'
+require 'camtool/edge_counter'
+require 'camtool/edge_validator'
+require 'camtool/vertex_counter'
+require 'camtool/log_breaker'
+require 'camtool/check_packet.rb'
+require 'camtool/breaker_mqtt'
+require 'camtool/version'

data/lib/camtool/breaker_mqtt.rb

@@ -0,0 +1,24 @@
+require 'mqtt'
+require 'zlib'
+require 'base64'
+
+module CamTool
+  class MQTTBreaker < LogBreaker
+    def initialize
+      conn_opts = {
+        remote_host: 'm12.cloudmqtt.com',
+        remote_port: 17065,
+        username: 'camflow',
+        password: 'test',
+      }
+      @c = MQTT::Client.connect conn_opts
+    end
+
+    def new_entry entry
+      comp = Zlib::Deflate.deflate entry
+      ready = Base64.encode64 comp
+      @c.publish 'camflow/test', ready
+      sleep 5
+    end
+  end
+end

data/lib/camtool/check_packet.rb

@@ -0,0 +1,10 @@
+module CamTool
+  class CheckPacket < ProvJSONParser
+    def initialize
+    end
+
+    def entity k, v
+      puts v['cf:jiffies']+"\t"+v['prov:label'] unless v['prov:type'] != 'packet'
+    end
+  end
+end

data/lib/camtool/edge_counter.rb

@@ -0,0 +1,41 @@
+module CamTool
+  class EdgeCounter < ProvJSONParser
+    attr_reader :map
+
+    def initialize
+      @map = Hash.new(0)
+    end
+
+    def add key
+      @map[key]=@map[key]+1
+    end
+
+    def used k, v
+      self.add v['prov:type']
+    end
+
+    def wasGeneratedBy k, v
+      self.add v['prov:type']
+    end
+
+    def wasDerivedFrom k, v
+      self.add v['prov:type']
+    end
+
+    def wasInformedBy k, v
+      self.add v['prov:type']
+    end
+
+    def wasAssociatedWith k, v
+      self.add v['prov:type']
+    end
+
+    def show
+      puts "\n\n-------------\nEdge types\n-------------\n"
+      @map = @map.sort_by { |key, value| value }.reverse
+      @map.each do |key, value|
+        puts "#{key}:#{value}"
+      end
+    end
+  end
+end

data/lib/camtool/edge_validator.rb

@@ -0,0 +1,74 @@
+module CamTool
+  class EdgeValidator < ProvJSONParser
+    def initialize
+    end
+
+    def print_relation_bytes id
+      id = id.gsub 'cf:', ''
+      bytes = Base64.decode64(id).bytes.to_a
+      type = bytes[7] & 0xF0
+      puts type.to_s(16)
+      type = bytes[8]
+      puts type.to_s(16)
+    end
+
+    def is_relation? id
+      id = id.gsub 'cf:', ''
+      bytes = Base64.decode64(id).bytes.to_a
+      type = bytes[7] & 0xF0
+      return type == 0x80
+    end
+
+    def is_activity? id
+      id = id.gsub 'cf:', ''
+      bytes = Base64.decode64(id).bytes.to_a
+      type = bytes[7] & 0xF0
+      return type == 0x40
+    end
+
+    def is_entity? id
+      id = id.gsub 'cf:', ''
+      bytes = Base64.decode64(id).bytes.to_a
+      type = bytes[7] & 0xF0
+      return type == 0x20
+    end
+
+    def is_agent? id
+      id = id.gsub 'cf:', ''
+      bytes = Base64.decode64(id).bytes.to_a
+      type = bytes[7] & 0xF0
+      return type == 0x10
+    end
+
+    def used k, v
+      abort "used prov:entity is of wrong type:  #{v}." unless is_entity? v['prov:entity']
+      abort "used prov:activity is of wrong type: #{v}." unless is_activity? v['prov:activity']
+    end
+
+    def wasGeneratedBy k, v
+      abort "wasGeneratedBy prov:entity is of wrong type:  #{v}." unless is_entity? v['prov:entity']
+      abort "wasGeneratedBy prov:activity is of wrong type: #{v}." unless is_activity? v['prov:activity']
+    end
+
+    def wasDerivedFrom k, v
+      abort "wasDerivedFrom prov:usedEntity is of wrong type:  #{v}." unless is_entity? v['prov:usedEntity']
+      abort "wasDerivedFrom prov:generatedEntity is of wrong type: #{v}." unless is_entity? v['prov:generatedEntity']
+    end
+
+    def wasInformedBy k, v
+      abort "wasInformedBy prov:informant is of wrong type:  #{v}." unless is_activity? v['prov:informant']
+      abort "wasInformedBy prov:informed is of wrong type: #{v}." unless is_activity? v['prov:informed']
+    end
+
+    def wasAssociatedWith k, v
+      abort "wasAssociatedWith prov:activity is of wrong type:  #{v}." unless is_activity? v['prov:activity']
+      abort "wasAssociatedWith prov:agent is of wrong type: #{v}." unless is_agent? v['prov:agent']
+    end
+
+    def relation k, v
+      print_relation_bytes k
+      puts v
+      abort "relation should not be used"
+    end
+  end
+end

data/lib/camtool/log_breaker.rb

@@ -0,0 +1,32 @@
+module CamTool
+  class LogBreaker
+    def break_log_file filename
+      if filename != nil
+        open(filename) do |file|
+          @i=0
+          file.each_line do |line|
+            ok = true
+            line = line.sub /\[[0-9 :-]*\]\t[A-Z]*[ ]*:[ ]*/, ''
+            line = line.gsub "\n", ''
+            begin
+              JSON.parse(line)
+            rescue JSON::ParserError
+              abort "Error #{ex.class}, message: #{ex.message}"
+            end
+            self.new_entry line unless !ok
+          end
+        end unless !File.file?(filename)
+        print "File does not exist\n" unless File.file?(filename)
+      end
+      self
+    end
+
+    def new_entry entry
+      fname="./#{@i}.break"
+      @i=@i+1
+      File.open(fname, 'w+') do |f|
+        f.write entry
+      end
+    end
+  end
+end

data/lib/camtool/provjson_parser.rb

@@ -0,0 +1,104 @@
+require 'json'
+
+module CamTool
+  class ProvJSONParser
+    def read_json_file filename
+      if filename != nil
+        parse_json File.read(filename).gsub("\n", '') unless !File.file?(filename)
+        print "File does not exist\n" unless File.file?(filename)
+      end
+      self
+    end
+
+    def read_log_file filename
+      if filename != nil
+        open(filename) do |file|
+          file.each_line do |line|
+            parse_json line.gsub("\n", '')
+          end
+        end unless !File.file?(filename)
+        print "File does not exist\n" unless File.file?(filename)
+      end
+      self
+    end
+
+    def used k, v
+    end
+
+    def wasGeneratedBy k, v
+    end
+
+    def wasDerivedFrom k, v
+    end
+
+    def wasInformedBy k, v
+    end
+
+    def wasAssociatedWith k, v
+    end
+
+    def relation k, v
+    end
+
+    def prefix k, v
+    end
+
+    def entity k, v
+    end
+
+    def activity k, v
+    end
+
+    def agent k, v
+    end
+
+    def parse_json string
+      begin
+        json = JSON.parse(string)
+      rescue JSON::ParserError => ex
+        puts "Length: #{string.length}\n"
+        abort "Error #{ex.class}, message: #{ex.message}"
+      end
+
+      json['prefix'].each do |k, v|
+        self.prefix k, v
+      end unless !json.key? 'prefix'
+
+      json['entity'].each do |k, v|
+        self.entity k, v
+      end unless !json.key? 'entity'
+
+      json['activity'].each do |k, v|
+        self.activity k, v
+      end unless !json.key? 'activity'
+
+      json['agent'].each do |k, v|
+        self.activity k, v
+      end unless !json.key? 'agent'
+
+      json['used'].each do |k, v|
+        self.used k, v
+      end unless !json.key? 'used'
+
+      json['wasGeneratedBy'].each do |k, v|
+        self.wasGeneratedBy k, v
+      end unless !json.key? 'wasGeneratedBy'
+
+      json['wasDerivedFrom'].each do |k, v|
+        self.wasDerivedFrom k, v
+      end unless !json.key? 'wasDerivedFrom'
+
+      json['wasInformedBy'].each do |k, v|
+        self.wasInformedBy k, v
+      end unless !json.key? 'wasInformedBy'
+
+      json['relation'].each do |k, v|
+        self.relation k, v
+      end unless !json.key? 'relation'
+
+      json['wasAssociatedWith'].each do |k, v|
+        self.wasAssociatedWith k, v
+      end unless !json.key? 'wasAssociatedWith'
+    end
+  end
+end

data/lib/camtool/provjson_to_rgl.rb

@@ -0,0 +1,103 @@
+require 'rgl/adjacency'
+require 'rgl/dot'
+require 'rgl/topsort'
+require 'rgl/transitivity'
+
+module CamTool
+  class ProvJSONtoRGL < ProvJSONParser
+    attr_reader :dg
+    attr_reader :map
+    attr_reader :edges
+
+    def initialize
+      @dg = RGL::DirectedAdjacencyGraph.new # initialise the graph structure
+      @map = Hash.new
+      @edges = Hash.new
+    end
+
+    def entity k, v
+      @map[k] = v
+    end
+
+    def activity k, v
+      @map[k] = v
+    end
+
+    def agent k, v
+      @map[k] = v
+    end
+
+    def used k, v
+      @dg.add_edge v['prov:activity'], v['prov:entity']
+      @edges[v['prov:activity']+v['prov:entity']] = v
+    end
+
+    def wasGeneratedBy k, v
+      @dg.add_edge v['prov:entity'], v['prov:activity']
+      @edges[v['prov:entity']+v['prov:activity']] = v
+    end
+
+    def wasDerivedFrom k, v
+      @dg.add_edge v['prov:generatedEntity'], v['prov:usedEntity']
+      @edges[v['prov:generatedEntity']+v['prov:usedEntity']] = v
+    end
+
+    def wasInformedBy k, v
+      @dg.add_edge v['prov:informed'], v['prov:informant']
+      @edges[v['prov:informed']+v['prov:informant']] = v
+    end
+
+    def wasAssociatedWith k, v
+      @dg.add_edge v['prov:activity'], v['prov:agent']
+      @edges[v['prov:activity']+v['prov:agent']] = v
+      @dg.add_edge v['prov:agent'], v['prov:plan'] unless !v.key? 'prov:plan'
+    end
+
+    def information
+      str = "Graph:\n"
+      if @dg.directed?
+        str += "directed\n"
+      else
+        str += "not directed\n"
+      end
+
+      if @dg.acyclic?
+        str += "acyclic\n"
+      else
+        str += "not acyclic\n"
+      end
+
+      str += @dg.num_edges.to_s() +" edges.\n"
+      str += @dg.num_vertices.to_s() +" vertices.\n"
+      str += (@dg.num_edges.to_f/@dg.num_vertices).to_s() + " edges/vertices ratio.\n"
+    end
+
+    def validate
+      abort 'Graph is not directed!' unless @dg.directed?
+      abort 'Graph is not acyclic!' unless @dg.acyclic?
+    end
+
+    def svg
+      @dg.write_to_graphic_file('svg')
+    end
+
+    def jpg
+      @dg.write_to_graphic_file('jpg')
+    end
+
+    def png
+      @dg.write_to_graphic_file('png')
+    end
+
+    def list types
+      a = @dg.topsort_iterator.to_a
+      a_type = types.split(',')
+      a.each do |v|
+        if a_type.include?(@map[v]['prov:type']) || types.nil?
+          puts v + ' ' + @map[v]['prov:label'] unless @map[v]['prov:label'].nil?
+          puts v + ' no label' unless !@map[v]['prov:label'].nil?
+        end
+      end
+    end
+  end
+end

data/lib/camtool/version.rb

@@ -0,0 +1,7 @@
+module CamTool
+  VERSION = "0.1.0"
+
+  def self.version
+    puts VERSION
+  end
+end

data/lib/camtool/vertex_counter.rb

@@ -0,0 +1,33 @@
+module CamTool
+  class VertexCounter < ProvJSONParser
+    attr_reader :map
+
+    def initialize
+      @map = Hash.new(0)
+    end
+
+    def add key
+      @map[key]=@map[key]+1
+    end
+
+    def entity k, v
+      self.add v['prov:type']
+    end
+
+    def activity k, v
+      self.add v['prov:type']
+    end
+
+    def agent k, v
+      self.add v['prov:type']
+    end
+
+    def show
+      puts "\n\n-------------\nVertex types\n-------------\n"
+      @map = @map.sort_by { |key, value| value }.reverse
+      @map.each do |key, value|
+        puts "#{key}:#{value}"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: camtool
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Thomas Pasquier
+- Xuehuan "Michael" Han
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rgl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: rinruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: mqtt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Allow to test, publish and manipulate camflow log files.
+email:
+- tfjmp@seas.harvard.edu
+executables:
+- camtool
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/camtool
+- lib/camtool.rb
+- lib/camtool/breaker_mqtt.rb
+- lib/camtool/check_packet.rb
+- lib/camtool/edge_counter.rb
+- lib/camtool/edge_validator.rb
+- lib/camtool/log_breaker.rb
+- lib/camtool/provjson_parser.rb
+- lib/camtool/provjson_to_rgl.rb
+- lib/camtool/version.rb
+- lib/camtool/vertex_counter.rb
+homepage: https://github.com/tfjmp/camtool
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.2
+signing_key: 
+specification_version: 4
+summary: Tool to manipulate CamFlow log files.
+test_files: []