encapsulator 0.1.0

data/README.md

@@ -0,0 +1,50 @@
+# Encapsulator
+
+[![DOI](https://zenodo.org/badge/94570522.svg)](https://zenodo.org/badge/latestdoi/94570522)
+
+## Installation
+
+### Fedora 25
+
+```
+sudo dnf install ruby
+gem install json rgl rake bundler
+git clone https://github.com/tfjmp/encapsulator.git
+cd encapsulator
+rake install
+encapsulator --install fedora
+encapsulator -h
+```
+
+### Ubuntu
+
+```
+sudo apt install ruby
+gem install json rgl rake bundler
+git clone https://github.com/tfjmp/encapsulator.git
+cd encapsulator
+rake install
+encapsulator --install ubuntu
+encapsulator -h
+```
+
+### MacOS
+
+```
+brew install ruby
+gem install json rgl rake bundler
+git clone https://github.com/tfjmp/encapsulator.git
+cd encapsulator
+rake install
+encapsulator --install mac
+encapsulator -h
+```
+
+## USAGE
+
+```
+# extract info from ddg.json
+encapsulator --info <your_RDT_prov.json>
+# select an output node to generate the code to be generate
+encapsulator --code <your_RDT_prov.json> <output_node_name> <new R script>
+```

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "encapsulator"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/encapsulator

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require "encapsulator"
+
+if ARGV[0] == '--info'
+	Encapsulator::CLI::info ARGV[1]
+elsif ARGV[0] == '--encapsulate'
+	system 'rm', '-rf', '.'+ARGV[1]
+	out_path = '.'+ARGV[1]+'/workspace'
+	system 'mkdir', '-p', out_path
+	for i in 3..ARGV.size
+		break unless !ARGV[i].nil?
+		out_name = '.'+ARGV[1]+'/workspace/'+ARGV[i]+'.R'
+		Encapsulator::CLI::source_code ARGV[2], ARGV[i], out_name
+    Encapsulator::ProvR.tidy out_name
+		Encapsulator::CLI::script_inputs ARGV[2], ARGV[i], out_path
+		puts "Generated script for output #{ARGV[i]}"
+	end
+	Encapsulator::CLI::encapsulate ARGV[1], ARGV[2]
+	puts 'WARNING: We are sorry due to remote API change, capsule need to be manually uploaded to: https://app.vagrantup.com'
+elsif ARGV[0] == '--decapsulate'
+  decapsulate ARGV[1]
+elsif ARGV[0] == '--jpg'
+  Encapsulator::CLI::get_jpg ARGV[1]
+elsif ARGV[0] == '--png'
+  Encapsulator::CLI::get_png ARGV[1]
+elsif ARGV[0] == '--svg'
+  Encapsulator::CLI::get_svg ARGV[1]
+elsif ARGV[0] == '--code'
+	out_path = '.encapsulator'
+	system 'rm', '-rf', out_path
+	system 'mkdir', '-p', out_path
+	for i in 2..ARGV.size
+		break unless !ARGV[i].nil?
+		out_name = out_path+'/'+ARGV[i]+'.R'
+		Encapsulator::CLI::source_code ARGV[1], ARGV[i], out_name
+    Encapsulator::ProvR.tidy out_name
+		Encapsulator::CLI::script_inputs ARGV[1], ARGV[i], out_path
+		puts "Generated script for output #{ARGV[i]}"
+	end
+	puts 'Scripts written in '+File.expand_path(out_path)
+elsif ARGV[0] == '--run'
+	Encapsulator::ProvR::run ARGV[1]
+elsif ARGV[0] == '--version'
+	puts Encapsulator::VERSION
+elsif ARGV[0] == '--install'
+	if ARGV[1] == 'fedora'
+    Encapsulator::Installer.install_fedora
+	elsif ARGV[1] == 'ubuntu'
+		Encapsulator::Installer::install_ubuntu
+	elsif ARGV[1] == 'mac'
+		Encapsulator::Installer::install_mac
+  end
+else
+  Encapsulator::CLI::usage
+end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/encapsulator.rb

@@ -0,0 +1,6 @@
+require 'encapsulator/version'
+require 'encapsulator/provjson_parser'
+require 'encapsulator/rjson_parser'
+require 'encapsulator/installer'
+require 'encapsulator/provr'
+require 'encapsulator/cli'

data/lib/encapsulator/cli.rb

@@ -0,0 +1,131 @@
+module Encapsulator
+  module CLI
+    def CLI.usage
+      puts "USAGE\n\n"
+      puts '--info <path to R script> display information about the provenance graph'
+      puts '--jpg <path to R script> output graph as jpg'
+      puts '--png <path to R script> output graph as png'
+      puts '--svg <path to R script> output graph as svg'
+      puts '--code <path to R script> <output> [output] ... [output] create the source necessary to generate the specified output'
+      puts '--run <path to R script> run the provided r script'
+    	puts '--encapsulate <user>/<project> <script> <output> [output] ... [output] create the specified capsule'
+    	puts '--decapsulate <user>/<project> download the coresponding capsule'
+    end
+
+    def CLI.vagrant vm_name, install_instructions
+      provision = Array.new
+      provision << '# -*- mode: ruby -*-'
+      provision << '# vi: set ft=ruby :'
+      provision << 'Vagrant.configure(2) do |config|'
+      provision << "  "+'config.vm.box = "jhcook/fedora25"'
+      provision << "  "+'config.vm.provider "virtualbox" do |vb|'
+      provision << "  "+'vb.gui = true'
+      provision << "  "+'vb.memory = 2048'
+      provision << "  "+'vb.customize ["modifyvm", :id, "--cpuexecutioncap", "70"]'
+      provision << "  "+'vb.cpus = 2'
+      provision << "  "+"vb.name = \"#{vm_name.gsub '/', '_'}\""
+      provision << 'end'
+      provision << 'config.vm.provision "shell", inline: <<-SHELL'
+    	provision << "  "+'cp -a /vagrant/workspace/. /home/vagrant/Documents'
+      provision << "  "+'sudo dnf -y -v install openssl-devel libxml2-devel'
+      provision << "  "+'sudo dnf -y -v install libcurl libcurl-devel'
+      provision << "  "+'sudo dnf -y -v install perl-CPAN'
+      provision << "  "+'sudo dnf -y -v install ruby'
+      provision << "  "+'wget https://atom.io/download/rpm'
+      provision << "  "+'mv ./rpm ./atom.rpm'
+      provision << "  "+'sudo dnf -y -v install ./atom.rpm'
+      provision << "  "+'rm -rf ./atom.rpm'
+      provision << install_instructions
+    	provision << " "+'wget https://download1.rstudio.org/rstudio-1.0.143-x86_64.rpm'
+    	provision << "  "+'sudo dnf -y -v install ./rstudio-1.0.143-x86_64.rpm'
+    	provision << "  "+'rm -rf ./rstudio-1.0.143-x86_64.rpm'
+      provision << 'SHELL'
+      provision << 'end'
+    end
+
+    def CLI.encapsulate capsule_name, script
+    	ProvR.run_script script do
+    		v = Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json')
+    		system 'mkdir', '-p', '../.'+capsule_name
+    		Dir.chdir '../.'+capsule_name do
+    			puts "In directory #{Dir.pwd}..."
+    			File.open("Vagrantfile", "w+") do |f|
+    				puts 'Encapsulator will attempt to install the following packages:'
+    				v.packages_show
+    				script = CLI.vagrant capsule_name, v.install
+    	  		f.puts(script)
+    			end
+    			puts 'Provision script is ready...'
+    			puts 'getting your capsule ready, be patient...'
+    			system("vagrant", "up", out: $stdout, err: $stdout)
+    			system("vagrant", "halt", out: $stdout, err: $stdout)
+    		end
+    		puts 'Your capsule should be visible in the virtualbox interface.'
+    	end
+    end
+
+    def CLI.decapsulate capsule_name
+    	puts "looking for the capsule #{vm}..."
+    	system("vagrant", "init", capsule_name, out: $stdout, err: $stdout)
+    	puts 'getting your capsule ready, be patient...'
+    	system("vagrant", "up", out: $stdout, err: $stdout)
+    	system("vagrant", "halt", out: $stdout, err: $stdout)
+    	puts 'you should find your capsule in the virtualbox GUI.'
+    end
+
+    def CLI.source_code script, target_output, destination
+    	ProvR.run_script script do
+    		script = Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').script target_output
+    	end
+    	if script.nil?
+    		abort "#{target_output} is not a valid output file."
+    	else
+    		File.open(destination, "w+") do |f|
+    			f.puts(script)
+    		end
+        ProvR.tidy destination
+    		puts "Script written to #{destination}"
+    	end
+    end
+
+    def CLI.get_jpg script
+      ProvR.run_script script do
+    		Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').jpg
+    	end
+    end
+
+    def CLI.get_png script
+      ProvR.run_script script do
+    		Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').png
+    	end
+    end
+
+    def CLI.get_svg script
+      ProvR.run_script script do
+    		Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').svg
+    	end
+    end
+
+    def CLI.info script
+      ProvR.run_script script do
+      	Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').show
+    	end
+    end
+
+    def CLI.script_inputs script, target_output, destination
+    	inputs = nil
+    	destination = '../'+destination
+    	ProvR.run_script script do
+    		inputs = Encapsulator::RJSONParser.new('../'+script).read_json_file('ddg.json').script_inputs target_output
+    		inputs.each do |path, file|
+    			next unless !file.include? destination
+    			dest_path = path.gsub '..', destination
+    			dest = file.gsub '..', destination
+    			system 'mkdir', '-p', dest_path
+    			system 'cp', '-f', file, dest
+    			puts "Saved input #{dest} for script #{target_output}.R"
+    		end
+    	end
+    end
+  end
+end

data/lib/encapsulator/installer.rb

@@ -0,0 +1,20 @@
+module Encapsulator
+  class Installer
+    def self.install_fedora
+      system 'wget', 'http://download.virtualbox.org/virtualbox/5.1.22/VirtualBox-5.1-5.1.22_115126_fedora25-1.x86_64.rpm', out: $stdout, err: $stdout
+      system 'sudo', 'dnf', '-y', '-v', 'install', './VirtualBox-5.1-5.1.22_115126_fedora25-1.x86_64.rpm', out: $stdout, err: $stdout
+      system 'sudo', 'dnf', '-y', '-v', 'install', 'vagrant', out: $stdout, err: $stdout
+    end
+
+    def self.install_ubuntu
+    	system 'sudo', 'apt', '-y', 'install', 'virtualbox', 'virtualbox-ext-pack', out: $stdout, err: $stdout
+    	system 'sudo', 'apt', '-y', 'install', 'vagrant', out: $stdout, err: $stdout
+    end
+
+    def self.install_mac
+    	system 'brew', 'cask', 'install', 'virtualbox', out: $stdout, err: $stdout
+    	system 'brew', 'cask', 'install', 'vagrant', out: $stdout, err: $stdout
+    	system 'brew', 'cask', 'install', 'vagrant-manager', out: $stdout, err: $stdout
+    end
+  end
+end

data/lib/encapsulator/provjson_parser.rb

@@ -0,0 +1,100 @@
+require 'json'
+
+module Encapsulator
+  class ProvJSONParser
+		attr_reader :filename
+
+    def read_json_file filename
+      if filename != nil
+				@filename=filename
+        parse_json File.read(filename) unless !File.file?(filename)
+        print "File does not exist\n" unless File.file?(filename)
+      end
+      self
+    end
+
+    def read_log_file filename
+      if filename != nil
+        open(filename) do |file|
+          file.each_line do |line|
+            parse_json line
+          end
+        end unless !File.file?(filename)
+        print "File does not exist\n" unless File.file?(filename)
+      end
+      self
+    end
+
+    def used k, v
+    end
+
+    def wasGeneratedBy k, v
+    end
+
+    def wasDerivedFrom k, v
+    end
+
+    def wasInformedBy k, v
+    end
+
+    def wasAssociatedWith k, v
+    end
+
+    def prefix k, v
+    end
+
+    def entity k, v
+    end
+
+    def activity k, v
+    end
+
+    def agent k, v
+    end
+
+    def parse_json string
+      begin
+        json = JSON.parse(string)
+      rescue JSON::ParserError
+        puts 'Failed parsing.'
+        abort
+      end
+
+      json['prefix'].each do |k, v|
+        self.prefix k, v
+      end unless !json.key? 'prefix'
+
+      json['entity'].each do |k, v|
+        self.entity k, v
+      end unless !json.key? 'entity'
+
+      json['activity'].each do |k, v|
+        self.activity k, v
+      end unless !json.key? 'activity'
+
+      json['agent'].each do |k, v|
+        self.activity k, v
+      end unless !json.key? 'agent'
+
+      json['used'].each do |k, v|
+        self.used k, v
+      end unless !json.key? 'used'
+
+      json['wasGeneratedBy'].each do |k, v|
+        self.wasGeneratedBy k, v
+      end unless !json.key? 'wasGeneratedBy'
+
+      json['wasDerivedFrom'].each do |k, v|
+        self.wasDerivedFrom k, v
+      end unless !json.key? 'wasDerivedFrom'
+
+      json['wasInformedBy'].each do |k, v|
+        self.wasInformedBy k, v
+      end unless !json.key? 'wasInformedBy'
+
+      json['wasAssociatedWith'].each do |k, v|
+        self.wasAssociatedWith k, v
+      end unless !json.key? 'wasAssociatedWith'
+    end
+  end
+end

data/lib/encapsulator/provr.rb

@@ -0,0 +1,33 @@
+module Encapsulator
+  module ProvR
+    def ProvR.run script
+    	require 'rinruby'
+      r = RinRuby.new(echo: false)
+      r.eval "install.packages('devtools')"
+    	r.eval "require('devtools')"
+    	r.eval "install_github('provtools/provR', ref='dev')"
+    	r.eval "require('provR')"
+      r.eval "prov.capture('#{script}', save=TRUE)"
+    end
+
+    def ProvR.tidy script
+      require 'rinruby'
+      r = RinRuby.new(echo: false)
+    	r.eval "install.packages('formatR', repos = 'http://cran.rstudio.com')"
+      r.eval "require('formatR')"
+      r.eval "tidy_file('#{script}')"
+    end
+
+    $ran_script=false
+    def ProvR.run_script script
+    	if !$ran_script
+    		run script
+    		$ran_script = true
+    	end
+    	prov_folder = script.gsub('.R', '_ddg')
+    	Dir.chdir prov_folder do
+    		yield
+    	end
+    end
+  end
+end

data/lib/encapsulator/rjson_parser.rb

@@ -0,0 +1,233 @@
+require 'find'
+require 'rgl/adjacency'
+require 'rgl/dot'
+require 'rgl/topsort'
+require 'rgl/transitivity'
+require 'rgl/traversal'
+
+module Encapsulator
+  class RJSONParser < ProvJSONParser
+    attr_reader :dg
+    attr_reader :map
+    attr_reader :files
+    attr_reader :instructions
+    attr_reader :libraries
+    attr_reader :packages
+    attr_reader :file
+
+    def initialize r_file
+      @dg = RGL::DirectedAdjacencyGraph.new # initialise the graph structure
+      @map = Hash.new(0)
+      @files = Hash.new
+      @copies = Hash.new
+      @instructions = Hash.new
+      @libraries = Hash.new
+      @packages = Hash.new
+      @script = IO.readlines(r_file)
+    end
+
+    def get_operation start_line, end_line
+      start_line-=1
+      end_line-=1
+      operation = ''
+      for i in start_line..end_line
+        operation += @script[i] unless @script[i].nil?
+      end
+      return operation
+    end
+
+    def add key
+      @map[key]=@map[key]+1
+    end
+
+    def used k, v
+      @dg.add_edge v['prov:activity'], v['prov:entity']
+    end
+
+    def wasGeneratedBy k, v
+      @dg.add_edge v['prov:entity'], v['prov:activity']
+    end
+
+    def wasDerivedFrom k, v
+      @dg.add_edge v['prov:generatedEntity'], v['prov:usedEntity']
+    end
+
+    def wasInformedBy k, v
+      #@dg.add_edge v['prov:informed'], v['prov:informant']
+    end
+
+    def wasAssociatedWith k, v
+      @dg.add_edge v['prov:activity'], v['prov:agent']
+      @dg.add_edge v['prov:agent'], v['prov:plan'] unless !v.key? 'prov:plan'
+    end
+
+    def entity k, v
+      self.add v['rdt:type']
+      if v['rdt:type'] == 'File'
+        @files[k]=v['rdt:name']
+      end
+    end
+
+    def activity k, v
+      self.add v['rdt:type']
+      if v['rdt:type'] == 'Operation'
+        @instructions[k]=get_operation(v['rdt:startLine'].to_i, v['rdt:endLine'].to_i)
+      end
+      if /(library|require)\(('|")[a-zA-Z]+('|")/.match v['rdt:name']
+        @libraries[k]=v['rdt:name']
+      end
+      if k == 'environment'
+        v['rdt:installedPackages'].each do |l|
+          packages[l['package']]=l['version']
+        end
+      end
+    end
+
+    def agent k, v
+      self.add v['rdt:type']
+    end
+
+    def information
+      str = "\n------\n"
+      str += "Graph:\n"
+      str += "------\n"
+      if @dg.directed?
+        str += "directed\n"
+      else
+        str += "not directed\n"
+      end
+
+      if @dg.acyclic?
+        str += "acyclic\n"
+      else
+        str += "not acyclic\n"
+      end
+
+      str += @dg.num_edges.to_s() +" edges.\n"
+      str += @dg.num_vertices.to_s() +" vertices.\n"
+      str += (@dg.num_edges.to_f/@dg.num_vertices).to_s() + " edges/vertices ratio.\n"
+      puts str
+    end
+
+    def svg
+      @dg.write_to_graphic_file('svg')
+    end
+
+    def jpg
+      @dg.write_to_graphic_file('jpg')
+    end
+
+    def png
+      @dg.write_to_graphic_file('png')
+    end
+
+		def get_list file
+			id = ''
+			@files.each do |k, v|
+				if v == file
+					id = k
+					break
+				end
+			end
+			if id==''
+				return Array.new
+			end
+      tree = @dg.bfs_search_tree_from(id)
+      g = @dg.vertices_filtered_by {|v| tree.has_vertex? v}
+      list = g.vertices
+		end
+
+    def source_code file
+			statements = Array.new
+      list = get_list file
+      list.delete_if { |v| !v.include?('p') }
+      list = list.sort_by{ |m| m.tr('p', '').to_i }
+      list.each do |v|
+				next unless !@instructions[v].nil?
+        statements << @instructions[v]
+      end
+      return statements
+    end
+
+		def script_inputs file
+			inputs = Hash.new
+			list = get_list file
+			list.delete_if { |v| !is_input_with_id?(v) }
+			list.each do |v|
+				next unless !@files[v].nil?
+				Find.find '..' do |path|
+					if path.include? '/'+@files[v]
+						inputs[path.gsub @files[v], '']= path
+					end
+				end
+      end
+			return inputs
+		end
+
+    def script output
+      script = Array.new
+      @libraries.each do |k, v|
+        script << v
+      end
+      statements = source_code output
+      script << statements unless statements.empty?
+    end
+
+    def show
+      file_show
+			puts "\n\n"
+      packages_show
+    end
+
+    def file_show
+      puts 'Files'
+			puts '-----'
+      @files.each do |key, value|
+        if is_input? value
+					puts "Input #{value}"
+				else
+					puts "Output #{value}"
+				end
+      end
+    end
+
+		def packages_show
+      puts 'Packages'
+			puts '--------'
+			@packages.each do |key, value|
+				if !['datasets', 'utils', 'graphics', 'grDevices', 'methods', 'stats', 'provR', 'devtools'].include?(key)
+        	puts "#{key} v#{value}"
+        end
+			end
+		end
+
+    def is_input? file_name
+			source_code(file_name).empty?
+		end
+
+		def is_input_with_id? id
+			@files.each do |key, value|
+        if id == key
+					if is_input? value
+						return true
+					end
+				end
+      end
+			return false
+		end
+
+    def install
+      instructions = Array.new
+      @packages.each do |key, value|
+        if key == 'base'
+          instructions << "  sudo dnf -y -v install R-#{value}"
+	        instructions << "  sudo dnf -y -v install R"
+          instructions << "  sudo su - -c \"R -e \\\\\\\"install.packages(\'devtools\', repos=\'http://cran.rstudio.com/\', dependencies = TRUE)\\\\\\\"\""
+        elsif !['datasets', 'utils', 'graphics', 'grDevices', 'methods', 'stats', 'provR', 'devtools'].include?(key)
+          instructions << "  sudo su - -c \"R -e \\\\\\\"require('devtools');install_version(\'#{key}\', version=\'#{value}\', repos=\'http://cran.rstudio.com/\')\\\\\\\"\""
+        end
+      end
+      return instructions
+    end
+  end
+end