thanthus 0.2.3

data/lib/xanthus/dataverse.rb

@@ -0,0 +1,179 @@
+require 'fileutils'
+require 'json'
+
+module Xanthus
+  class Dataverse < Repository
+    attr_accessor :server
+    attr_accessor :repo
+    attr_accessor :token
+    attr_accessor :dataset_name
+    attr_accessor :author
+    attr_accessor :affiliation
+    attr_accessor :email
+    attr_accessor :description
+    attr_accessor :subject
+    attr_accessor :doi
+
+    def initialize
+      super
+      @server = 'default_server'
+      @repo = 'default_repo'
+      @token = 'default_token'
+      @dataset_name = 'default_name'
+      @author = 'default_author'
+      @affiliation = 'default_affiliation'
+      @email = 'default_email'
+      @description = 'default_description'
+      @subject = 'default_subject'
+      @doi = 'not_set'
+    end
+
+    def dataset_json
+json = %Q{
+  {
+    "datasetVersion": {
+      "metadataBlocks": {
+        "citation": {
+          "fields": [
+            {
+              "value": "#{@dataset_name}",
+              "typeClass": "primitive",
+              "multiple": false,
+              "typeName": "title"
+            },
+            {
+              "value": [
+                {
+                  "authorName": {
+                    "value": "#{@author}",
+                    "typeClass": "primitive",
+                    "multiple": false,
+                    "typeName": "authorName"
+                  },
+                  "authorAffiliation": {
+                    "value": "#{@affiliation}",
+                    "typeClass": "primitive",
+                    "multiple": false,
+                    "typeName": "authorAffiliation"
+                  }
+                }
+              ],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "author"
+            },
+            {
+              "value": [
+                  { "datasetContactEmail" : {
+                      "typeClass": "primitive",
+                      "multiple": false,
+                      "typeName": "datasetContactEmail",
+                      "value" : "#{@email}"
+                  },
+                  "datasetContactName" : {
+                      "typeClass": "primitive",
+                      "multiple": false,
+                      "typeName": "datasetContactName",
+                      "value": "#{@author}"
+                  }
+              }],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "datasetContact"
+            },
+            {
+              "value": [ {
+                 "dsDescriptionValue":{
+                  "value": "#{@description.gsub(/\r/," ").gsub(/\n/," ")}",
+                  "multiple":false,
+                 "typeClass": "primitive",
+                 "typeName": "dsDescriptionValue"
+              }}],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "dsDescription"
+            },
+            {
+              "value": [
+                "#{@subject}"
+              ],
+              "typeClass": "controlledVocabulary",
+              "multiple": true,
+              "typeName": "subject"
+            }
+          ],
+          "displayName": "Citation Metadata"
+        }
+      }
+    }
+  }
+}
+      return json
+    end
+
+    def create_dataset
+      Dir.chdir 'dataverse_dataset' do
+        File.open('dataset.json', 'w+') do |f|
+          f.write(self.dataset_json)
+        end
+        puts "Creating dataverse #{@dataset_name} in #{@repo} at #{@server}..."
+        output = `curl --speed-time 15 --speed-limit 1000 --retry 50 --retry-max-time 0 -H X-Dataverse-key:#{@token} -X POST #{@server}/api/dataverses/#{@repo}/datasets --upload-file dataset.json`
+        puts output # needed to escape curl output
+        parsed = JSON.parse(output)
+        @doi = parsed['data']['persistentId']
+        puts "Dataverse #{@doi} created."
+      end
+    end
+
+    def init config
+      # initialize with config information
+      @author = config.authors
+      @affiliation = config.affiliation
+      @email = config.email
+      @description = config.description
+      @dataset_name = config.name+'-'+Time.now.strftime("%Y-%m-%d_%H-%M")
+
+      FileUtils.mkdir_p 'dataverse_dataset'
+      self.create_dataset
+      Dir.chdir 'dataverse_dataset' do
+        FileUtils.mkdir_p 'repo'
+        Dir.chdir 'repo' do
+          self.xanthus_file
+          self.readme_file config
+          self.inputs_file config
+        end
+      end
+    end
+
+    def add_file_to_dataverse name, description, folder
+      output = `curl --speed-time 15 --speed-limit 1000 --retry 50 --retry-max-time 0 -H X-Dataverse-key:#{@token} -X POST -F "file=@#{name}" -F 'jsonData={"description":"#{description}","directoryLabel":"#{folder}","categories":["Data"], "restrict":"false"}' "#{@server}/api/datasets/:persistentId/add?persistentId=#{@doi}"`
+      puts output
+    end
+
+    def xanthus_file
+      self.prepare_xanthus_file
+      self.add_file_to_dataverse '.xanthus', 'xanthus file used to generate the data.', 'metadata'
+    end
+
+    def readme_file config
+      self.prepare_readme_file config
+      self.add_file_to_dataverse 'README.md', 'readme describing the dataset.', 'metadata'
+    end
+
+    def inputs_file config
+      config.jobs.each do |name,job|
+        job.inputs.each do |k, files|
+          files.each do |file|
+            system('cp', '-f', "../../#{file}", "#{file}")
+            self.add_file_to_dataverse file, 'Job input file.', 'metadata'
+          end
+        end
+      end
+    end
+
+    def add content
+      self.add_file_to_dataverse content, "#{content} is a Xanthus generated file (check metadata for description)", 'data'
+    end
+
+  end
+end

data/lib/xanthus/default.rb

@@ -0,0 +1,21 @@
+module Xanthus
+CAMFLOW_START = %q{%{
+camflow -a true
+sleep 1
+}}
+
+CAMFLOW_STOP = %q{%{
+camflow -a false
+sleep 20
+}}
+
+SPADE_START = %q{%{
+echo spade | sudo -H -u spade ../SPADE/bin/spade start
+sleep 20
+}}
+
+SPADE_STOP = %q{%{
+echo spade | sudo -H -u spade ../SPADE/bin/spade stop
+sleep 20
+}}
+end

data/lib/xanthus/github.rb

@@ -0,0 +1,90 @@
+require 'fileutils'
+
+module Xanthus
+  class GitHub < Repository
+    attr_accessor :repo
+    attr_accessor :token
+    attr_accessor :folder
+
+    def initialize
+      super
+      @repo = ''
+      @token = ''
+      @folder = Time.now.strftime("%Y-%m-%d_%H-%M")
+    end
+
+    def lfs
+      system('git', 'lfs', 'install')
+      system('git', 'lfs', 'track', '*.tar.gz')
+      system('git', 'add', '.gitattributes')
+      system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
+    end
+
+    def xanthus_file
+      self.prepare_xanthus_file
+      system('git', 'add', '.xanthus')
+      system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/.xanthus :horse:")
+      system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
+    end
+
+    def readme_file config
+      self.prepare_readme_file config
+      system('git', 'add', 'README.md')
+      system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/README.md :horse:")
+      system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
+    end
+
+    def inputs_file config
+      config.jobs.each do |name,job|
+        job.inputs.each do |k, files|
+          files.each do |file|
+            system('cp', '-f', "../../#{file}", "#{file}")
+            system('git', 'add', "#{file}")
+            system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/#{file} :horse:")
+            system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
+          end
+        end
+      end
+    end
+
+    def init config
+      system('git', 'clone', "https://#{@token}@github.com/#{@repo}", 'repo')
+      Dir.chdir 'repo' do
+        self.lfs
+        FileUtils.mkdir_p @folder
+        Dir.chdir @folder do
+          self.xanthus_file
+          self.readme_file config
+          self.inputs_file config
+        end
+      end
+    end
+
+    def add content
+      Dir.chdir 'repo' do
+        FileUtils.mkdir_p @folder
+        system('mv', "../#{content}", "#{@folder}/#{content}")
+        system('git', 'add', "#{@folder}/#{content}")
+        system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/#{content} :horse:")
+      end
+    end
+
+    def push
+      Dir.chdir 'repo' do
+        system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
+        system('rm', '-rf', @folder)
+      end
+    end
+
+    def tag
+      Dir.chdir 'repo' do
+        system('git', 'tag', '-a', "xanthus-#{@folder}", '-m', '"Xanthus automated dataset generation."')
+        system('git', 'push', '--tags', "https://#{@token}@github.com/#{@repo}")
+      end
+    end
+
+    def clean
+      system('rm', '-rf', 'repo')
+    end
+  end
+end

data/lib/xanthus/init.rb

@@ -0,0 +1,156 @@
+require 'fileutils'
+
+module Xanthus
+  class Init
+    @@name
+
+    def self.header file
+      file.write("# -*- mode: ruby -*-\n")
+      file.write("# vi: set ft=ruby\n\n")
+    end
+
+    def self.config file
+      script = %Q{
+# -*- mode: ruby -*-
+# vi: set ft=ruby
+
+Xanthus.configure do |config|
+  config.name = '#{@@name}'
+  config.authors = 'John Doe'
+  config.affiliation = 'Somewhere University'
+  config.email = 'john.doe@somewhere.edu'
+  config.description = %q{
+Describe my super experiment.
+
+It is very cool and interesting!
+}
+  config.seed = #{Random.new_seed}
+
+  config.script :pre do
+    %q{%{
+      mkdir wgets
+      cd wgets
+    }}
+  end
+
+  config.script :camflow_start do
+    Xanthus::CAMFLOW_START
+  end
+
+  config.script :spade_start do
+    Xanthus::SPADE_START
+  end
+
+  config.script :normal do
+    %q{
+    2.times.collect do
+      'wget http://www.google.com'
+    end
+    }
+  end
+
+  config.script :attack do
+    %q{
+    2.times.collect do
+      'wget http://www.google.com'
+    end
+    }
+  end
+
+  config.script :camflow_stop do
+    Xanthus::CAMFLOW_STOP
+  end
+
+  config.script :spade_stop do
+    Xanthus::SPADE_STOP
+  end
+
+  config.script :post do
+    %q{%{
+      cd ..
+      rm -rf wgets
+    }}
+  end
+
+  config.script :server do
+    %q{%{
+      mkdir test
+    }}
+  end
+
+  config.vm :camflow do |vm|
+    vm.box = 'michaelh/ubuncam'
+    vm.version = '0.0.3'
+    vm.ip = '192.168.33.8'
+  end
+
+  config.vm :spade do |vm|
+    vm.box = 'michaelh/spade'
+    vm.memory = 8192
+    vm.version = '0.0.3'
+    vm.ip = '192.168.33.8'
+  end
+
+  config.vm :server do |vm|
+    vm.box = 'bento/ubuntu-18.04'
+    vm.version = '201812.27.0'
+    vm.ip = '192.168.33.3'
+  end
+
+  config.job :normal_camflow do |job|
+    job.iterations = 2
+    job.tasks = {camflow: [:pre, :camflow_start, :normal, :camflow_stop, :post]}
+    job.outputs = {camflow: {config: '/etc/camflow.ini', trace: '/tmp/audit.log'}}
+  end
+
+  config.job :attack_camflow do |job|
+    job.iterations = 2
+    job.tasks = {server: [:server], camflow: [:pre, :camflow_start, :attack, :camflow_stop, :post]}
+    job.outputs = {camflow: {config: '/etc/camflow.ini', trace: '/tmp/audit.log'}}
+  end
+
+  config.job :normal_spade do |job|
+    job.iterations = 2
+    job.tasks = {spade: [:pre, :spade_start, :normal, :spade_stop, :post]}
+    job.outputs = {spade: {trace: '/tmp/audit_cdm.avro'}}
+  end
+
+  config.job :attack_spade do |job|
+    job.iterations = 2
+    job.tasks = {server: [:server], spade: [:pre, :spade_start, :attack, :spade_stop, :post]}
+    job.outputs = {spade: {trace: '/tmp/audit_cdm.avro'}}
+  end
+
+  config.dataverse do |dataverse|
+    dataverse.server = <ADD DATAVERSE BASE URL>
+    dataverse.repo = <PROVIDE DATAVERSE NAME>
+    dataverse.token = <PROVIDE DATAVERSE TOKEN>
+    dataverse.subject = <PROVIDE DATAVERSE SUBJECT (e.g. engineering)>
+  end
+
+  # config.github do |github|
+  #   github.repo = '<ADD GITHUB REPO user/name>'
+  #   github.token = '<ADD GITHUB TOKEN>'
+  # end
+end
+}
+      file.write(script)
+    end
+
+    def self.init name
+      @@name = name
+      abort("Error: #{@@name} already exists.") unless !File.exists? name
+      FileUtils.mkdir_p @@name
+      Dir.chdir @@name do
+        puts "Creating experiment #{@@name}..."
+        File.open('.xanthus', 'w+') do |f|
+          self.header f
+          self.config f
+        end
+      end
+      puts 'Experiment created.'
+      puts "Edit #{@@name}/.xanthus to configure your experiment."
+      puts 'To run your experiment "xanthus run".'
+    end
+  end
+end

data/lib/xanthus/job.rb

@@ -0,0 +1,135 @@
+require 'fileutils'
+require_relative '../utils/os'
+
+module Xanthus
+  class Job
+    attr_accessor :name
+    attr_accessor :iterations
+    attr_accessor :tasks
+    attr_accessor :outputs
+    attr_accessor :inputs
+    attr_accessor :pre_instructions
+    attr_accessor :post_instructions
+
+    def initialize
+      @iterations = 0
+      @tasks = Hash.new
+      @outputs = Hash.new
+      @inputs = Hash.new
+      @pre_instructions = nil
+      @post_instructions = nil
+    end
+
+    def output_script machine, outputs
+      script = "vagrant plugin install vagrant-scp\n"
+      outputs.each do |name, path|
+        script += "vagrant scp :#{path} output/#{name}.data\n"
+      end
+      return script
+    end
+
+    def setup_env machine, scripts, config
+      puts 'Setting up task on machine '+machine.to_s+'...'
+      FileUtils.mkdir_p machine.to_s
+      Dir.chdir machine.to_s do
+        if !@inputs[machine].nil?
+          @inputs[machine].each do |name|
+            system('cp', '-f', "../../#{name}", "#{name}")
+          end
+        end
+        FileUtils.mkdir_p 'output'
+        puts 'Creating provision files...'
+        File.open('Vagrantfile', 'w+') do |f|
+          f.write(config.vms[machine].to_vagrant)
+        end
+        script = Script.new(scripts, config).to_s
+        File.open('provision.sh', 'w+') do |f|
+          f.write(script)
+        end
+        script = 'echo "nothing to do"'
+        script = self.output_script(machine, @outputs[machine]) unless  @outputs[machine].nil?
+        # add simple support for Windows
+        before_halt_hook = "before_halt.#{sys_script_ext}"
+        File.open(before_halt_hook, 'w+') do |f|
+          f.write(script)
+        end
+        system('chmod', '+x', before_halt_hook)
+      end
+    end
+
+    def host_scripts config
+      puts 'Setting up host scripts...'
+      if !@pre_instructions.nil?
+        script = Script.new(@pre_instructions, config).to_s
+        File.open('pre.sh', 'w+') do |f|
+          f.write(script)
+        end
+      end
+
+      if !@post_instructions.nil?
+        script = Script.new(@post_instructions, config).to_s
+        File.open('post.sh', 'w+') do |f|
+          f.write(script)
+        end
+      end
+    end
+
+    def execute_pre_instructions
+      puts 'Running pre instructions...'
+      system('sh', './pre.sh')
+    end
+
+    def run machine
+      Dir.chdir machine.to_s do
+        system('vagrant', 'up')
+      end
+    end
+
+    def execute_post_instructions
+      puts 'Running post instructions...'
+      system('sh', './post.sh')
+    end
+
+    def halt machine
+      Dir.chdir machine.to_s do
+        system('vagrant', 'halt')
+      end
+    end
+
+    def destroy machine
+      Dir.chdir machine.to_s do
+        system('vagrant', 'destroy', '-f')
+        system('rm', '-rf', '.vagrant')
+      end
+    end
+
+    def execute config, iteration
+      puts "Running job #{name.to_s}-#{iteration.to_s}..."
+      FileUtils.mkdir_p 'tmp'
+      Dir.chdir 'tmp' do
+        self.host_scripts config
+        @tasks.each do |machine, templates|
+          self.setup_env machine, templates, config
+        end
+        self.execute_pre_instructions unless @pre_instructions.nil?
+        @tasks.each do |machine, templates|
+          self.run machine
+        end
+        self.execute_post_instructions unless @post_instructions.nil?
+        @tasks.each do |machine, templates|
+          self.halt machine
+        end
+        @tasks.each do |machine, templates|
+          self.destroy machine
+        end
+      end
+      system('mv', 'tmp', "#{name.to_s}-#{iteration.to_s}")
+      system('tar', '-czvf', "#{name.to_s}-#{iteration.to_s}.tar.gz", "#{name.to_s}-#{iteration.to_s}")
+      system('rm', '-rf', "#{name.to_s}-#{iteration.to_s}")
+      config.github_conf.add("#{name.to_s}-#{iteration.to_s}.tar.gz") unless config.github_conf.nil?
+      config.github_conf.push unless config.github_conf.nil?
+      config.dataverse_conf.add("#{name.to_s}-#{iteration.to_s}.tar.gz") unless config.dataverse_conf.nil?
+      puts "Job #{name.to_s}-#{iteration.to_s} done."
+    end
+  end
+end