xanthus 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66257493ff2d3fe1d22a7ca0706fdf65ee3bb61d7546ed5d63372dab6b5e603a
-  data.tar.gz: fd7b4332a60cb729740443a33edbfc199ac971d5b928ae67d500b85bcbb34e51
+  metadata.gz: 2b041aa460e8d8de1945b504ee7527a776d0cb338fe16ab594a3462cece5ef85
+  data.tar.gz: 167816e4a9725fe18b7b989a21a56adc8b22bab1d50d9032566a28c12027ea0f
 SHA512:
-  metadata.gz: a1d94ad4ee1548a22e23106d0fc3728d6595794f1f7e05ad86a831aa97547921a08c33be44e9d95688dbae5d234cd0a936f1ed65e5ed8f1e173fb7b41a03d05c
-  data.tar.gz: 4437c5b2b627079279b1c08655b22ec6009dd7b3d96300e2bccd38161873d0fe8d37547cf90e2f635ca6f788e0475ff360214c9d706263a7003126dc05ba405e
+  metadata.gz: 7132c95f0535f042b0ebd54df8e1d09076e951fda8903e6cd0e1d447817a7c93b893f2eb15aa736a49b5cd12560b19b951c1d450f4ef53a1ac479578099747e5
+  data.tar.gz: 46ee72461803cf16f538b7d1625e847dd02023dd1fc42f476c7f37d99779685ecaff9c0f33b80d456cb7da4cc96c5cb0d1042c95734d5c79b62384f7f3ea1475

data/lib/xanthus.rb

@@ -4,7 +4,9 @@ require "xanthus/script"
 require "xanthus/virtual_machine"
 require "xanthus/job"
 require "xanthus/default"
+require "xanthus/repository"
 require "xanthus/github"
+require "xanthus/dataverse"
 require "xanthus/configuration"
 
 module Xanthus

data/lib/xanthus/configuration.rb

@@ -2,6 +2,8 @@ module Xanthus
   class Configuration
     attr_accessor :name
     attr_accessor :authors
+    attr_accessor :affiliation
+    attr_accessor :email
     attr_accessor :description
     attr_accessor :seed
     attr_accessor :params
@@ -9,6 +11,7 @@ module Xanthus
     attr_accessor :scripts
     attr_accessor :jobs
     attr_accessor :github_conf
+    attr_accessor :dataverse_conf
 
     def initialize
       @params = Hash.new
@@ -41,13 +44,21 @@ module Xanthus
       @github_conf = github
     end
 
+    def dataverse
+      dataverse = Dataverse.new
+      yield(dataverse)
+      @dataverse_conf = dataverse
+    end
+
     def to_readme_md
       %Q{
 # #{@name}
 
-Authors: #{@authors}
+authors: #{@authors}
+affiliation: #{@affiliation}
+email: #{@email}
 
-Seed: #{@seed}
+seed: #{@seed}
 
 ## Description
 
@@ -64,12 +75,19 @@ Seed: #{@seed}
     config.vms.each do |k, v|
       v.generate_box config
     end
+
+    # initializing storage backends
     config.github_conf.init(config) unless config.github_conf.nil?
+    config.dataverse_conf.init(config) unless config.dataverse_conf.nil?
+
+    # executing jobs
     config.jobs.each do |name,job|
       for i in 0..(job.iterations-1) do
         job.execute config, i
       end
     end
+
+    # finalizing storage backends
     config.github_conf.tag  unless config.github_conf.nil?
     config.github_conf.clean  unless config.github_conf.nil?
   end

data/lib/xanthus/dataverse.rb

@@ -0,0 +1,179 @@
+require 'fileutils'
+require 'json'
+
+module Xanthus
+  class Dataverse < Repository
+    attr_accessor :server
+    attr_accessor :repo
+    attr_accessor :token
+    attr_accessor :dataset_name
+    attr_accessor :author
+    attr_accessor :affiliation
+    attr_accessor :email
+    attr_accessor :description
+    attr_accessor :subject
+    attr_accessor :doi
+
+    def initialize
+      super
+      @server = 'default_server'
+      @repo = 'default_repo'
+      @token = 'default_token'
+      @dataset_name = 'default_name'
+      @author = 'default_author'
+      @affiliation = 'default_affiliation'
+      @email = 'default_email'
+      @description = 'default_description'
+      @subject = 'default_subject'
+      @doi = 'not_set'
+    end
+
+    def dataset_json
+json = %Q{
+  {
+    "datasetVersion": {
+      "metadataBlocks": {
+        "citation": {
+          "fields": [
+            {
+              "value": "#{@dataset_name}",
+              "typeClass": "primitive",
+              "multiple": false,
+              "typeName": "title"
+            },
+            {
+              "value": [
+                {
+                  "authorName": {
+                    "value": "#{@author}",
+                    "typeClass": "primitive",
+                    "multiple": false,
+                    "typeName": "authorName"
+                  },
+                  "authorAffiliation": {
+                    "value": "#{@affiliation}",
+                    "typeClass": "primitive",
+                    "multiple": false,
+                    "typeName": "authorAffiliation"
+                  }
+                }
+              ],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "author"
+            },
+            {
+              "value": [
+                  { "datasetContactEmail" : {
+                      "typeClass": "primitive",
+                      "multiple": false,
+                      "typeName": "datasetContactEmail",
+                      "value" : "#{@email}"
+                  },
+                  "datasetContactName" : {
+                      "typeClass": "primitive",
+                      "multiple": false,
+                      "typeName": "datasetContactName",
+                      "value": "#{@author}"
+                  }
+              }],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "datasetContact"
+            },
+            {
+              "value": [ {
+                 "dsDescriptionValue":{
+                  "value": "#{@description.gsub(/\r/," ").gsub(/\n/," ")}",
+                  "multiple":false,
+                 "typeClass": "primitive",
+                 "typeName": "dsDescriptionValue"
+              }}],
+              "typeClass": "compound",
+              "multiple": true,
+              "typeName": "dsDescription"
+            },
+            {
+              "value": [
+                "#{@subject}"
+              ],
+              "typeClass": "controlledVocabulary",
+              "multiple": true,
+              "typeName": "subject"
+            }
+          ],
+          "displayName": "Citation Metadata"
+        }
+      }
+    }
+  }
+}
+      return json
+    end
+
+    def create_dataset
+      Dir.chdir 'dataverse_dataset' do
+        File.open('dataset.json', 'w+') do |f|
+          f.write(self.dataset_json)
+        end
+        puts "Creating dataverse #{@dataset_name} in #{@repo} at #{@server}..."
+        output = `curl -H X-Dataverse-key:#{@token} -X POST #{@server}/api/dataverses/#{@repo}/datasets --upload-file dataset.json`
+        puts output # needed to escape curl output
+        parsed = JSON.parse(output)
+        @doi = parsed['data']['persistentId']
+        puts "Dataverse #{@doi} created."
+      end
+    end
+
+    def init config
+      # initialize with config information
+      @author = config.authors
+      @affiliation = config.affiliation
+      @email = config.email
+      @description = config.description
+      @dataset_name = config.name+'-'+Time.now.strftime("%Y-%m-%d_%H-%M")
+
+      FileUtils.mkdir_p 'dataverse_dataset'
+      self.create_dataset
+      Dir.chdir 'dataverse_dataset' do
+        FileUtils.mkdir_p 'repo'
+        Dir.chdir 'repo' do
+          self.xanthus_file
+          self.readme_file config
+          self.inputs_file config
+        end
+      end
+    end
+
+    def add_file_to_dataverse name, description, folder
+      output = `curl -H X-Dataverse-key:#{@token} -X POST -F "file=@#{name}" -F 'jsonData={"description":"#{description}","directoryLabel":"#{folder}","categories":["Data"], "restrict":"false"}' "#{@server}/api/datasets/:persistentId/add?persistentId=#{@doi}"`
+      puts output
+    end
+
+    def xanthus_file
+      self.prepare_xanthus_file
+      self.add_file_to_dataverse '.xanthus', 'xanthus file used to generate the data.', 'metadata'
+    end
+
+    def readme_file config
+      self.prepare_readme_file config
+      self.add_file_to_dataverse 'README.md', 'readme describing the dataset.', 'metadata'
+    end
+
+    def inputs_file config
+      config.jobs.each do |name,job|
+        job.inputs.each do |k, files|
+          files.each do |file|
+            system('cp', '-f', "../../#{file}", "#{file}")
+            self.add_file_to_dataverse file, 'Job input file.', 'metadata'
+          end
+        end
+      end
+    end
+
+    def add content
+      self.add_file_to_dataverse content, "#{content} is a Xanthus generated file (check metadata for description)", 'data'
+    end
+
+  end
+end

data/lib/xanthus/github.rb

@@ -1,12 +1,13 @@
 require 'fileutils'
 
 module Xanthus
-  class GitHub
+  class GitHub < Repository
     attr_accessor :repo
     attr_accessor :token
     attr_accessor :folder
 
     def initialize
+      super
       @repo = ''
       @token = ''
       @folder = Time.now.strftime("%Y-%m-%d_%H-%M")
@@ -20,23 +21,14 @@ module Xanthus
     end
 
     def xanthus_file
-      script = ''
-      File.readlines('../../.xanthus').each do |line|
-        script += line unless line.include? 'github.token'
-        script += "\t\tgithub.token = 'REMOVED'\n" unless !line.include? 'github.token'
-      end
-      File.open('.xanthus', 'w+') do |f|
-        f.write(script)
-      end
+      self.prepare_xanthus_file
       system('git', 'add', '.xanthus')
       system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/.xanthus :horse:")
       system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
     end
 
     def readme_file config
-      File.open('README.md', 'w+') do |f|
-        f.write(config.to_readme_md)
-      end
+      self.prepare_readme_file config
       system('git', 'add', 'README.md')
       system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/README.md :horse:")
       system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')

data/lib/xanthus/init.rb

@@ -17,6 +17,8 @@ module Xanthus
 Xanthus.configure do |config|
   config.name = '#{@@name}'
   config.authors = 'John Doe'
+  config.affiliation = 'Somewhere University'
+  config.email = 'john.doe@somewhere.edu'
   config.description = %q{
 Describe my super experiment.
 
@@ -119,10 +121,17 @@ It is very cool and interesting!
     job.outputs = {spade: {trace: '/tmp/audit_cdm.avro'}}
   end
 
-  config.github do |github|
-    github.repo = '<ADD GITHUB REPO user/name>'
-    github.token = '<ADD GITHUB TOKEN>'
+  config.dataverse do |dataverse|
+    dataverse.server = <ADD DATAVERSE BASE URL>
+    dataverse.repo = <PROVIDE DATAVERSE NAME>
+    dataverse.token = <PROVIDE DATAVERSE TOKEN>
+    dataverse.subject = <PROVIDE DATAVERSE SUBJECT (e.g. engineering)>
   end
+
+  # config.github do |github|
+  #   github.repo = '<ADD GITHUB REPO user/name>'
+  #   github.token = '<ADD GITHUB TOKEN>'
+  # end
 end
 }
       file.write(script)

data/lib/xanthus/job.rb

@@ -19,10 +19,10 @@ module Xanthus
       @post_instructions = nil
     end
 
-    def output_script outputs
-      script = ''
+    def output_script machine, outputs
+      script = "vagrant plugin install vagrant-scp\n"
       outputs.each do |name, path|
-        script += "cp -f #{path} /vagrant/output/#{name}.data\n"
+        script += "vagrant scp :#{path} output/#{name}.data\n"
       end
       return script
     end
@@ -45,10 +45,11 @@ module Xanthus
         File.open('provision.sh', 'w+') do |f|
           f.write(script)
         end
-        script = self.output_script(@outputs[machine]) unless  @outputs[machine].nil?
+        script = self.output_script(machine, @outputs[machine]) unless  @outputs[machine].nil?
         File.open('before_halt.sh', 'w+') do |f|
           f.write(script)
         end
+        system('chmod', '+x', 'before_halt.sh')
       end
     end
 
@@ -123,6 +124,7 @@ module Xanthus
       system('rm', '-rf', "#{name.to_s}-#{iteration.to_s}")
       config.github_conf.add("#{name.to_s}-#{iteration.to_s}.tar.gz") unless config.github_conf.nil?
       config.github_conf.push unless config.github_conf.nil?
+      config.dataverse_conf.add("#{name.to_s}-#{iteration.to_s}.tar.gz") unless config.dataverse_conf.nil?
       puts "Job #{name.to_s}-#{iteration.to_s} done."
     end
   end

data/lib/xanthus/repository.rb

@@ -0,0 +1,28 @@
+module Xanthus
+  class Repository
+
+    def initialize
+    end
+
+    def prepare_xanthus_file
+      script = ''
+      File.readlines('../../.xanthus').each do |line|
+        script += line unless line.include?('github.token') || line.include?('dataverse.token')
+
+        # remove github token
+        script += "\t\t# github.token = 'REMOVED'\n" unless !line.include? 'github.token'
+        # remove dataverse token
+        script += "\t\t# dataverse.token = 'REMOVED'\n" unless !line.include? 'dataverse.token'
+      end
+      File.open('.xanthus', 'w+') do |f|
+        f.write(script)
+      end
+    end
+
+    def prepare_readme_file config
+      File.open('README.md', 'w+') do |f|
+        f.write(config.to_readme_md)
+      end
+    end
+  end
+end

data/lib/xanthus/version.rb

@@ -1,5 +1,5 @@
 module Xanthus
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 
   def self.version
     puts VERSION

data/lib/xanthus/virtual_machine.rb

@@ -11,6 +11,14 @@ module Xanthus
     attr_accessor :boxing
     attr_accessor :ssh_username
     attr_accessor :ssh_key_path
+    attr_accessor :on_aws
+    attr_accessor :aws_env_key_id
+    attr_accessor :aws_env_key_secret
+    attr_accessor :aws_key_pair_name
+    attr_accessor :aws_region
+    attr_accessor :aws_ami
+    attr_accessor :aws_instance_type
+    attr_accessor :aws_security_group
 
     def initialize
       @name = :default
@@ -33,6 +41,9 @@ Vagrant.configure(2) do |config|
   config.vm.box_version = "#{@version}"
   config.vm.network "private_network", ip: "#{@ip}"
 }
+script += %Q{
+  config.vm.synced_folder ".", "/vagrant", disabled: false, type: 'rsync'
+} unless !@on_aws
 script += %Q{
   config.ssh.username = "#{@ssh_username}"
 } unless ssh_username.nil?
@@ -40,18 +51,30 @@ script += %Q{
   config.ssh.private_key_path = "#{@ssh_key_path}"
 } unless ssh_key_path.nil?
 script += %Q{
-  config.vm.provider "virtualbox" do |vb|
+  config.vm.provider "virtualbox" do |vb, override|
    vb.gui = #{@gui}
    vb.memory = #{@memory}
    vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{@cpu_cap}"]
    vb.cpus = #{@cpus}
    vb.name = "#{@name}"
   end
-  config.vm.provision "shell", path: "provision.sh"
+} unless @on_aws
+script += %Q{
+  config.vm.provider "aws" do |aws, override|
+   aws.access_key_id = ENV['#{@aws_env_key_id}']
+   aws.secret_access_key = ENV['#{@aws_env_key_secret}']
+   aws.keypair_name = '#{@aws_key_pair_name}'
+   aws.region = '#{@aws_region}'
+   aws.ami =  '#{@aws_ami}'
+   aws.instance_type = '#{@aws_instance_type}'
+   aws.security_groups = ['#{@aws_security_group}']
+  end
+} unless !@on_aws
+script += %Q{  config.vm.provision "shell", path: "provision.sh"
 
   config.trigger.before :halt do |trigger|
     trigger.info = "Retrieving data before halt..."
-    trigger.run_remote = {path: "before_halt.sh"}
+    trigger.run = {path: "before_halt.sh"}
   end
 end
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xanthus
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Thomas Pasquier
@@ -9,34 +9,48 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-31 00:00:00.000000000 Z
+date: 2020-04-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Automated intrusion detection dataset generation framework.
@@ -52,10 +66,12 @@ files:
 - bin/xanthus
 - lib/xanthus.rb
 - lib/xanthus/configuration.rb
+- lib/xanthus/dataverse.rb
 - lib/xanthus/default.rb
 - lib/xanthus/github.rb
 - lib/xanthus/init.rb
 - lib/xanthus/job.rb
+- lib/xanthus/repository.rb
 - lib/xanthus/script.rb
 - lib/xanthus/version.rb
 - lib/xanthus/virtual_machine.rb
@@ -79,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Automated intrusion detection dataset generation framework.