xanthus 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +222 -0
- data/bin/xanthus +25 -0
- data/lib/xanthus.rb +12 -0
- data/lib/xanthus/configuration.rb +73 -0
- data/lib/xanthus/default.rb +21 -0
- data/lib/xanthus/github.rb +84 -0
- data/lib/xanthus/init.rb +147 -0
- data/lib/xanthus/job.rb +103 -0
- data/lib/xanthus/version.rb +7 -0
- data/lib/xanthus/virtual_machine.rb +42 -0
- metadata +85 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e366be3c9180e9788c19dfaa31e52222cd0782ce8cce0d99c3d05d6a509e8454
|
4
|
+
data.tar.gz: d44b89c4285251a10c172bea19064607f63f9b79e84ed1c8ee814f4296ec43eb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a16dd967a99ec4d5f20f949d42a803de47ca4e60265b7474e77bdef3bcfacced87da0d86d5b62371ff42b44b2f7c84a888e77bc08c738938098c002f61085561
|
7
|
+
data.tar.gz: 385c29998fe84c0a5d3c3c1c0c401587ce9c940f6b7e84527d161632c934a20b3d7815e0eb1d0ce6c4d71f2240e1cb9b24d00cf6561bda4b3deb87361cbf920f
|
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2019 Thomas Pasquier
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,222 @@
|
|
1
|
+
# Xanthus: Automated Reproducible Data Generation for Evaluating Intrusion Detection Systems
|
2
|
+
|
3
|
+
Fairly evaluating and comparing the efficacy of different intrusion detection systems (IDS) requires that experimental data
|
4
|
+
be generated in a similar mechanism and/or shared across these systems.
|
5
|
+
The reality, unfortunately, is that there exist few public repositories (e.g., DARPA 1998/1999/2000, KDD Cup99, DARPA TC Engagement 3)
|
6
|
+
containing experimental data captured solely for the purpose of security analysis.
|
7
|
+
Among those public data repositories, most are outdated because a tremendous amount of manual labor is almost always
|
8
|
+
necessary to capture the data (e.g., DARPA TC program involves a number of teams from
|
9
|
+
across the academia and the industry and it spans over many a year).
|
10
|
+
Consequently, some newly-developed systems, in order to be able to compare against older systems,
|
11
|
+
are evaluated using the data that is a decade or two older than the systems themselves
|
12
|
+
(and usually and unsurprisingly exhibit good results).
|
13
|
+
Given that there is a perpetual arms race between the defenders and the offenders in the realm of cyber security
|
14
|
+
and that new cyber-threats are manufactured every day,
|
15
|
+
a successful defence against a decade-old exploit is hardly an achievement.
|
16
|
+
|
17
|
+
Many existing systems, acknowledging this fact and ready to showcase their detection capability,
|
18
|
+
design their own experiments and produce their own dataset as a result.
|
19
|
+
Although the experiments are sometimes carefully described in their associated publications (e.g., in academic projects),
|
20
|
+
such dataset suffers from the following drawbacks:
|
21
|
+
|
22
|
+
- In the cases where the dataset is made public, later systems can but consume only a subset of the dataset for analysis.
|
23
|
+
Therefore, if they require e.g., additional features from the dataset in the analysis, they must rerun the experiments
|
24
|
+
to capture the data themselves again, instead of simply re-using the available dataset.
|
25
|
+
Moreover, some systems publish only pre-processed dataset, which usually eliminates information from the original,
|
26
|
+
raw dataset that is not relevant to their analysis, even though such information may be relevant for other systems.
|
27
|
+
|
28
|
+
- When raw dataset is made public, it provides later systems with richer information content.
|
29
|
+
However, the underlying systems that capture the raw dataset (e.g., audit systems) are also constantly evolving,
|
30
|
+
generating finer-grained, more accurate information or
|
31
|
+
offering a completely different perspective through which one understands system behavior (e.g., provenance systems).
|
32
|
+
Security systems that take advantage of such advancement in the underlying systems
|
33
|
+
may very well find even the raw data provided by previous systems insufficient.
|
34
|
+
|
35
|
+
- If later systems must resort to reproducing dataset themselves as a result of the reasons listed above,
|
36
|
+
they need to rely on descriptions provided by previous systems to ensure high-fidelity experiment replay.
|
37
|
+
Even if we assume that previous systems provide sufficiently detailed descriptions to understand the experiment
|
38
|
+
(which certainly is not always the case),
|
39
|
+
there still exist a number of challenges.
|
40
|
+
|
41
|
+
- The experiment must be conducted using the exact software involved with matching versions.
|
42
|
+
In many cases, security experts have since identified and patched vulnerabilities in the exploitable software
|
43
|
+
used in security-related experiments, and thus the software itself usually has been updated to a newer version.
|
44
|
+
Downgrading the target software and its dependencies is therefore necessary to reproduce the experiment. This
|
45
|
+
sometimes cannot be automatically configured through existing package management systems and requires significant
|
46
|
+
manual configuration.
|
47
|
+
|
48
|
+
- Some vulnerability may affect only a particular version of the operating system. This requirement no doubt
|
49
|
+
further complicates the experimental setup and demands additional engineering effort.
|
50
|
+
|
51
|
+
- Other controllable factors may be omitted in the description that may or may not affect the final results of the
|
52
|
+
experiment. For example, background activities may have been included in the dataset but was not discussed in detail.
|
53
|
+
|
54
|
+
Before we go into any detail about using **Xanthus** for automated, reproducible data generation for security analysis,
|
55
|
+
we describe a pipeline in which we create dataset for a *specific* attack in a push-button fashion. **Xanthus** is
|
56
|
+
a higher-level abstracted framework that generates such a pipeline for *any* attack that existing or future IDS intend to
|
57
|
+
evaluate.
|
58
|
+
|
59
|
+
## Primer to Xanthus: A Specific Pipeline
|
60
|
+
|
61
|
+
We introduce a specific pipeline that automates data capture for a particular attack.
|
62
|
+
In this pipeline, we deploy virtual machines (VM), set up a virtual environment that recreates the attack scenario,
|
63
|
+
and run the attack, while capturing data from a whole-system provenance capture system.
|
64
|
+
Code is publicly available online at [GitHub](https://github.com/crimson-unicorn/demo/tree/master/wget).
|
65
|
+
Please refer to the code while finishing off the rest of this section.
|
66
|
+
|
67
|
+
### Prerequisites
|
68
|
+
|
69
|
+
We assume that you understand the following terms and concepts.
|
70
|
+
If not, click on the item that you do not understand to read more about it:
|
71
|
+
|
72
|
+
* [Virtual machines](https://en.wikipedia.org/wiki/Virtual_machine)
|
73
|
+
* [Makefile](https://en.wikipedia.org/wiki/Makefile)
|
74
|
+
* [VirtualBox](https://www.virtualbox.org/manual/ch01.html)
|
75
|
+
* [Vagrant](https://www.vagrantup.com/intro/index.html), [Vagrantfile](https://www.vagrantup.com/docs/vagrantfile/)
|
76
|
+
and [provisioning](https://www.vagrantup.com/docs/provisioning/index.html)
|
77
|
+
* [CamFlow](http://camflow.org)
|
78
|
+
|
79
|
+
You may want to understand the following terms and concepts if you want to fully understand the attack
|
80
|
+
that we will describe in the next section:
|
81
|
+
|
82
|
+
* [Trojan software](https://en.wikipedia.org/wiki/Advanced_persistent_threat)
|
83
|
+
and [reverse shell](https://resources.infosecinstitute.com/icmp-reverse-shell/#gref)
|
84
|
+
|
85
|
+
### A Brief Attack Description
|
86
|
+
|
87
|
+
You could better understand the pipeline with the knowledge of the attack that we would like to reproduce automatically.
|
88
|
+
The attacker aims to invade a victim machine through a vulnerable (or exploitable) `wget`.
|
89
|
+
The attacker sets up a malicious (or compromised) `HTTP` server that redirects any requests to a malicious `FTP` server
|
90
|
+
that contains a `Debian` package with a Trojan backdoor.
|
91
|
+
The package appears to be the same as its legitimate version and may even work the same way,
|
92
|
+
but the moment the package is installed on the victim machine, it will initiate a reverse TCP connection to the attacker
|
93
|
+
who is listening for connections and create a reverse shell that allows the attacker to infiltrate into the victim machine.
|
94
|
+
|
95
|
+
When the victim machine attempts to download the benign package from the `HTTP` server using `wget`,
|
96
|
+
`wget` allows arbitrary remote file upload to the host system.
|
97
|
+
Meaning that, instead of fetching the intended benign package, it allows redirection of the `HTTP` server and downloads
|
98
|
+
the malicious one.
|
99
|
+
The user is unaware of such behavior and install the package through the package manager `dpkg`.
|
100
|
+
The installed Trojan software establishes a connection to the attacker and the attack succeeds.
|
101
|
+
|
102
|
+
### Software Involved
|
103
|
+
|
104
|
+
* `wget` v1.17 or older
|
105
|
+
* Any `Debian` package with a Trojan backdoor. The `Debian` package must be installable (both benign and malicious version).
|
106
|
+
* Functioning `HTTP` and `FTP` server
|
107
|
+
* `dpkg` package manager
|
108
|
+
* `CamFlow` whole-system provenance capture system
|
109
|
+
|
110
|
+
### Execution Platform
|
111
|
+
|
112
|
+
As expected, `Debian` package can only run on any `Debian`-based operating systems. This particular pipeline is run on
|
113
|
+
`Ubuntu 18.04` (both the client and the server).
|
114
|
+
|
115
|
+
### The Pipeline
|
116
|
+
|
117
|
+
#### Installation
|
118
|
+
|
119
|
+
To run this pipeline, you need to install at least the following items:
|
120
|
+
|
121
|
+
* `Vagrant`
|
122
|
+
* Oracle `VirtualBox`
|
123
|
+
|
124
|
+
#### Usage
|
125
|
+
|
126
|
+
If you `git clone` the entire repository from [GitHub](https://github.com/crimson-unicorn/demo/), `cd` into `wget` directory.
|
127
|
+
We assume this directory would be your working directory.
|
128
|
+
|
129
|
+
We write a `Makefile` to run our attack scenario for many times. If you want to run it once only,
|
130
|
+
modify this line: `[ $${cnt} -lt 25 ]` to `[ $${cnt} -lt 1 ]` in the `Makefile`.
|
131
|
+
(In `Xanthus`, we would be able to configure this easily without actually modifying the code.)
|
132
|
+
|
133
|
+
If you are running on `Mac`:
|
134
|
+
```
|
135
|
+
make test_mac
|
136
|
+
```
|
137
|
+
On `Linux`, you would run:
|
138
|
+
```
|
139
|
+
make test_linux
|
140
|
+
```
|
141
|
+
We do *not* support `Windows` operating system for now.
|
142
|
+
You would locate the output data file in `data/` directory.
|
143
|
+
|
144
|
+
#### Behind the Scenes
|
145
|
+
|
146
|
+
This pipeline seems to be very user-friendly. So, one might ask, why do we bother to design and implement `Xanthus`?
|
147
|
+
The truth is, we have done a lot of heavy-lifting for you behind the scenes. Let's take a closer look.
|
148
|
+
|
149
|
+
The `Makefile` you run starts the `vagrant` process, which would boot up two virtual machines, one `server` and one
|
150
|
+
`client` (now, take a look into `Vagrantfile`).
|
151
|
+
|
152
|
+
The `server` machine is provisioned by `provision/server.sh` script.
|
153
|
+
It configures an `FTP` and an `HTTP` server and puts the malicious `Debian` package in the `FTP` server.
|
154
|
+
Of course, the user must provide the pipeline with the package.
|
155
|
+
We build the package ourselves in [Kali Linux](https://en.wikipedia.org/wiki/Makefile)
|
156
|
+
with [TheFatRat](https://github.com/Screetsec/TheFatRat). You are free to use any tools at your disposal.
|
157
|
+
We also put the benign one in the `HTTP` server to trick the user to download it.
|
158
|
+
|
159
|
+
The `client` machine involves more operations.
|
160
|
+
First, unlike the `server` machine that simply uses a `Ubuntu 18.04` base operating system
|
161
|
+
(as seen in `server.vm.box = "bento/ubuntu-18.04"`),
|
162
|
+
the `client` machine uses our customized `VirtualBox` box called `michaelh/ubuncam`.
|
163
|
+
This box is built with the following specifications:
|
164
|
+
|
165
|
+
* It is built upon the original `Ubuntu 18.04` base box from `Vagrant`.
|
166
|
+
* It is installed with `CamFlow` as its provenance-capture system.
|
167
|
+
* It downgrades `wget` to its desired version (`v1.17`) that contains the vulnerability.
|
168
|
+
* It can install `Debian` packages in the experiment.
|
169
|
+
|
170
|
+
Note that it is always desirable to package such a box and upload it to the `VagrantCloud` so that we can
|
171
|
+
configure once and reuse many times.
|
172
|
+
One can always use a base box and configure the above specifications on-the-fly,
|
173
|
+
but it is not guaranteed that the configuration would work in the distant future.
|
174
|
+
For example, the link to download an older version of `wget` may expire without notice.
|
175
|
+
`Xanthus` allows users to either provide a customized virtual box or configure a base box through provisioning.
|
176
|
+
If an online configuration is provided, `Xanthus` would automatically generate a customized box for the user
|
177
|
+
to prevent future re-configuration or possible failure in future configuration.
|
178
|
+
|
179
|
+
The `client` machine runs the script in `provision/attack`.
|
180
|
+
The user must provide such a script.
|
181
|
+
In our case, we automatically generate attack scripts using `wget-attack-script-gen.py`.
|
182
|
+
`Xanthus` allows users to provide logic to generate scripts or simply provide scripts to run during the experiment.
|
183
|
+
|
184
|
+
## Xanthus
|
185
|
+
|
186
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/xanthus`. To experiment with that code, run `bin/console` for an interactive prompt.
|
187
|
+
|
188
|
+
TODO: Delete this and the text above, and describe your gem
|
189
|
+
|
190
|
+
### Installation
|
191
|
+
|
192
|
+
Add this line to your application's Gemfile:
|
193
|
+
|
194
|
+
```ruby
|
195
|
+
gem 'xanthus'
|
196
|
+
```
|
197
|
+
|
198
|
+
And then execute:
|
199
|
+
|
200
|
+
$ bundle
|
201
|
+
|
202
|
+
Or install it yourself as:
|
203
|
+
|
204
|
+
$ gem install xanthus
|
205
|
+
|
206
|
+
### Usage
|
207
|
+
|
208
|
+
TODO: Write usage instructions here
|
209
|
+
|
210
|
+
### Development
|
211
|
+
|
212
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
213
|
+
|
214
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
215
|
+
|
216
|
+
### Contributing
|
217
|
+
|
218
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/xanthus.
|
219
|
+
|
220
|
+
### License
|
221
|
+
|
222
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/bin/xanthus
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "xanthus"
|
4
|
+
|
5
|
+
instruction = ARGV[0]
|
6
|
+
param1 = ARGV[1]
|
7
|
+
|
8
|
+
if (instruction == 'version')
|
9
|
+
Xanthus.version
|
10
|
+
elsif (instruction == 'init')
|
11
|
+
Xanthus::Init.init param1
|
12
|
+
elsif (instruction == 'run')
|
13
|
+
load('./.xanthus')
|
14
|
+
elsif (instruction == 'help')
|
15
|
+
puts 'xanthus version | return version number.'
|
16
|
+
puts 'xanthus depencies | installation instruction for system dependencies.'
|
17
|
+
puts 'xanthus init <project name> | inialise a new project.'
|
18
|
+
puts 'xanthus run | run .xanthus file contained in the current folder.'
|
19
|
+
elsif (instruction == 'dependencies')
|
20
|
+
puts 'You need to install the following software on your system for xanthus to run:'
|
21
|
+
puts 'git (see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)'
|
22
|
+
puts 'git lfs (see https://help.github.com/en/articles/installing-git-large-file-storage)'
|
23
|
+
puts 'virtualbox (see https://www.virtualbox.org/wiki/Downloads)'
|
24
|
+
puts 'vagrant (see https://www.vagrantup.com/docs/installation/)'
|
25
|
+
end
|
data/lib/xanthus.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require "xanthus/version"
|
2
|
+
require "xanthus/init"
|
3
|
+
require "xanthus/virtual_machine"
|
4
|
+
require "xanthus/job"
|
5
|
+
require "xanthus/default"
|
6
|
+
require "xanthus/github"
|
7
|
+
require "xanthus/configuration"
|
8
|
+
|
9
|
+
module Xanthus
|
10
|
+
class Error < StandardError; end
|
11
|
+
# Your code goes here...
|
12
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Xanthus
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :name
|
4
|
+
attr_accessor :authors
|
5
|
+
attr_accessor :description
|
6
|
+
attr_accessor :seed
|
7
|
+
attr_accessor :params
|
8
|
+
attr_accessor :vms
|
9
|
+
attr_accessor :scripts
|
10
|
+
attr_accessor :jobs
|
11
|
+
attr_accessor :github_conf
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@params = Hash.new
|
15
|
+
@vms = Hash.new
|
16
|
+
@scripts = Hash.new
|
17
|
+
@jobs = Hash.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def vm name
|
21
|
+
vm = VirtualMachine.new
|
22
|
+
yield(vm)
|
23
|
+
vm.name = name
|
24
|
+
@vms[name] = vm
|
25
|
+
end
|
26
|
+
|
27
|
+
def script name
|
28
|
+
@scripts[name] = yield
|
29
|
+
end
|
30
|
+
|
31
|
+
def job name
|
32
|
+
v = Job.new
|
33
|
+
yield(v)
|
34
|
+
v.name = name
|
35
|
+
@jobs[name] = v
|
36
|
+
end
|
37
|
+
|
38
|
+
def github
|
39
|
+
github = GitHub.new
|
40
|
+
yield(github)
|
41
|
+
@github_conf = github
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_readme_md
|
45
|
+
%Q{
|
46
|
+
# #{@name}
|
47
|
+
|
48
|
+
Authors: #{@authors}
|
49
|
+
|
50
|
+
Seed: #{@seed}
|
51
|
+
|
52
|
+
## Description
|
53
|
+
|
54
|
+
#{@description}
|
55
|
+
}
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.configure
|
60
|
+
config = Configuration.new
|
61
|
+
yield(config)
|
62
|
+
puts "Running experiment #{config.name} with seed #{config.seed}."
|
63
|
+
srand config.seed
|
64
|
+
config.github_conf.init(config) unless config.github_conf.nil?
|
65
|
+
config.jobs.each do |name,job|
|
66
|
+
for i in 0..(job.iterations-1) do
|
67
|
+
job.execute config, i
|
68
|
+
end
|
69
|
+
end
|
70
|
+
config.github_conf.tag unless config.github_conf.nil?
|
71
|
+
config.github_conf.clean unless config.github_conf.nil?
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Xanthus
|
2
|
+
CAMFLOW_START = %q{%{
|
3
|
+
camflow -a true
|
4
|
+
sleep 1
|
5
|
+
}}
|
6
|
+
|
7
|
+
CAMFLOW_STOP = %q{%{
|
8
|
+
camflow -a false
|
9
|
+
sleep 20
|
10
|
+
}}
|
11
|
+
|
12
|
+
SPADE_START = %q{%{
|
13
|
+
echo spade | sudo -H -u spade ../SPADE/bin/spade start
|
14
|
+
sleep 1
|
15
|
+
}}
|
16
|
+
|
17
|
+
SPADE_STOP = %q{%{
|
18
|
+
echo spade | sudo -H -u spade ../SPADE/bin/spade stop
|
19
|
+
sleep 20
|
20
|
+
}}
|
21
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Xanthus
|
4
|
+
class GitHub
|
5
|
+
attr_accessor :repo
|
6
|
+
attr_accessor :token
|
7
|
+
attr_accessor :folder
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@repo = ''
|
11
|
+
@token = ''
|
12
|
+
@folder = Time.now.strftime("%Y-%m-%d_%H-%M")
|
13
|
+
end
|
14
|
+
|
15
|
+
def lfs
|
16
|
+
system('git', 'lfs', 'install')
|
17
|
+
system('git', 'lfs', 'track', '*.tar.gz')
|
18
|
+
system('git', 'add', '.gitattributes')
|
19
|
+
system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
|
20
|
+
end
|
21
|
+
|
22
|
+
def xanthus_file
|
23
|
+
script = ''
|
24
|
+
File.readlines('../../.xanthus').each do |line|
|
25
|
+
script += line unless line.include? 'github.token'
|
26
|
+
script += "\t\tgithub.token = 'REMOVED'\n" unless !line.include? 'github.token'
|
27
|
+
end
|
28
|
+
File.open('.xanthus', 'w+') do |f|
|
29
|
+
f.write(script)
|
30
|
+
end
|
31
|
+
system('git', 'add', '.xanthus')
|
32
|
+
system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/.xanthus :horse:")
|
33
|
+
system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
|
34
|
+
end
|
35
|
+
|
36
|
+
def readme_file config
|
37
|
+
File.open('README.md', 'w+') do |f|
|
38
|
+
f.write(config.to_readme_md)
|
39
|
+
end
|
40
|
+
system('git', 'add', 'README.md')
|
41
|
+
system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/README.md :horse:")
|
42
|
+
system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
|
43
|
+
end
|
44
|
+
|
45
|
+
def init config
|
46
|
+
system('git', 'clone', "https://#{@token}@github.com/#{@repo}", 'repo')
|
47
|
+
Dir.chdir 'repo' do
|
48
|
+
self.lfs
|
49
|
+
FileUtils.mkdir_p @folder
|
50
|
+
Dir.chdir @folder do
|
51
|
+
self.xanthus_file
|
52
|
+
self.readme_file config
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def add content
|
58
|
+
Dir.chdir 'repo' do
|
59
|
+
FileUtils.mkdir_p @folder
|
60
|
+
system('mv', "../#{content}", "#{@folder}/#{content}")
|
61
|
+
system('git', 'add', "#{@folder}/#{content}")
|
62
|
+
system('git', 'commit', '-m', "[Xanthus] :horse: pushed #{@folder}/#{content} :horse:")
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def push
|
67
|
+
Dir.chdir 'repo' do
|
68
|
+
system('git', 'push', "https://#{@token}@github.com/#{@repo}", 'master')
|
69
|
+
system('rm', '-rf', @folder)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def tag
|
74
|
+
Dir.chdir 'repo' do
|
75
|
+
system('git', 'tag', '-a', "xanthus-#{@folder}", '-m', '"Xanthus automated dataset generation."')
|
76
|
+
system('git', 'push', '--tags', "https://#{@token}@github.com/#{@repo}")
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def clean
|
81
|
+
system('rm', '-rf', 'repo')
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/xanthus/init.rb
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Xanthus
|
4
|
+
class Init
|
5
|
+
@@name
|
6
|
+
|
7
|
+
def self.header file
|
8
|
+
file.write("# -*- mode: ruby -*-\n")
|
9
|
+
file.write("# vi: set ft=ruby\n\n")
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.config file
|
13
|
+
script = %Q{
|
14
|
+
# -*- mode: ruby -*-
|
15
|
+
# vi: set ft=ruby
|
16
|
+
|
17
|
+
Xanthus.configure do |config|
|
18
|
+
config.name = '#{@@name}'
|
19
|
+
config.authors = 'John Doe'
|
20
|
+
config.description = %q{
|
21
|
+
Describe my super experiment.
|
22
|
+
|
23
|
+
It is very cool and interesting!
|
24
|
+
}
|
25
|
+
config.seed = #{Random.new_seed}
|
26
|
+
|
27
|
+
config.script :pre do
|
28
|
+
%q{%{
|
29
|
+
mkdir wgets
|
30
|
+
cd wgets
|
31
|
+
}}
|
32
|
+
end
|
33
|
+
|
34
|
+
config.script :camflow_start do
|
35
|
+
Xanthus::CAMFLOW_START
|
36
|
+
end
|
37
|
+
|
38
|
+
config.script :spade_start do
|
39
|
+
Xanthus::SPADE_START
|
40
|
+
end
|
41
|
+
|
42
|
+
config.script :normal do
|
43
|
+
%q{
|
44
|
+
2.times.collect do
|
45
|
+
'wget http://www.google.com'
|
46
|
+
end
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
config.script :attack do
|
51
|
+
%q{
|
52
|
+
2.times.collect do
|
53
|
+
'wget http://www.google.com'
|
54
|
+
end
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
config.script :camflow_stop do
|
59
|
+
Xanthus::CAMFLOW_STOP
|
60
|
+
end
|
61
|
+
|
62
|
+
config.script :spade_stop do
|
63
|
+
Xanthus::SPADE_STOP
|
64
|
+
end
|
65
|
+
|
66
|
+
config.script :post do
|
67
|
+
%q{%{
|
68
|
+
cd ..
|
69
|
+
rm -rf wgets
|
70
|
+
}}
|
71
|
+
end
|
72
|
+
|
73
|
+
config.script :server do
|
74
|
+
%q{%{
|
75
|
+
mkdir test
|
76
|
+
}}
|
77
|
+
end
|
78
|
+
|
79
|
+
config.vm :camflow do |vm|
|
80
|
+
vm.box = 'michaelh/ubuncam'
|
81
|
+
vm.version = '0.0.3'
|
82
|
+
vm.ip = '192.168.33.8'
|
83
|
+
end
|
84
|
+
|
85
|
+
config.vm :spade do |vm|
|
86
|
+
vm.box = 'michaelh/spade'
|
87
|
+
vm.memory = 8192
|
88
|
+
vm.version = '0.0.3'
|
89
|
+
vm.ip = '192.168.33.8'
|
90
|
+
end
|
91
|
+
|
92
|
+
config.vm :server do |vm|
|
93
|
+
vm.box = 'bento/ubuntu-18.04'
|
94
|
+
vm.version = '201812.27.0'
|
95
|
+
vm.ip = '192.168.33.3'
|
96
|
+
end
|
97
|
+
|
98
|
+
config.job :normal_camflow do |job|
|
99
|
+
job.iterations = 2
|
100
|
+
job.tasks = {camflow: [:pre, :camflow_start, :normal, :camflow_stop, :post]}
|
101
|
+
job.outputs = {camflow: {config: '/etc/camflow.ini', trace: '/tmp/audit.log'}}
|
102
|
+
end
|
103
|
+
|
104
|
+
config.job :attack_camflow do |job|
|
105
|
+
job.iterations = 2
|
106
|
+
job.tasks = {server: [:server], camflow: [:pre, :camflow_start, :attack, :camflow_stop, :post]}
|
107
|
+
job.outputs = {camflow: {config: '/etc/camflow.ini', trace: '/tmp/audit.log'}}
|
108
|
+
end
|
109
|
+
|
110
|
+
config.job :normal_spade do |job|
|
111
|
+
job.iterations = 2
|
112
|
+
job.tasks = {spade: [:pre, :spade_start, :normal, :spade_stop, :post]}
|
113
|
+
job.outputs = {spade: {trace: '/tmp/audit_cmd.avro'}}
|
114
|
+
end
|
115
|
+
|
116
|
+
config.job :attack_spade do |job|
|
117
|
+
job.iterations = 2
|
118
|
+
job.tasks = {server: [:server], spade: [:pre, :spade_start, :attack, :spade_stop, :post]}
|
119
|
+
job.outputs = {spade: {trace: '/tmp/audit_cmd.avro'}}
|
120
|
+
end
|
121
|
+
|
122
|
+
config.github do |github|
|
123
|
+
github.repo = '<ADD GITHUB REPO user/name>'
|
124
|
+
github.token = '<ADD GITHUB TOKEN>'
|
125
|
+
end
|
126
|
+
end
|
127
|
+
}
|
128
|
+
file.write(script)
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.init name
|
132
|
+
@@name = name
|
133
|
+
abort("Error: #{@@name} already exists.") unless !File.exists? name
|
134
|
+
FileUtils.mkdir_p @@name
|
135
|
+
Dir.chdir @@name do
|
136
|
+
puts "Creating experiment #{@@name}..."
|
137
|
+
File.open('.xanthus', 'w+') do |f|
|
138
|
+
self.header f
|
139
|
+
self.config f
|
140
|
+
end
|
141
|
+
end
|
142
|
+
puts 'Experiment created.'
|
143
|
+
puts "Edit #{@@name}/.xanthus to configure your experiment."
|
144
|
+
puts 'To run your experiment "xanthus run".'
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
data/lib/xanthus/job.rb
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Xanthus
|
4
|
+
class Job
|
5
|
+
attr_accessor :name
|
6
|
+
attr_accessor :iterations
|
7
|
+
attr_accessor :tasks
|
8
|
+
attr_accessor :outputs
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@iterations = 0
|
12
|
+
@tasks = Hash.new
|
13
|
+
@outputs = Hash.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def output_script outputs
|
17
|
+
script = ''
|
18
|
+
outputs.each do |name, path|
|
19
|
+
script += "cp -f #{path} /vagrant/output/#{name}.data\n"
|
20
|
+
end
|
21
|
+
return script
|
22
|
+
end
|
23
|
+
|
24
|
+
def setup_env machine, scripts, config
|
25
|
+
puts 'Setting up task on machine '+machine.to_s+'...'
|
26
|
+
script = ''
|
27
|
+
scripts.each do |t|
|
28
|
+
v = eval(config.scripts[t])
|
29
|
+
if v.kind_of?(Array)
|
30
|
+
v.each do |w|
|
31
|
+
script+=w+"\n"
|
32
|
+
end
|
33
|
+
else
|
34
|
+
script+=v
|
35
|
+
end
|
36
|
+
end
|
37
|
+
script += self.output_script(@outputs[machine]) unless @outputs[machine].nil?
|
38
|
+
|
39
|
+
script_to_clean = script
|
40
|
+
script = ''
|
41
|
+
script_to_clean.each_line do |s|
|
42
|
+
script += s.strip + "\n" unless s=="\n"
|
43
|
+
end
|
44
|
+
script = script.gsub "\n\n", "\n"
|
45
|
+
|
46
|
+
FileUtils.mkdir_p machine.to_s
|
47
|
+
Dir.chdir machine.to_s do
|
48
|
+
FileUtils.mkdir_p 'output'
|
49
|
+
puts 'Creating provision files...'
|
50
|
+
File.open('Vagrantfile', 'w+') do |f|
|
51
|
+
f.write(config.vms[machine].to_vagrant)
|
52
|
+
end
|
53
|
+
File.open('provision.sh', 'w+') do |f|
|
54
|
+
f.write(script)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def run machine
|
60
|
+
Dir.chdir machine.to_s do
|
61
|
+
system('vagrant', 'up')
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def halt machine
|
66
|
+
Dir.chdir machine.to_s do
|
67
|
+
system('vagrant', 'halt')
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def destroy machine
|
72
|
+
Dir.chdir machine.to_s do
|
73
|
+
system('vagrant', 'destroy', '-f')
|
74
|
+
system('rm', '-rf', '.vagrant')
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def execute config, iteration
|
79
|
+
puts "Running job #{name.to_s}-#{iteration.to_s}..."
|
80
|
+
FileUtils.mkdir_p 'tmp'
|
81
|
+
Dir.chdir 'tmp' do
|
82
|
+
@tasks.each do |machine, templates|
|
83
|
+
self.setup_env machine, templates, config
|
84
|
+
end
|
85
|
+
@tasks.each do |machine, templates|
|
86
|
+
self.run machine
|
87
|
+
end
|
88
|
+
@tasks.each do |machine, templates|
|
89
|
+
self.halt machine
|
90
|
+
end
|
91
|
+
@tasks.each do |machine, templates|
|
92
|
+
self.destroy machine
|
93
|
+
end
|
94
|
+
end
|
95
|
+
system('mv', 'tmp', "#{name.to_s}-#{iteration.to_s}")
|
96
|
+
system('tar', '-czvf', "#{name.to_s}-#{iteration.to_s}.tar.gz", "#{name.to_s}-#{iteration.to_s}")
|
97
|
+
system('rm', '-rf', "#{name.to_s}-#{iteration.to_s}")
|
98
|
+
config.github_conf.add("#{name.to_s}-#{iteration.to_s}.tar.gz") unless config.github_conf.nil?
|
99
|
+
config.github_conf.push unless config.github_conf.nil?
|
100
|
+
puts "Job #{name.to_s}-#{iteration.to_s} done."
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Xanthus
|
2
|
+
class VirtualMachine
|
3
|
+
attr_accessor :name
|
4
|
+
attr_accessor :box
|
5
|
+
attr_accessor :version
|
6
|
+
attr_accessor :cpus
|
7
|
+
attr_accessor :cpu_cap
|
8
|
+
attr_accessor :memory
|
9
|
+
attr_accessor :ip
|
10
|
+
attr_accessor :gui
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@name = :default
|
14
|
+
@box = 'jhcook/fedora27'
|
15
|
+
@version = '4.13.12.300'
|
16
|
+
@ip = '192.168.33.8'
|
17
|
+
@memory = 4096
|
18
|
+
@cpus = 2
|
19
|
+
@cpu_cap = 70
|
20
|
+
@gui = false
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_vagrant
|
24
|
+
%Q{
|
25
|
+
Vagrant.configure(2) do |config|
|
26
|
+
config.vm.box = "#{@box}"
|
27
|
+
config.vm.box_version = "#{@version}"
|
28
|
+
config.vm.network "private_network", ip: "#{@ip}"
|
29
|
+
|
30
|
+
config.vm.provider "virtualbox" do |vb|
|
31
|
+
vb.gui = #{@gui}
|
32
|
+
vb.memory = #{@memory}
|
33
|
+
vb.customize ["modifyvm", :id, "--cpuexecutioncap", "#{@cpu_cap}"]
|
34
|
+
vb.cpus = #{@cpus}
|
35
|
+
vb.name = "#{@name}"
|
36
|
+
end
|
37
|
+
config.vm.provision "shell", path: "provision.sh"
|
38
|
+
end
|
39
|
+
}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
metadata
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: xanthus
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Thomas Pasquier
|
8
|
+
- Xueyuan "Michael" Han
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2019-03-05 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '2.0'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '2.0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '10.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '10.0'
|
42
|
+
description: Automated intrusion detection dataset generation framework.
|
43
|
+
email:
|
44
|
+
- thomas.pasquier@bristol.ac.uk
|
45
|
+
executables:
|
46
|
+
- xanthus
|
47
|
+
extensions: []
|
48
|
+
extra_rdoc_files: []
|
49
|
+
files:
|
50
|
+
- LICENSE
|
51
|
+
- README.md
|
52
|
+
- bin/xanthus
|
53
|
+
- lib/xanthus.rb
|
54
|
+
- lib/xanthus/configuration.rb
|
55
|
+
- lib/xanthus/default.rb
|
56
|
+
- lib/xanthus/github.rb
|
57
|
+
- lib/xanthus/init.rb
|
58
|
+
- lib/xanthus/job.rb
|
59
|
+
- lib/xanthus/version.rb
|
60
|
+
- lib/xanthus/virtual_machine.rb
|
61
|
+
homepage: http://camflow.org
|
62
|
+
licenses:
|
63
|
+
- MIT
|
64
|
+
metadata: {}
|
65
|
+
post_install_message:
|
66
|
+
rdoc_options: []
|
67
|
+
require_paths:
|
68
|
+
- lib
|
69
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
requirements: []
|
80
|
+
rubyforge_project:
|
81
|
+
rubygems_version: 2.7.6
|
82
|
+
signing_key:
|
83
|
+
specification_version: 4
|
84
|
+
summary: Automated intrusion detection dataset generation framework.
|
85
|
+
test_files: []
|