dap 1.2.9 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +5 -2
- data/CONTRIBUTING.md +3 -5
- data/Dockerfile.testing +4 -5
- data/Gemfile +1 -13
- data/Gemfile.lock +86 -85
- data/README.md +1 -2
- data/Rakefile +2 -2
- data/dap.gemspec +3 -1
- data/lib/dap/version.rb +1 -1
- metadata +32 -5
- data/.travis.yml +0 -11
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e1a23a5661dc9029223cea577a085c1089f639bd97d8ab32f29f0be9d7ad7c61
|
|
4
|
+
data.tar.gz: a4ebe748a2a14c6e48722ade5886ee5c87c23572dfbf9e06b66275699b981a69
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4eaf16735fcac8f8096c8656888fda8c1214e85aad436905da7269e77792aeebeaba3e54436c6fc07f16f1a84ece982c17e48fbec09bc307a13597d06efed15a
|
|
7
|
+
data.tar.gz: 67fc3b87dc8ceca8bcb10a6719a3b0bc7263a3d8f0f13a09b89b2bae065c8c696234f39c65e461114cb15275ae7a25c0af88b4f37d24a6089a37a605085b48ae
|
data/.gitignore
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
# Ruby and tooling specific
|
|
2
|
+
.yardoc
|
|
3
|
+
doc/
|
|
4
|
+
/pkg/
|
|
5
|
+
|
|
1
6
|
# Ignore rvm files
|
|
2
7
|
.ruby-version
|
|
3
8
|
.ruby-gemset
|
|
@@ -8,8 +13,6 @@ data/GeoLite2-ASN.mmdb
|
|
|
8
13
|
data/GeoLite2-City.mmdb
|
|
9
14
|
data/GeoLite2-Country.mmdb
|
|
10
15
|
|
|
11
|
-
/pkg/
|
|
12
|
-
|
|
13
16
|
## Environment normalization:
|
|
14
17
|
/.bundle/
|
|
15
18
|
/vendor/bundle
|
data/CONTRIBUTING.md
CHANGED
|
@@ -81,9 +81,7 @@ Finally, submit the PR. Navigate to ```https://github.com/<your-github-username
|
|
|
81
81
|
|
|
82
82
|
### Testing
|
|
83
83
|
|
|
84
|
-
|
|
85
|
-
of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are
|
|
86
|
-
encourage to perform testing _before_ submitting the PR. There are two types of tests in place:
|
|
84
|
+
You are encourage to perform testing _before_ submitting the PR. There are two types of tests in place:
|
|
87
85
|
run `bundle exec rspec spec`. # Testing
|
|
88
86
|
|
|
89
87
|
There are two testing frameworks in place.
|
|
@@ -91,10 +89,10 @@ There are two testing frameworks in place.
|
|
|
91
89
|
* Ruby `rspec`
|
|
92
90
|
* [bats](https://github.com/sstephenson/bats) integration tests
|
|
93
91
|
|
|
94
|
-
To run these
|
|
92
|
+
To run these tests locally, run:
|
|
95
93
|
```
|
|
96
94
|
docker build -t dap_testing -f Dockerfile.testing . && \
|
|
97
|
-
docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.
|
|
95
|
+
docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.7.6 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/test_helper/ | xargs -n1 bats"
|
|
98
96
|
```
|
|
99
97
|
|
|
100
98
|
## Landing PRs
|
data/Dockerfile.testing
CHANGED
|
@@ -11,10 +11,10 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import -
|
|
|
11
11
|
RUN curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
|
|
12
12
|
RUN curl -sSL https://get.rvm.io | bash -s stable
|
|
13
13
|
RUN /bin/bash -l -c "rvm requirements"
|
|
14
|
-
RUN /bin/bash -l -c "rvm install 2.
|
|
15
|
-
RUN /bin/bash -l -c "rvm use 2.
|
|
16
|
-
|
|
17
|
-
RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.
|
|
14
|
+
RUN /bin/bash -l -c "rvm install 2.7.6"
|
|
15
|
+
RUN /bin/bash -l -c "rvm use 2.7.6 && gem update --system && gem install bundler"
|
|
16
|
+
COPY . $TEST_DIR/
|
|
17
|
+
RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.7.6 && bundle update --bundler && bundle install"
|
|
18
18
|
|
|
19
19
|
# install maxmind legacy data
|
|
20
20
|
RUN mkdir /var/lib/geoip
|
|
@@ -36,4 +36,3 @@ COPY test/test_data/geoip2/*.mmdb /var/lib/geoip2/
|
|
|
36
36
|
RUN git clone https://github.com/sstephenson/bats.git && cd bats && ./install.sh /usr
|
|
37
37
|
|
|
38
38
|
WORKDIR /opt/bats_testing
|
|
39
|
-
COPY . .
|
data/Gemfile
CHANGED
|
@@ -1,15 +1,3 @@
|
|
|
1
1
|
source 'https://rubygems.org'
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
gem 'htmlentities'
|
|
5
|
-
gem 'net-dns'
|
|
6
|
-
gem 'bit-struct'
|
|
7
|
-
gem 'geoip-c'
|
|
8
|
-
gem 'maxmind-db', '~> 1.0.0'
|
|
9
|
-
gem 'recog', '>=2.3.21'
|
|
10
|
-
|
|
11
|
-
group :test do
|
|
12
|
-
gem 'rspec', '~> 3.9.0'
|
|
13
|
-
gem 'cucumber', '~> 4.0.0'
|
|
14
|
-
gem 'aruba', '~> 0.6.2'
|
|
15
|
-
end
|
|
3
|
+
gemspec
|
data/Gemfile.lock
CHANGED
|
@@ -1,102 +1,103 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
dap (1.3.1)
|
|
5
|
+
bit-struct
|
|
6
|
+
geoip-c
|
|
7
|
+
htmlentities
|
|
8
|
+
maxmind-db
|
|
9
|
+
net-dns
|
|
10
|
+
nokogiri
|
|
11
|
+
oj
|
|
12
|
+
recog
|
|
13
|
+
|
|
1
14
|
GEM
|
|
2
15
|
remote: https://rubygems.org/
|
|
3
16
|
specs:
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
rspec-expectations (>= 2.7.0)
|
|
13
|
-
bit-struct (0.16)
|
|
17
|
+
aruba (2.1.0)
|
|
18
|
+
bundler (>= 1.17, < 3.0)
|
|
19
|
+
childprocess (>= 2.0, < 5.0)
|
|
20
|
+
contracts (>= 0.16.0, < 0.18.0)
|
|
21
|
+
cucumber (>= 4.0, < 9.0)
|
|
22
|
+
rspec-expectations (~> 3.4)
|
|
23
|
+
thor (~> 1.0)
|
|
24
|
+
bit-struct (0.17)
|
|
14
25
|
builder (3.2.4)
|
|
15
|
-
childprocess (4.
|
|
16
|
-
|
|
17
|
-
cucumber (
|
|
18
|
-
builder (~> 3.2, >= 3.2.
|
|
19
|
-
cucumber-
|
|
20
|
-
cucumber-
|
|
21
|
-
cucumber-
|
|
22
|
-
cucumber-
|
|
23
|
-
cucumber-
|
|
24
|
-
cucumber-
|
|
25
|
-
diff-lcs (~> 1.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
cucumber-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
cucumber-
|
|
36
|
-
cucumber-messages (~>
|
|
37
|
-
cucumber-
|
|
38
|
-
|
|
39
|
-
cucumber-
|
|
40
|
-
cucumber-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
cucumber-messages (~> 12.1, >= 12.1.1)
|
|
44
|
-
diff-lcs (1.3)
|
|
45
|
-
ffi (1.13.1)
|
|
26
|
+
childprocess (4.1.0)
|
|
27
|
+
contracts (0.16.1)
|
|
28
|
+
cucumber (8.0.0)
|
|
29
|
+
builder (~> 3.2, >= 3.2.4)
|
|
30
|
+
cucumber-ci-environment (~> 9.0, >= 9.0.4)
|
|
31
|
+
cucumber-core (~> 11.0, >= 11.0.0)
|
|
32
|
+
cucumber-cucumber-expressions (~> 15.1, >= 15.1.1)
|
|
33
|
+
cucumber-gherkin (~> 23.0, >= 23.0.1)
|
|
34
|
+
cucumber-html-formatter (~> 19.1, >= 19.1.0)
|
|
35
|
+
cucumber-messages (~> 18.0, >= 18.0.0)
|
|
36
|
+
diff-lcs (~> 1.5, >= 1.5.0)
|
|
37
|
+
mime-types (~> 3.4, >= 3.4.1)
|
|
38
|
+
multi_test (~> 1.1, >= 1.1.0)
|
|
39
|
+
sys-uname (~> 1.2, >= 1.2.2)
|
|
40
|
+
cucumber-ci-environment (9.1.0)
|
|
41
|
+
cucumber-core (11.0.0)
|
|
42
|
+
cucumber-gherkin (~> 23.0, >= 23.0.1)
|
|
43
|
+
cucumber-messages (~> 18.0, >= 18.0.0)
|
|
44
|
+
cucumber-tag-expressions (~> 4.1, >= 4.1.0)
|
|
45
|
+
cucumber-cucumber-expressions (15.2.0)
|
|
46
|
+
cucumber-gherkin (23.0.1)
|
|
47
|
+
cucumber-messages (~> 18.0, >= 18.0.0)
|
|
48
|
+
cucumber-html-formatter (19.2.0)
|
|
49
|
+
cucumber-messages (~> 18.0, >= 18.0.0)
|
|
50
|
+
cucumber-messages (18.0.0)
|
|
51
|
+
cucumber-tag-expressions (4.1.0)
|
|
52
|
+
diff-lcs (1.5.0)
|
|
53
|
+
ffi (1.15.5)
|
|
46
54
|
geoip-c (0.9.1)
|
|
47
55
|
htmlentities (4.3.4)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
mini_portile2 (2.
|
|
53
|
-
|
|
54
|
-
multi_test (0.1.2)
|
|
56
|
+
maxmind-db (1.1.1)
|
|
57
|
+
mime-types (3.4.1)
|
|
58
|
+
mime-types-data (~> 3.2015)
|
|
59
|
+
mime-types-data (3.2022.0105)
|
|
60
|
+
mini_portile2 (2.8.0)
|
|
61
|
+
multi_test (1.1.0)
|
|
55
62
|
net-dns (0.9.0)
|
|
56
|
-
nokogiri (1.
|
|
57
|
-
mini_portile2 (~> 2.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
thread_safe
|
|
64
|
-
recog (2.3.21)
|
|
63
|
+
nokogiri (1.13.9)
|
|
64
|
+
mini_portile2 (~> 2.8.0)
|
|
65
|
+
racc (~> 1.4)
|
|
66
|
+
oj (3.13.21)
|
|
67
|
+
racc (1.6.0)
|
|
68
|
+
rake (13.0.6)
|
|
69
|
+
recog (3.0.3)
|
|
65
70
|
nokogiri
|
|
66
|
-
rspec (3.
|
|
67
|
-
rspec-core (~> 3.
|
|
68
|
-
rspec-expectations (~> 3.
|
|
69
|
-
rspec-mocks (~> 3.
|
|
70
|
-
rspec-core (3.
|
|
71
|
-
rspec-support (~> 3.
|
|
72
|
-
rspec-expectations (3.
|
|
71
|
+
rspec (3.11.0)
|
|
72
|
+
rspec-core (~> 3.11.0)
|
|
73
|
+
rspec-expectations (~> 3.11.0)
|
|
74
|
+
rspec-mocks (~> 3.11.0)
|
|
75
|
+
rspec-core (3.11.0)
|
|
76
|
+
rspec-support (~> 3.11.0)
|
|
77
|
+
rspec-expectations (3.11.1)
|
|
73
78
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
74
|
-
rspec-support (~> 3.
|
|
75
|
-
rspec-mocks (3.
|
|
79
|
+
rspec-support (~> 3.11.0)
|
|
80
|
+
rspec-mocks (3.11.1)
|
|
76
81
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
77
|
-
rspec-support (~> 3.
|
|
78
|
-
rspec-support (3.
|
|
79
|
-
sys-uname (1.2.
|
|
80
|
-
ffi (
|
|
81
|
-
thor (1.
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
82
|
+
rspec-support (~> 3.11.0)
|
|
83
|
+
rspec-support (3.11.1)
|
|
84
|
+
sys-uname (1.2.2)
|
|
85
|
+
ffi (~> 1.1)
|
|
86
|
+
thor (1.2.1)
|
|
87
|
+
webrick (1.7.0)
|
|
88
|
+
yard (0.9.28)
|
|
89
|
+
webrick (~> 1.7.0)
|
|
85
90
|
|
|
86
91
|
PLATFORMS
|
|
87
92
|
ruby
|
|
88
93
|
|
|
89
94
|
DEPENDENCIES
|
|
90
|
-
aruba
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
net-dns
|
|
97
|
-
oj
|
|
98
|
-
recog (>= 2.3.21)
|
|
99
|
-
rspec (~> 3.9.0)
|
|
95
|
+
aruba
|
|
96
|
+
cucumber
|
|
97
|
+
dap!
|
|
98
|
+
rake
|
|
99
|
+
rspec
|
|
100
|
+
yard
|
|
100
101
|
|
|
101
102
|
BUNDLED WITH
|
|
102
|
-
|
|
103
|
+
2.3.24
|
data/README.md
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# DAP: The Data Analysis Pipeline
|
|
2
2
|
|
|
3
3
|
[](http://badge.fury.io/rb/dap)
|
|
4
|
-
[](https://travis-ci.org/rapid7/dap)
|
|
5
4
|
|
|
6
5
|
DAP was created to transform text-based data on the command-line, specializing in transforms that are annoying or difficult to do with existing tools.
|
|
7
6
|
|
|
@@ -13,7 +12,7 @@ DAP was written to process terabyte-sized public scan datasets, such as those pr
|
|
|
13
12
|
|
|
14
13
|
### Prerequisites
|
|
15
14
|
|
|
16
|
-
DAP requires Ruby and is best suited for systems with a relatively current version with 2.
|
|
15
|
+
DAP requires Ruby and is best suited for systems with a relatively current version with 2.6.x being the minimum requirement.
|
|
17
16
|
Ideally, this will be managed with either
|
|
18
17
|
[`rbenv`](https://github.com/rbenv/rbenv) or [`rvm`](https://rvm.io/) with the bundler gem also installed and up to date.
|
|
19
18
|
Using system managed/installed Rubies is possible but fraught with peril.
|
data/Rakefile
CHANGED
|
@@ -2,7 +2,7 @@ require "bundler/gem_tasks"
|
|
|
2
2
|
|
|
3
3
|
require 'rspec/core/rake_task'
|
|
4
4
|
RSpec::Core::RakeTask.new do |t|
|
|
5
|
-
t.pattern =
|
|
5
|
+
t.pattern = 'spec/**/*_spec.rb'
|
|
6
6
|
end
|
|
7
7
|
|
|
8
8
|
require 'yard'
|
|
@@ -15,7 +15,7 @@ require 'cucumber'
|
|
|
15
15
|
require 'cucumber/rake/task'
|
|
16
16
|
|
|
17
17
|
Cucumber::Rake::Task.new(:features) do |t|
|
|
18
|
-
t.cucumber_opts =
|
|
18
|
+
t.cucumber_opts = %w(features --format pretty)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
task :default => [ :spec, :features, :yard ]
|
data/dap.gemspec
CHANGED
|
@@ -5,7 +5,7 @@ require 'dap/version'
|
|
|
5
5
|
Gem::Specification.new do |s|
|
|
6
6
|
s.name = 'dap'
|
|
7
7
|
s.version = Dap::VERSION
|
|
8
|
-
s.required_ruby_version = '>= 2.
|
|
8
|
+
s.required_ruby_version = '>= 2.6'
|
|
9
9
|
s.authors = [
|
|
10
10
|
'Rapid7 Research'
|
|
11
11
|
]
|
|
@@ -28,7 +28,9 @@ Gem::Specification.new do |s|
|
|
|
28
28
|
|
|
29
29
|
# ---- Dependencies ----
|
|
30
30
|
|
|
31
|
+
s.add_development_dependency 'rake'
|
|
31
32
|
s.add_development_dependency 'rspec'
|
|
33
|
+
s.add_development_dependency 'yard'
|
|
32
34
|
s.add_development_dependency 'cucumber'
|
|
33
35
|
s.add_development_dependency 'aruba'
|
|
34
36
|
|
data/lib/dap/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dap
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rapid7 Research
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-10-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: rake
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
13
27
|
- !ruby/object:Gem::Dependency
|
|
14
28
|
name: rspec
|
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -24,6 +38,20 @@ dependencies:
|
|
|
24
38
|
- - ">="
|
|
25
39
|
- !ruby/object:Gem::Version
|
|
26
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: yard
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
27
55
|
- !ruby/object:Gem::Dependency
|
|
28
56
|
name: cucumber
|
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -179,7 +207,6 @@ files:
|
|
|
179
207
|
- ".gitignore"
|
|
180
208
|
- ".gitmodules"
|
|
181
209
|
- ".rspec"
|
|
182
|
-
- ".travis.yml"
|
|
183
210
|
- CONTRIBUTING.md
|
|
184
211
|
- Dockerfile
|
|
185
212
|
- Dockerfile.testing
|
|
@@ -275,14 +302,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
275
302
|
requirements:
|
|
276
303
|
- - ">="
|
|
277
304
|
- !ruby/object:Gem::Version
|
|
278
|
-
version: '2.
|
|
305
|
+
version: '2.6'
|
|
279
306
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
280
307
|
requirements:
|
|
281
308
|
- - ">="
|
|
282
309
|
- !ruby/object:Gem::Version
|
|
283
310
|
version: '0'
|
|
284
311
|
requirements: []
|
|
285
|
-
rubygems_version: 3.
|
|
312
|
+
rubygems_version: 3.1.6
|
|
286
313
|
signing_key:
|
|
287
314
|
specification_version: 4
|
|
288
315
|
summary: 'DAP: The Data Analysis Pipeline'
|
data/.travis.yml
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
language: generic
|
|
2
|
-
|
|
3
|
-
services:
|
|
4
|
-
- docker
|
|
5
|
-
|
|
6
|
-
before_install:
|
|
7
|
-
- docker build -t dap_testing -f Dockerfile.testing .
|
|
8
|
-
|
|
9
|
-
script:
|
|
10
|
-
- docker build -t dap_testing -f Dockerfile.testing .
|
|
11
|
-
- docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/test_helper/ | xargs -n1 bats"
|