dap 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +6 -0
- data/.travis.yml +11 -12
- data/CONTRIBUTING.md +15 -1
- data/Dockerfile.testing +29 -0
- data/lib/dap/version.rb +1 -1
- data/test/filters.bats +130 -0
- data/test/inputs.bats +15 -0
- data/test/test_common.bash +24 -0
- metadata +11 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9102bebe944c3db1ab080d8bfae9990a1bcd4830
|
|
4
|
+
data.tar.gz: a3baf5ac569e0bf6871f12a7fa11aa8c6911135c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 54bd662a9ac9363f2a5bca07abf398796310cedd08a97f1f90619146699a8288ff230e78f636c6b108aeadcabf1b54e50b87309ed13b0dc1cd94070dc6007409
|
|
7
|
+
data.tar.gz: a16f006e782180fbb418f1f9032fb361bca480dac9b6b2724f6d85e0c314b048a85032a07523cb48138fc6704d9065149dce3f8c8be7b323463d160265cbc6f0
|
data/.gitmodules
ADDED
data/.travis.yml
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
language:
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
script: bundle exec rspec spec
|
|
1
|
+
language: generic
|
|
2
|
+
|
|
3
|
+
services:
|
|
4
|
+
- docker
|
|
5
|
+
|
|
6
|
+
before_install:
|
|
7
|
+
- docker build -t dap_testing -f Dockerfile.testing .
|
|
8
|
+
|
|
9
|
+
script:
|
|
10
|
+
- docker build -t dap_testing -f Dockerfile.testing .
|
|
11
|
+
- docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/test_helper/ | xargs -n1 bats"
|
data/CONTRIBUTING.md
CHANGED
|
@@ -81,7 +81,21 @@ Finally, submit the PR. Navigate to ```https://github.com/<your-github-username
|
|
|
81
81
|
|
|
82
82
|
### Testing
|
|
83
83
|
|
|
84
|
-
When your PR is submitted, it will be automatically subjected to the full run
|
|
84
|
+
When your PR is submitted, it will be automatically subjected to the full run
|
|
85
|
+
of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are
|
|
86
|
+
encourage to perform testing _before_ submitting the PR. There are two types of tests in place:
|
|
87
|
+
run `bundle exec rspec spec`. # Testing
|
|
88
|
+
|
|
89
|
+
There are two testing frameworks in place.
|
|
90
|
+
|
|
91
|
+
* Ruby `rspec`
|
|
92
|
+
* [bats](https://github.com/sstephenson/bats) integration tests
|
|
93
|
+
|
|
94
|
+
To run these outside of travis-ci, run:
|
|
95
|
+
```
|
|
96
|
+
docker build -t dap_testing -f Dockerfile.testing . && \
|
|
97
|
+
docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/ test_helper/ | xargs -n1 bats"
|
|
98
|
+
```
|
|
85
99
|
|
|
86
100
|
## Landing PRs
|
|
87
101
|
|
data/Dockerfile.testing
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
FROM ubuntu:18.04
|
|
2
|
+
|
|
3
|
+
ENV TEST_DIR /opt/bats_testing
|
|
4
|
+
RUN apt-get update
|
|
5
|
+
RUN apt-get install -y build-essential ca-certificates curl git jq libffi-dev libgeoip-dev libxml2-dev wget zlib1g-dev
|
|
6
|
+
|
|
7
|
+
# install rvm and necessary ruby bits
|
|
8
|
+
RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import -
|
|
9
|
+
RUN curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
|
|
10
|
+
RUN curl -sSL https://get.rvm.io | bash -s stable
|
|
11
|
+
RUN /bin/bash -l -c "rvm requirements"
|
|
12
|
+
RUN /bin/bash -l -c "rvm install 2.4.5"
|
|
13
|
+
RUN /bin/bash -l -c "rvm use 2.4.5 && gem update --system && gem install bundler"
|
|
14
|
+
ADD Gemfile* $TEST_DIR/
|
|
15
|
+
RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.4.5 && bundle install"
|
|
16
|
+
|
|
17
|
+
# install maxmind data
|
|
18
|
+
RUN mkdir /var/lib/geoip
|
|
19
|
+
WORKDIR /var/lib/geoip
|
|
20
|
+
RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIP.dat
|
|
21
|
+
RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPCity.dat -O GeoLiteCity.dat
|
|
22
|
+
RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPASNum.dat
|
|
23
|
+
RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPOrg.dat -O geoip_org.dat
|
|
24
|
+
|
|
25
|
+
# install bats
|
|
26
|
+
RUN git clone https://github.com/sstephenson/bats.git && cd bats && ./install.sh /usr
|
|
27
|
+
|
|
28
|
+
WORKDIR /opt/bats_testing
|
|
29
|
+
COPY . .
|
data/lib/dap/version.rb
CHANGED
data/test/filters.bats
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env bats
|
|
2
|
+
|
|
3
|
+
load ./test_common
|
|
4
|
+
|
|
5
|
+
@test "rename" {
|
|
6
|
+
run bash -c 'echo world | $DAP_EXECUTABLE lines + rename line=hello + json'
|
|
7
|
+
assert_success
|
|
8
|
+
assert_output '{"hello":"world"}'
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
@test "not_exists" {
|
|
12
|
+
run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
|
|
13
|
+
assert_success
|
|
14
|
+
assert_output ''
|
|
15
|
+
run bash -c "echo '{\"bar\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
|
|
16
|
+
assert_success
|
|
17
|
+
assert_output '{"bar":"bar"}'
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
@test "split_comma" {
|
|
21
|
+
run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + split_comma foo + json | jq -Sc ."
|
|
22
|
+
assert_success
|
|
23
|
+
assert_line --index 0 '{"foo":"bar,baz","foo.word":"bar"}'
|
|
24
|
+
assert_line --index 1 '{"foo":"bar,baz","foo.word":"baz"}'
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
@test "field_split_line" {
|
|
28
|
+
run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + field_split_line foo + json | jq -Sc ."
|
|
29
|
+
assert_success
|
|
30
|
+
assert_output '{"foo":"bar\nbaz","foo.f1":"bar","foo.f2":"baz"}'
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
@test "not_empty" {
|
|
34
|
+
# only exists in godap currently
|
|
35
|
+
skip
|
|
36
|
+
run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + not_empty foo + json | jq -Sc ."
|
|
37
|
+
assert_success
|
|
38
|
+
assert_output '{"foo":"bar,baz"}'
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
@test "field_split_tab" {
|
|
42
|
+
run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + field_split_tab foo + json | jq -Sc ."
|
|
43
|
+
assert_success
|
|
44
|
+
assert_output '{"foo":"bar\tbaz","foo.f1":"bar","foo.f2":"baz"}'
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
@test "truncate" {
|
|
48
|
+
run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + truncate foo + json | jq -Sc ."
|
|
49
|
+
assert_success
|
|
50
|
+
assert_output '{"foo":""}'
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@test "insert" {
|
|
54
|
+
run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + insert a=b + json | jq -Sc ."
|
|
55
|
+
assert_success
|
|
56
|
+
assert_output '{"a":"b","foo":"bar\tbaz"}'
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
@test "field_split_array" {
|
|
60
|
+
run bash -c "echo '{\"foo\":[\"a\",2]}' | $DAP_EXECUTABLE json + field_split_array foo + json | jq -Sc ."
|
|
61
|
+
assert_success
|
|
62
|
+
assert_output '{"foo":["a",2],"foo.f1":"a","foo.f2":2}'
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
@test "exists" {
|
|
66
|
+
run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists a + json | jq -Sc ."
|
|
67
|
+
assert_success
|
|
68
|
+
assert_output ''
|
|
69
|
+
run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists foo + json | jq -Sc ."
|
|
70
|
+
assert_success
|
|
71
|
+
assert_output '{"foo":"bar\tbaz"}'
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
@test "split_line" {
|
|
75
|
+
run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + split_line foo + json | jq -Sc ."
|
|
76
|
+
assert_success
|
|
77
|
+
assert_line --index 0 '{"foo":"bar\nbaz","foo.line":"bar"}'
|
|
78
|
+
assert_line --index 1 '{"foo":"bar\nbaz","foo.line":"baz"}'
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
@test "select" {
|
|
82
|
+
run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo + json | jq -Sc ."
|
|
83
|
+
assert_success
|
|
84
|
+
assert_output '{"foo":"bar"}'
|
|
85
|
+
run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo baz + json | jq -Sc ."
|
|
86
|
+
assert_success
|
|
87
|
+
assert_output '{"baz":"qux","foo":"bar"}'
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
@test "remove" {
|
|
91
|
+
run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + remove foo baz + json | jq -Sc ."
|
|
92
|
+
assert_success
|
|
93
|
+
assert_output '{"a":"b"}'
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
@test "include" {
|
|
97
|
+
run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=c + json | jq -Sc ."
|
|
98
|
+
assert_success
|
|
99
|
+
assert_output ''
|
|
100
|
+
run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=b + json | jq -Sc ."
|
|
101
|
+
assert_success
|
|
102
|
+
assert_output '{"a":"b","baz":"qux","foo":"bar"}'
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
@test "transform" {
|
|
106
|
+
run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + transform foo=base64encode + json | jq -Sc ."
|
|
107
|
+
assert_success
|
|
108
|
+
assert_output '{"foo":"YmFy"}'
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
@test "recog_match" {
|
|
112
|
+
# currently differs from godap, need to figure out which is correct
|
|
113
|
+
skip
|
|
114
|
+
run bash -c "echo '9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
|
|
115
|
+
assert_success
|
|
116
|
+
assert_output '{"line":"9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2","line.recog.os.cpe23":"cpe:/o:redhat:enterprise_linux:6","line.recog.os.family":"Linux","line.recog.os.product":"Enterprise Linux","line.recog.os.vendor":"Red Hat","line.recog.os.version":"6","line.recog.os.version.version":"9","line.recog.service.cpe23":"cpe:/a:isc:bind:9.8.2rc1","line.recog.service.family":"BIND","line.recog.service.product":"BIND","line.recog.service.vendor":"ISC","line.recog.service.version":"9.8.2rc1"}'
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
@test "recog_nomatch" {
|
|
120
|
+
run bash -c "echo 'should not match' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
|
|
121
|
+
assert_success
|
|
122
|
+
assert_output '{"line":"should not match"}'
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
@test "recog_invalid_arg" {
|
|
126
|
+
# currently fails in dap, passes in godap
|
|
127
|
+
skip
|
|
128
|
+
run bash -c "echo 'test' | $DAP_EXECUTABLE lines + recog + json"
|
|
129
|
+
assert_failure
|
|
130
|
+
}
|
data/test/inputs.bats
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#!/usr/bin/env bats
|
|
2
|
+
|
|
3
|
+
load ./test_common
|
|
4
|
+
|
|
5
|
+
@test "reads json" {
|
|
6
|
+
run bash -c 'echo "{\"foo\": 1 }" | dap json + json'
|
|
7
|
+
assert_success
|
|
8
|
+
assert_output '{"foo":1}'
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
@test "reads lines" {
|
|
12
|
+
run bash -c 'echo hello world | dap lines + json'
|
|
13
|
+
assert_success
|
|
14
|
+
assert_output '{"line":"hello world"}'
|
|
15
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
|
2
|
+
|
|
3
|
+
load "${TEST_DIR}/test_helper/bats-support/load.bash"
|
|
4
|
+
load "${TEST_DIR}/test_helper/bats-assert/load.bash"
|
|
5
|
+
|
|
6
|
+
function setup_workdir() {
|
|
7
|
+
WORK_DIR=`mktemp -d /tmp/output.XXXXXX`
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function teardown_workdir() {
|
|
11
|
+
cd
|
|
12
|
+
if [ -z "${DISABLE_BATS_TEARDOWN}" ]; then
|
|
13
|
+
test -d $WORK_DIR && rm -Rf $WORK_DIR
|
|
14
|
+
fi
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function setup() {
|
|
18
|
+
setup_workdir
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function teardown() {
|
|
22
|
+
teardown_workdir
|
|
23
|
+
}
|
|
24
|
+
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dap
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rapid7 Research
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-
|
|
11
|
+
date: 2019-04-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rspec
|
|
@@ -163,10 +163,12 @@ extensions: []
|
|
|
163
163
|
extra_rdoc_files: []
|
|
164
164
|
files:
|
|
165
165
|
- ".gitignore"
|
|
166
|
+
- ".gitmodules"
|
|
166
167
|
- ".rspec"
|
|
167
168
|
- ".travis.yml"
|
|
168
169
|
- CONTRIBUTING.md
|
|
169
170
|
- Dockerfile
|
|
171
|
+
- Dockerfile.testing
|
|
170
172
|
- Gemfile
|
|
171
173
|
- Gemfile.lock
|
|
172
174
|
- LICENSE
|
|
@@ -230,6 +232,9 @@ files:
|
|
|
230
232
|
- spec/dap/proto/ipmi_spec.rb
|
|
231
233
|
- spec/dap/proto/ldap_proto_spec.rb
|
|
232
234
|
- spec/spec_helper.rb
|
|
235
|
+
- test/filters.bats
|
|
236
|
+
- test/inputs.bats
|
|
237
|
+
- test/test_common.bash
|
|
233
238
|
- tools/geo-ip-summary.rb
|
|
234
239
|
- tools/ipmi-vulns.rb
|
|
235
240
|
- tools/json-summarize.rb
|
|
@@ -255,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
255
260
|
version: '0'
|
|
256
261
|
requirements: []
|
|
257
262
|
rubyforge_project:
|
|
258
|
-
rubygems_version: 2.6.
|
|
263
|
+
rubygems_version: 2.6.11
|
|
259
264
|
signing_key:
|
|
260
265
|
specification_version: 4
|
|
261
266
|
summary: 'DAP: The Data Analysis Pipeline'
|
|
@@ -269,3 +274,6 @@ test_files:
|
|
|
269
274
|
- spec/dap/proto/ipmi_spec.rb
|
|
270
275
|
- spec/dap/proto/ldap_proto_spec.rb
|
|
271
276
|
- spec/spec_helper.rb
|
|
277
|
+
- test/filters.bats
|
|
278
|
+
- test/inputs.bats
|
|
279
|
+
- test/test_common.bash
|