dap 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1ed9f67bd94ba672f1edc3bc992267aacc039211
4
- data.tar.gz: c090c26ebc17f4e16a9eada5c36410f273c8a7dc
3
+ metadata.gz: 9102bebe944c3db1ab080d8bfae9990a1bcd4830
4
+ data.tar.gz: a3baf5ac569e0bf6871f12a7fa11aa8c6911135c
5
5
  SHA512:
6
- metadata.gz: 6e28f7b1cdbdc6587cc686fe8f77f078986128956ea629da49cbd6cf4a6e1d0be04c3739eeb07e41cc664015ad2278c311ba79ce672940bca32c9a7c68df7309
7
- data.tar.gz: cb5887d3c09fc28a2476553cb30ae42e42405d9e9edab5af41ce6121885b02a30139363c13b2a42577170314b24a8110352a4ad326ed0370df833c9cb29693a3
6
+ metadata.gz: 54bd662a9ac9363f2a5bca07abf398796310cedd08a97f1f90619146699a8288ff230e78f636c6b108aeadcabf1b54e50b87309ed13b0dc1cd94070dc6007409
7
+ data.tar.gz: a16f006e782180fbb418f1f9032fb361bca480dac9b6b2724f6d85e0c314b048a85032a07523cb48138fc6704d9065149dce3f8c8be7b323463d160265cbc6f0
@@ -0,0 +1,6 @@
1
+ [submodule "test/test_helper/bats-support"]
2
+ path = test/test_helper/bats-support
3
+ url = https://github.com/ztombol/bats-support
4
+ [submodule "test/test_helper/bats-assert"]
5
+ path = test/test_helper/bats-assert
6
+ url = https://github.com/ztombol/bats-assert
@@ -1,12 +1,11 @@
1
- language: ruby
2
- cache: bundler
3
- sudo: false
4
- rvm:
5
- - 2.4.5
6
- - 2.5.3
7
- addons:
8
- apt:
9
- packages:
10
- - libgeoip-dev
11
- - libgeoip1
12
- script: bundle exec rspec spec
1
+ language: generic
2
+
3
+ services:
4
+ - docker
5
+
6
+ before_install:
7
+ - docker build -t dap_testing -f Dockerfile.testing .
8
+
9
+ script:
10
+ - docker build -t dap_testing -f Dockerfile.testing .
11
+ - docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/test_helper/ | xargs -n1 bats"
@@ -81,7 +81,21 @@ Finally, submit the PR. Navigate to ```https://github.com/<your-github-username
81
81
 
82
82
  ### Testing
83
83
 
84
- When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are encourage to perform testing _before_ submitting the PR. To do this, simply run `bundle exec rspec spec`.
84
+ When your PR is submitted, it will be automatically subjected to the full run
85
+ of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are
86
+ encourage to perform testing _before_ submitting the PR. There are two types of tests in place:
87
+ run `bundle exec rspec spec`. # Testing
88
+
89
+ There are two testing frameworks in place.
90
+
91
+ * Ruby `rspec`
92
+ * [bats](https://github.com/sstephenson/bats) integration tests
93
+
94
+ To run these outside of travis-ci, run:
95
+ ```
96
+ docker build -t dap_testing -f Dockerfile.testing . && \
97
+ docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/ test_helper/ | xargs -n1 bats"
98
+ ```
85
99
 
86
100
  ## Landing PRs
87
101
 
@@ -0,0 +1,29 @@
1
+ FROM ubuntu:18.04
2
+
3
+ ENV TEST_DIR /opt/bats_testing
4
+ RUN apt-get update
5
+ RUN apt-get install -y build-essential ca-certificates curl git jq libffi-dev libgeoip-dev libxml2-dev wget zlib1g-dev
6
+
7
+ # install rvm and necessary ruby bits
8
+ RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import -
9
+ RUN curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
10
+ RUN curl -sSL https://get.rvm.io | bash -s stable
11
+ RUN /bin/bash -l -c "rvm requirements"
12
+ RUN /bin/bash -l -c "rvm install 2.4.5"
13
+ RUN /bin/bash -l -c "rvm use 2.4.5 && gem update --system && gem install bundler"
14
+ ADD Gemfile* $TEST_DIR/
15
+ RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.4.5 && bundle install"
16
+
17
+ # install maxmind data
18
+ RUN mkdir /var/lib/geoip
19
+ WORKDIR /var/lib/geoip
20
+ RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIP.dat
21
+ RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPCity.dat -O GeoLiteCity.dat
22
+ RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPASNum.dat
23
+ RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPOrg.dat -O geoip_org.dat
24
+
25
+ # install bats
26
+ RUN git clone https://github.com/sstephenson/bats.git && cd bats && ./install.sh /usr
27
+
28
+ WORKDIR /opt/bats_testing
29
+ COPY . .
@@ -1,3 +1,3 @@
1
1
  module Dap
2
- VERSION = "1.0.1"
2
+ VERSION = "1.0.2"
3
3
  end
@@ -0,0 +1,130 @@
1
+ #!/usr/bin/env bats
2
+
3
+ load ./test_common
4
+
5
+ @test "rename" {
6
+ run bash -c 'echo world | $DAP_EXECUTABLE lines + rename line=hello + json'
7
+ assert_success
8
+ assert_output '{"hello":"world"}'
9
+ }
10
+
11
+ @test "not_exists" {
12
+ run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
13
+ assert_success
14
+ assert_output ''
15
+ run bash -c "echo '{\"bar\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
16
+ assert_success
17
+ assert_output '{"bar":"bar"}'
18
+ }
19
+
20
+ @test "split_comma" {
21
+ run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + split_comma foo + json | jq -Sc ."
22
+ assert_success
23
+ assert_line --index 0 '{"foo":"bar,baz","foo.word":"bar"}'
24
+ assert_line --index 1 '{"foo":"bar,baz","foo.word":"baz"}'
25
+ }
26
+
27
+ @test "field_split_line" {
28
+ run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + field_split_line foo + json | jq -Sc ."
29
+ assert_success
30
+ assert_output '{"foo":"bar\nbaz","foo.f1":"bar","foo.f2":"baz"}'
31
+ }
32
+
33
+ @test "not_empty" {
34
+ # only exists in godap currently
35
+ skip
36
+ run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + not_empty foo + json | jq -Sc ."
37
+ assert_success
38
+ assert_output '{"foo":"bar,baz"}'
39
+ }
40
+
41
+ @test "field_split_tab" {
42
+ run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + field_split_tab foo + json | jq -Sc ."
43
+ assert_success
44
+ assert_output '{"foo":"bar\tbaz","foo.f1":"bar","foo.f2":"baz"}'
45
+ }
46
+
47
+ @test "truncate" {
48
+ run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + truncate foo + json | jq -Sc ."
49
+ assert_success
50
+ assert_output '{"foo":""}'
51
+ }
52
+
53
+ @test "insert" {
54
+ run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + insert a=b + json | jq -Sc ."
55
+ assert_success
56
+ assert_output '{"a":"b","foo":"bar\tbaz"}'
57
+ }
58
+
59
+ @test "field_split_array" {
60
+ run bash -c "echo '{\"foo\":[\"a\",2]}' | $DAP_EXECUTABLE json + field_split_array foo + json | jq -Sc ."
61
+ assert_success
62
+ assert_output '{"foo":["a",2],"foo.f1":"a","foo.f2":2}'
63
+ }
64
+
65
+ @test "exists" {
66
+ run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists a + json | jq -Sc ."
67
+ assert_success
68
+ assert_output ''
69
+ run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists foo + json | jq -Sc ."
70
+ assert_success
71
+ assert_output '{"foo":"bar\tbaz"}'
72
+ }
73
+
74
+ @test "split_line" {
75
+ run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + split_line foo + json | jq -Sc ."
76
+ assert_success
77
+ assert_line --index 0 '{"foo":"bar\nbaz","foo.line":"bar"}'
78
+ assert_line --index 1 '{"foo":"bar\nbaz","foo.line":"baz"}'
79
+ }
80
+
81
+ @test "select" {
82
+ run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo + json | jq -Sc ."
83
+ assert_success
84
+ assert_output '{"foo":"bar"}'
85
+ run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo baz + json | jq -Sc ."
86
+ assert_success
87
+ assert_output '{"baz":"qux","foo":"bar"}'
88
+ }
89
+
90
+ @test "remove" {
91
+ run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + remove foo baz + json | jq -Sc ."
92
+ assert_success
93
+ assert_output '{"a":"b"}'
94
+ }
95
+
96
+ @test "include" {
97
+ run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=c + json | jq -Sc ."
98
+ assert_success
99
+ assert_output ''
100
+ run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=b + json | jq -Sc ."
101
+ assert_success
102
+ assert_output '{"a":"b","baz":"qux","foo":"bar"}'
103
+ }
104
+
105
+ @test "transform" {
106
+ run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + transform foo=base64encode + json | jq -Sc ."
107
+ assert_success
108
+ assert_output '{"foo":"YmFy"}'
109
+ }
110
+
111
+ @test "recog_match" {
112
+ # currently differs from godap, need to figure out which is correct
113
+ skip
114
+ run bash -c "echo '9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
115
+ assert_success
116
+ assert_output '{"line":"9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2","line.recog.os.cpe23":"cpe:/o:redhat:enterprise_linux:6","line.recog.os.family":"Linux","line.recog.os.product":"Enterprise Linux","line.recog.os.vendor":"Red Hat","line.recog.os.version":"6","line.recog.os.version.version":"9","line.recog.service.cpe23":"cpe:/a:isc:bind:9.8.2rc1","line.recog.service.family":"BIND","line.recog.service.product":"BIND","line.recog.service.vendor":"ISC","line.recog.service.version":"9.8.2rc1"}'
117
+ }
118
+
119
+ @test "recog_nomatch" {
120
+ run bash -c "echo 'should not match' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
121
+ assert_success
122
+ assert_output '{"line":"should not match"}'
123
+ }
124
+
125
+ @test "recog_invalid_arg" {
126
+ # currently fails in dap, passes in godap
127
+ skip
128
+ run bash -c "echo 'test' | $DAP_EXECUTABLE lines + recog + json"
129
+ assert_failure
130
+ }
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env bats
2
+
3
+ load ./test_common
4
+
5
+ @test "reads json" {
6
+ run bash -c 'echo "{\"foo\": 1 }" | dap json + json'
7
+ assert_success
8
+ assert_output '{"foo":1}'
9
+ }
10
+
11
+ @test "reads lines" {
12
+ run bash -c 'echo hello world | dap lines + json'
13
+ assert_success
14
+ assert_output '{"line":"hello world"}'
15
+ }
@@ -0,0 +1,24 @@
1
+ TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
2
+
3
+ load "${TEST_DIR}/test_helper/bats-support/load.bash"
4
+ load "${TEST_DIR}/test_helper/bats-assert/load.bash"
5
+
6
+ function setup_workdir() {
7
+ WORK_DIR=`mktemp -d /tmp/output.XXXXXX`
8
+ }
9
+
10
+ function teardown_workdir() {
11
+ cd
12
+ if [ -z "${DISABLE_BATS_TEARDOWN}" ]; then
13
+ test -d $WORK_DIR && rm -Rf $WORK_DIR
14
+ fi
15
+ }
16
+
17
+ function setup() {
18
+ setup_workdir
19
+ }
20
+
21
+ function teardown() {
22
+ teardown_workdir
23
+ }
24
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dap
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rapid7 Research
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-03-26 00:00:00.000000000 Z
11
+ date: 2019-04-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -163,10 +163,12 @@ extensions: []
163
163
  extra_rdoc_files: []
164
164
  files:
165
165
  - ".gitignore"
166
+ - ".gitmodules"
166
167
  - ".rspec"
167
168
  - ".travis.yml"
168
169
  - CONTRIBUTING.md
169
170
  - Dockerfile
171
+ - Dockerfile.testing
170
172
  - Gemfile
171
173
  - Gemfile.lock
172
174
  - LICENSE
@@ -230,6 +232,9 @@ files:
230
232
  - spec/dap/proto/ipmi_spec.rb
231
233
  - spec/dap/proto/ldap_proto_spec.rb
232
234
  - spec/spec_helper.rb
235
+ - test/filters.bats
236
+ - test/inputs.bats
237
+ - test/test_common.bash
233
238
  - tools/geo-ip-summary.rb
234
239
  - tools/ipmi-vulns.rb
235
240
  - tools/json-summarize.rb
@@ -255,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
255
260
  version: '0'
256
261
  requirements: []
257
262
  rubyforge_project:
258
- rubygems_version: 2.6.14.3
263
+ rubygems_version: 2.6.11
259
264
  signing_key:
260
265
  specification_version: 4
261
266
  summary: 'DAP: The Data Analysis Pipeline'
@@ -269,3 +274,6 @@ test_files:
269
274
  - spec/dap/proto/ipmi_spec.rb
270
275
  - spec/dap/proto/ldap_proto_spec.rb
271
276
  - spec/spec_helper.rb
277
+ - test/filters.bats
278
+ - test/inputs.bats
279
+ - test/test_common.bash