dap 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ed9f67bd94ba672f1edc3bc992267aacc039211
-  data.tar.gz: c090c26ebc17f4e16a9eada5c36410f273c8a7dc
+  metadata.gz: 9102bebe944c3db1ab080d8bfae9990a1bcd4830
+  data.tar.gz: a3baf5ac569e0bf6871f12a7fa11aa8c6911135c
 SHA512:
-  metadata.gz: 6e28f7b1cdbdc6587cc686fe8f77f078986128956ea629da49cbd6cf4a6e1d0be04c3739eeb07e41cc664015ad2278c311ba79ce672940bca32c9a7c68df7309
-  data.tar.gz: cb5887d3c09fc28a2476553cb30ae42e42405d9e9edab5af41ce6121885b02a30139363c13b2a42577170314b24a8110352a4ad326ed0370df833c9cb29693a3
+  metadata.gz: 54bd662a9ac9363f2a5bca07abf398796310cedd08a97f1f90619146699a8288ff230e78f636c6b108aeadcabf1b54e50b87309ed13b0dc1cd94070dc6007409
+  data.tar.gz: a16f006e782180fbb418f1f9032fb361bca480dac9b6b2724f6d85e0c314b048a85032a07523cb48138fc6704d9065149dce3f8c8be7b323463d160265cbc6f0

data/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "test/test_helper/bats-support"]
+	path = test/test_helper/bats-support
+	url = https://github.com/ztombol/bats-support
+[submodule "test/test_helper/bats-assert"]
+	path = test/test_helper/bats-assert
+	url = https://github.com/ztombol/bats-assert

data/.travis.yml

@@ -1,12 +1,11 @@
-language: ruby
-cache: bundler
-sudo: false
-rvm:
-  - 2.4.5
-  - 2.5.3
-addons:
-  apt:
-    packages:
-    - libgeoip-dev
-    - libgeoip1
-script: bundle exec rspec spec
+language: generic
+
+services:
+  - docker
+
+before_install:
+  - docker build -t dap_testing -f Dockerfile.testing .
+
+script:
+  - docker build -t dap_testing -f Dockerfile.testing .
+  - docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/test_helper/ | xargs -n1 bats"

data/CONTRIBUTING.md

@@ -81,7 +81,21 @@ Finally, submit the PR.  Navigate to ```https://github.com/<your-github-username
 
 ### Testing
 
-When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are encourage to perform testing _before_ submitting the PR.  To do this, simply run `bundle exec rspec spec`.
+When your PR is submitted, it will be automatically subjected to the full run
+of tests in [Travis](https://travis-ci.org/rapid7/dap/), however you are
+encourage to perform testing _before_ submitting the PR.  There are two types of tests in place:
+run `bundle exec rspec spec`.  # Testing
+
+There are two testing frameworks in place.
+
+* Ruby `rspec`
+* [bats](https://github.com/sstephenson/bats) integration tests
+
+To run these outside of travis-ci, run:
+```
+docker build -t dap_testing -f Dockerfile.testing . && \
+docker run --rm --name dap_testing -it -e DAP_EXECUTABLE=dap dap_testing /bin/bash -l -c "rvm use 2.4.5 && gem build dap && gem install dap*.gem && bundle exec rspec spec && find /opt/bats_testing -name \*.bats | grep -v test/ test_helper/ | xargs -n1 bats"
+```
 
 ## Landing PRs
 

data/Dockerfile.testing

@@ -0,0 +1,29 @@
+FROM ubuntu:18.04
+
+ENV TEST_DIR /opt/bats_testing
+RUN apt-get update
+RUN apt-get install -y build-essential ca-certificates curl git jq libffi-dev libgeoip-dev libxml2-dev wget zlib1g-dev
+
+# install rvm and necessary ruby bits
+RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import -
+RUN curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
+RUN curl -sSL https://get.rvm.io | bash -s stable
+RUN /bin/bash -l -c "rvm requirements"
+RUN /bin/bash -l -c "rvm install 2.4.5"
+RUN /bin/bash -l -c "rvm use 2.4.5 && gem update --system && gem install bundler"
+ADD Gemfile* $TEST_DIR/
+RUN /bin/bash -l -c "cd $TEST_DIR && rvm use 2.4.5 && bundle install"
+
+# install maxmind data
+RUN mkdir /var/lib/geoip
+WORKDIR /var/lib/geoip
+RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIP.dat
+RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPCity.dat -O GeoLiteCity.dat
+RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPASNum.dat
+RUN wget -q https://github.com/maxmind/geoip-api-php/raw/master/tests/data/GeoIPOrg.dat -O geoip_org.dat
+
+# install bats
+RUN git clone https://github.com/sstephenson/bats.git && cd bats && ./install.sh /usr
+
+WORKDIR /opt/bats_testing
+COPY . .

data/lib/dap/version.rb

@@ -1,3 +1,3 @@
 module Dap
-  VERSION = "1.0.1"
+  VERSION = "1.0.2"
 end

data/test/filters.bats

@@ -0,0 +1,130 @@
+#!/usr/bin/env bats
+
+load ./test_common
+
+@test "rename" {
+  run bash -c 'echo world | $DAP_EXECUTABLE lines + rename line=hello + json'
+  assert_success
+  assert_output '{"hello":"world"}'
+}
+
+@test "not_exists" {
+  run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
+  assert_success
+  assert_output ''
+  run bash -c "echo '{\"bar\":\"bar\"}' | $DAP_EXECUTABLE json + not_exists foo + json"
+  assert_success
+  assert_output '{"bar":"bar"}'
+}
+
+@test "split_comma" {
+  run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + split_comma foo + json | jq -Sc ."
+  assert_success
+  assert_line --index 0 '{"foo":"bar,baz","foo.word":"bar"}'
+  assert_line --index 1 '{"foo":"bar,baz","foo.word":"baz"}'
+}
+
+@test "field_split_line" {
+  run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + field_split_line foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"bar\nbaz","foo.f1":"bar","foo.f2":"baz"}'
+}
+
+@test  "not_empty" {
+  # only exists in godap currently
+  skip
+  run bash -c "echo '{\"foo\":\"bar,baz\"}' | $DAP_EXECUTABLE json + not_empty foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"bar,baz"}'
+}
+
+@test "field_split_tab" {
+  run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + field_split_tab foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"bar\tbaz","foo.f1":"bar","foo.f2":"baz"}'
+}
+
+@test "truncate" {
+  run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + truncate foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":""}'
+}
+
+@test "insert" {
+  run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + insert a=b + json | jq -Sc ."
+  assert_success
+  assert_output '{"a":"b","foo":"bar\tbaz"}'
+}
+
+@test "field_split_array" {
+  run bash -c "echo '{\"foo\":[\"a\",2]}' | $DAP_EXECUTABLE json + field_split_array foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":["a",2],"foo.f1":"a","foo.f2":2}'
+}
+
+@test "exists" {
+  run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists a + json | jq -Sc ."
+  assert_success
+  assert_output ''
+  run bash -c "echo '{\"foo\":\"bar\tbaz\"}' | $DAP_EXECUTABLE json + exists foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"bar\tbaz"}'
+}
+
+@test "split_line" {
+  run bash -c "echo '{\"foo\":\"bar\nbaz\"}' | $DAP_EXECUTABLE json + split_line foo + json | jq -Sc ."
+  assert_success
+  assert_line --index 0 '{"foo":"bar\nbaz","foo.line":"bar"}'
+  assert_line --index 1 '{"foo":"bar\nbaz","foo.line":"baz"}'
+}
+
+@test "select" {
+  run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"bar"}'
+  run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + select foo baz + json | jq -Sc ."
+  assert_success
+  assert_output '{"baz":"qux","foo":"bar"}'
+}
+
+@test "remove" {
+  run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + remove foo baz + json | jq -Sc ."
+  assert_success
+  assert_output '{"a":"b"}'
+}
+
+@test "include" {
+  run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=c + json | jq -Sc ."
+  assert_success
+  assert_output ''
+  run bash -c "echo '{\"foo\":\"bar\", \"baz\":\"qux\", \"a\":\"b\"}' | $DAP_EXECUTABLE json + include a=b + json | jq -Sc ."
+  assert_success
+  assert_output '{"a":"b","baz":"qux","foo":"bar"}'
+}
+
+@test "transform" {
+  run bash -c "echo '{\"foo\":\"bar\"}' | $DAP_EXECUTABLE json + transform foo=base64encode + json | jq -Sc ."
+  assert_success
+  assert_output '{"foo":"YmFy"}'
+}
+
+@test "recog_match" {
+  # currently differs from godap, need to figure out which is correct
+  skip
+  run bash -c "echo '9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
+  assert_success
+  assert_output '{"line":"9.8.2rc1-RedHat-9.8.2-0.62.rc1.el6_9.2","line.recog.os.cpe23":"cpe:/o:redhat:enterprise_linux:6","line.recog.os.family":"Linux","line.recog.os.product":"Enterprise Linux","line.recog.os.vendor":"Red Hat","line.recog.os.version":"6","line.recog.os.version.version":"9","line.recog.service.cpe23":"cpe:/a:isc:bind:9.8.2rc1","line.recog.service.family":"BIND","line.recog.service.product":"BIND","line.recog.service.vendor":"ISC","line.recog.service.version":"9.8.2rc1"}'
+}
+
+@test "recog_nomatch" {
+  run bash -c "echo 'should not match' | $DAP_EXECUTABLE lines + recog line=dns.versionbind + json | jq -Sc ."
+  assert_success
+  assert_output '{"line":"should not match"}'
+}
+
+@test "recog_invalid_arg" {
+  # currently fails in dap, passes in godap
+  skip
+  run bash -c "echo 'test' | $DAP_EXECUTABLE lines + recog + json"
+  assert_failure
+}

data/test/inputs.bats

@@ -0,0 +1,15 @@
+#!/usr/bin/env bats
+
+load ./test_common
+
+@test "reads json" {
+  run bash -c 'echo "{\"foo\": 1 }" | dap json + json'
+  assert_success
+  assert_output '{"foo":1}'
+}
+
+@test "reads lines" {
+  run bash -c 'echo hello world | dap lines + json'
+  assert_success
+  assert_output '{"line":"hello world"}'
+}

data/test/test_common.bash

@@ -0,0 +1,24 @@
+TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+load "${TEST_DIR}/test_helper/bats-support/load.bash"
+load "${TEST_DIR}/test_helper/bats-assert/load.bash"
+
+function setup_workdir() {
+  WORK_DIR=`mktemp -d /tmp/output.XXXXXX`
+}
+
+function teardown_workdir() {
+  cd
+  if [ -z "${DISABLE_BATS_TEARDOWN}" ]; then
+    test -d $WORK_DIR && rm -Rf $WORK_DIR
+  fi
+}
+
+function setup() {
+  setup_workdir
+}
+
+function teardown() {
+  teardown_workdir
+}
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -163,10 +163,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitmodules"
 - ".rspec"
 - ".travis.yml"
 - CONTRIBUTING.md
 - Dockerfile
+- Dockerfile.testing
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -230,6 +232,9 @@ files:
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb
 - spec/spec_helper.rb
+- test/filters.bats
+- test/inputs.bats
+- test/test_common.bash
 - tools/geo-ip-summary.rb
 - tools/ipmi-vulns.rb
 - tools/json-summarize.rb
@@ -255,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14.3
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: 'DAP: The Data Analysis Pipeline'
@@ -269,3 +274,6 @@ test_files:
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb
 - spec/spec_helper.rb
+- test/filters.bats
+- test/inputs.bats
+- test/test_common.bash