threatinator 0.1.1

data/spec/threatinator/fetcher_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'threatinator/fetcher'
+
+describe Threatinator::Fetcher do
+  let(:fetcher) { Threatinator::Fetcher.new }
+  describe "#fetch" do
+    it "should raise NotImplementedError because it's not implemented" do
+      expect {fetcher.fetch}.to raise_error(NotImplementedError)
+    end
+  end
+end
+

data/spec/threatinator/fetchers/http_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'threatinator/fetchers/http'
+
+describe Threatinator::Fetchers::Http do
+    let(:url) { "http://foobar.com/bla.json" }
+    let(:fetcher_builder) { lambda { described_class.new(url: url) } }
+    let(:fetcher_builder_different) { lambda { described_class.new(url: "http://foobar.com/sdf.json") } }
+    let(:fetcher) { fetcher_builder.call() }
+
+  it_should_behave_like "a fetcher" do
+    before :each do
+      stub_request(:get, url).to_return(:body => expected_data)
+    end
+  end
+
+  describe "#url" do
+    it "should return the value of the URL" do
+      expect(fetcher.url).to eq(url)
+    end
+  end
+
+  [404, 500].each do |status_code|
+    context "when an HTTP response has a status code of #{status_code}" do
+      it_should_behave_like "a #fetch call that failed" do
+        before :each do
+          stub_request(:get, url)
+            .to_return(:status => status_code)
+        end
+      end
+    end
+  end
+end

data/spec/threatinator/filter_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'threatinator/filter'
+
+describe Threatinator::Filter do
+  let(:filter) { Threatinator::Filter.new }
+  describe "#filter?(record)" do
+    it "should raise NotImplementedError because it's not implemented" do
+      expect {filter.filter?(1234)}.to raise_error(NotImplementedError)
+    end
+  end
+end
+
+

data/spec/threatinator/filters/block_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'threatinator/record'
+require 'threatinator/filters/block'
+
+describe Threatinator::Filters::Block do
+  it_should_behave_like "a filter" do
+    let(:filter_block) {
+      lambda do |record|
+        record.data =~ /^FILTERME$/
+      end
+    }
+    let(:filter) { described_class.new(filter_block) }
+    let(:should_filter) { Threatinator::Record.new("FILTERME") }
+    let(:shouldnt_filter) { Threatinator::Record.new("I'm OK!") }
+  end
+end

data/spec/threatinator/filters/comments_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'threatinator/record'
+require 'threatinator/filters/comments'
+
+describe Threatinator::Filters::Comments do
+  it_should_behave_like "a filter" do
+    let(:filter) { described_class.new() }
+    let(:should_filter) { Threatinator::Record.new("# Here's my comment") }
+    let(:shouldnt_filter) { Threatinator::Record.new("I'm OK # wooo!") }
+  end
+end
+
+

data/spec/threatinator/filters/whitespace_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'threatinator/record'
+require 'threatinator/filters/whitespace'
+
+describe Threatinator::Filters::Whitespace do
+  it_should_behave_like "a filter" do
+    let(:filter) { described_class.new() }
+    let(:should_filter) { Threatinator::Record.new("    ") }
+    let(:shouldnt_filter) { Threatinator::Record.new("    I'm OK!") }
+  end
+end
+

data/spec/threatinator/parser_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'threatinator/parser'
+
+describe Threatinator::Parser do
+  let(:parser) { Threatinator::Parser.new() }
+  describe "#run" do
+    it "should raise NotImplementedError because it's not implemented" do
+      expect { |b| parser.run(double("io"), &b) }.to raise_error(NotImplementedError)
+    end
+  end
+end
+
+

data/spec/threatinator/parsers/csv/parser_spec.rb

@@ -0,0 +1,202 @@
+require 'spec_helper'
+require 'threatinator/parsers/csv/parser'
+require 'stringio'
+require 'threatinator/record'
+
+describe Threatinator::Parsers::CSV::Parser do
+  it_should_behave_like "a parser when compared to an identically configured parser" do
+    let(:parser1) { described_class.new(:row_separator => "X") }
+    let(:parser2) { described_class.new(:row_separator => "X") }
+  end
+  
+  context "two instances with differently configured row_separators" do
+    it_should_behave_like "a parser when compared to a differently configured parser" do
+      let(:parser1) { described_class.new(:row_separator => "X") }
+      let(:parser2) { described_class.new(:row_separator => "Y") }
+    end
+  end
+
+  context "by default" do
+    it "should parse \\n as a row separator" do
+      str = "abc\ndef\nhij"
+      sio = StringIO.new(str)
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new(['abc'], line_number: 1, pos_start: 0, pos_end: 4), 
+        Threatinator::Record.new(['def'], line_number: 2, pos_start: 4, pos_end: 8), 
+        Threatinator::Record.new(['hij'], line_number: 3, pos_start: 8, pos_end: 11), 
+      )
+    end
+
+    it "should parse \\r\\n as a row separator"  do
+      str = "abc\r\ndef\r\nhij"
+      sio = StringIO.new(str)
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new(['abc'], line_number: 1, pos_start: 0, pos_end: 5), 
+        Threatinator::Record.new(['def'], line_number: 2, pos_start: 5, pos_end: 10), 
+        Threatinator::Record.new(['hij'], line_number: 3, pos_start: 10, pos_end: 13), 
+      )
+    end
+
+    it "should parse \\r as a row separator"  do
+      str = "abc\rdef\rhij"
+      sio = StringIO.new(str)
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new(['abc'], line_number: 1, pos_start: 0, pos_end: 4), 
+        Threatinator::Record.new(['def'], line_number: 2, pos_start: 4, pos_end: 8), 
+        Threatinator::Record.new(['hij'], line_number: 3, pos_start: 8, pos_end: 11), 
+      )
+    end
+
+    it "should use commas as column separators" do
+      str = "1,2,3\n4,5,6\n7,8,9"
+      sio = StringIO.new(str)
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new(['1','2','3'], line_number: 1, pos_start: 0, pos_end: 6), 
+        Threatinator::Record.new(['4','5','6'], line_number: 2, pos_start: 6, pos_end: 12), 
+        Threatinator::Record.new(['7','8','9'], line_number: 3, pos_start: 12, pos_end: 17), 
+      )
+    end
+
+    it "should ignore separators contained in between a set of double-quotes" do
+      str = "1,\"2\",3\n\"4,5\",6\n\"7,8,9\""
+      sio = StringIO.new(str)
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new(['1','2','3'], line_number: 1, pos_start: 0, pos_end: 8), 
+        Threatinator::Record.new(['4,5','6'], line_number: 2, pos_start: 8, pos_end: 16), 
+        Threatinator::Record.new(['7,8,9'], line_number: 3, pos_start: 16, pos_end: 23), 
+      )
+    end
+  end
+
+  describe "configuration options" do
+    describe ":row_separator" do
+      it "should specify the row separator" do
+        str = "1,2,3X4,5,6X7,8,9"
+        sio = StringIO.new(str)
+        parser = described_class.new(row_separator: "X")
+
+        expect { |b|
+          parser.run(sio, &b)
+        }.to yield_successive_args(
+          Threatinator::Record.new(['1','2','3'], line_number: 1, pos_start: 0, pos_end: 6), 
+          Threatinator::Record.new(['4','5','6'], line_number: 2, pos_start: 6, pos_end: 12), 
+          Threatinator::Record.new(['7','8','9'], line_number: 3, pos_start: 12, pos_end: 17), 
+        )
+      end
+      it "should be accessible via #row_separator" do
+        parser = described_class.new(row_separator: "X")
+        expect(parser.row_separator).to eq("X")
+      end
+    end
+
+    describe ":col_separator" do
+      it "should specify the column separator" do
+        str = "1X2X3\n4X5X6\n7X8X9"
+        sio = StringIO.new(str)
+        parser = described_class.new(col_separator: "X")
+
+        expect { |b|
+          parser.run(sio, &b)
+        }.to yield_successive_args(
+          Threatinator::Record.new(['1','2','3'], line_number: 1, pos_start: 0, pos_end: 6), 
+          Threatinator::Record.new(['4','5','6'], line_number: 2, pos_start: 6, pos_end: 12), 
+          Threatinator::Record.new(['7','8','9'], line_number: 3, pos_start: 12, pos_end: 17), 
+        )
+      end
+      it "should be accessible via #col_separator" do
+        parser = described_class.new(col_separator: "X")
+        expect(parser.col_separator).to eq("X")
+      end
+    end
+    describe ":headers" do
+      it "should be accessible via #headers" do
+        parser = described_class.new(headers: :first_row)
+        expect(parser.headers).to eq(:first_row)
+      end
+      context "when set to false" do
+        it "should not use any headers" do 
+          str = "1,2,3\n4,5,6\n7,8,9"
+          sio = StringIO.new(str)
+          parser = described_class.new(headers: false)
+
+          expect { |b|
+            parser.run(sio, &b)
+          }.to yield_successive_args(
+            Threatinator::Record.new(['1','2','3'], line_number: 1, pos_start: 0, pos_end: 6), 
+            Threatinator::Record.new(['4','5','6'], line_number: 2, pos_start: 6, pos_end: 12), 
+            Threatinator::Record.new(['7','8','9'], line_number: 3, pos_start: 12, pos_end: 17), 
+          )
+        end
+      end
+      context "when set to true" do
+        it "should read the first row as the headers, returning Hashes with the headers as keys" do
+
+          str = "a,b,c\n1,2,3\n4,5,6\n7,8,9"
+          sio = StringIO.new(str)
+          parser = described_class.new(headers: true)
+
+          expect { |b|
+            parser.run(sio, &b)
+          }.to yield_successive_args(
+            Threatinator::Record.new({'a'=>'1','b'=>'2','c'=>'3'}, line_number: 2, pos_start: 6, pos_end: 12), 
+            Threatinator::Record.new({'a'=>'4','b'=>'5','c'=>'6'}, line_number: 3, pos_start: 12, pos_end: 18), 
+            Threatinator::Record.new({'a'=>'7','b'=>'8','c'=>'9'}, line_number: 4, pos_start: 18, pos_end: 23) 
+          )
+        end
+      end
+      context "when set to :first_row" do
+        it "should read the first row as the headers, returning Hashes with the headers as keys" do
+
+          str = "a,b,c\n1,2,3\n4,5,6\n7,8,9"
+          sio = StringIO.new(str)
+          parser = described_class.new(headers: :first_row)
+
+          expect { |b|
+            parser.run(sio, &b)
+          }.to yield_successive_args(
+            Threatinator::Record.new({'a'=>'1','b'=>'2','c'=>'3'}, line_number: 2, pos_start: 6, pos_end: 12), 
+            Threatinator::Record.new({'a'=>'4','b'=>'5','c'=>'6'}, line_number: 3, pos_start: 12, pos_end: 18), 
+            Threatinator::Record.new({'a'=>'7','b'=>'8','c'=>'9'}, line_number: 4, pos_start: 18, pos_end: 23) 
+          )
+        end
+      end
+      context "when set to an array of strings" do
+        it "should use the array as the headers, returning Hashes with the headers as keys" do
+
+          str = "1,2,3\n4,5,6\n7,8,9"
+          sio = StringIO.new(str)
+          parser = described_class.new(headers: ['a', 'b', 'c'])
+
+          expect { |b|
+            parser.run(sio, &b)
+          }.to yield_successive_args(
+            Threatinator::Record.new({'a'=>'1','b'=>'2','c'=>'3'}, line_number: 2, pos_start: 0, pos_end: 6), 
+            Threatinator::Record.new({'a'=>'4','b'=>'5','c'=>'6'}, line_number: 3, pos_start: 6, pos_end: 12), 
+            Threatinator::Record.new({'a'=>'7','b'=>'8','c'=>'9'}, line_number: 4, pos_start: 12, pos_end: 17) 
+          )
+        end
+      end
+    end
+  end
+end
+

data/spec/threatinator/parsers/getline/parser_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+require 'threatinator/parsers/getline/parser'
+require 'stringio'
+require 'threatinator/record'
+
+describe Threatinator::Parsers::Getline::Parser do
+  it_should_behave_like "a parser when compared to an identically configured parser" do
+    let(:parser1) { described_class.new(:separator => "X") }
+    let(:parser2) { described_class.new(:separator => "X") }
+  end
+  
+  context "two instances with differently configured separators" do
+    it_should_behave_like "a parser when compared to a differently configured parser" do
+      let(:parser1) { described_class.new(:separator => "X") }
+      let(:parser2) { described_class.new(:separator => "Y") }
+    end
+  end
+
+  it "should default to newline as a separator" do
+    str = "line 1\nline 2\nline3!"
+    sio = StringIO.new(str)
+    parser = described_class.new()
+
+    expect { |b|
+      parser.run(sio, &b)
+    }.to yield_successive_args(
+      Threatinator::Record.new("line 1\n", line_number: 1, pos_start: 0, pos_end: 7), 
+      Threatinator::Record.new("line 2\n", line_number: 2, pos_start: 7, pos_end: 14), 
+      Threatinator::Record.new("line3!", line_number: 3, pos_start: 14, pos_end: 20)
+    )
+  end
+  describe ":separator" do
+    it "should be accessible via the #separator method" do
+      parser = described_class.new(:separator => "Z")
+      expect(parser.separator).to eq("Z")
+    end
+    it "should specify a separator" do
+      str = "line 1\0line 2\0line3!"
+      sio = StringIO.new(str)
+      parser = described_class.new(:separator => "\0")
+
+      expect { |b|
+        parser.run(sio, &b)
+      }.to yield_successive_args(
+        Threatinator::Record.new("line 1\0", line_number: 1, pos_start: 0, pos_end: 7), 
+        Threatinator::Record.new("line 2\0", line_number: 2, pos_start: 7, pos_end: 14), 
+        Threatinator::Record.new("line3!", line_number: 3, pos_start: 14, pos_end: 20)
+      )
+    end
+  end
+
+  it "shouldn't split anything up if the separator isn't found" do
+    str = "line 1\0line 2\0line3!"
+    sio = StringIO.new(str)
+    parser = described_class.new(:separator => "\n")
+
+    expect { |b|
+      parser.run(sio, &b)
+    }.to yield_successive_args(
+      Threatinator::Record.new("line 1\0line 2\0line3!", line_number: 1, pos_start: 0, pos_end: 20) 
+)
+  end
+
+  it "should raise an ArgumentError if the separator isn't a single character" do
+    sio = StringIO.new()
+    expect { described_class.new(:separator => "asdf") }.to raise_error(ArgumentError)
+  end
+
+  it "should work on really, really long lines" do
+    line1 = ("A" * 100_000) + "\n"
+    line2 = ("B" * 100_000) + "\n"
+    line3 = "C" * 100_000
+    record1 = Threatinator::Record.new(line1, :line_number => 1, :pos_start => 0, :pos_end => 100_001)
+    record2 = Threatinator::Record.new(line2, :line_number => 2, :pos_start => 100_001, :pos_end => 200_002)
+    record3 = Threatinator::Record.new(line3, :line_number => 3, :pos_start => 200_002, :pos_end => 300_002)
+    sio = StringIO.new(line1 + line2 + line3)
+    parser = described_class.new(:separator => "\n")
+
+    expect { |b|
+      parser.run(sio, &b)
+    }.to yield_successive_args(record1, record2, record3)
+  end
+end

data/spec/threatinator/parsers/json/parser_spec.rb

@@ -0,0 +1,106 @@
+require 'spec_helper'
+require 'threatinator/parsers/json/parser'
+require 'stringio'
+require 'multi_json'
+
+describe Threatinator::Parsers::JSON::Parser do
+  it_should_behave_like "a parser when compared to an identically configured parser" do
+    let(:parser1) { described_class.new() }
+    let(:parser2) { described_class.new() }
+  end
+
+  let(:element1) { 
+    { 
+      'id' => 1,
+      'marbles' => [
+        { 'color' => 'red', 'count' => 20 },
+        { 'color' => 'blue', 'count' => 15 }
+      ],
+      'owner' => { 'name' => 'Billy', 'age' => '6' }
+    }
+  }
+  let(:element2) { 
+    { 
+      'id' => 2,
+      'marbles' => [
+        { 'color' => 'red', 'count' => 3 },
+        { 'color' => 'green', 'count' => 37 }
+      ],
+      'owner' => { 'name' => 'Sarah', 'age' => '7' }
+    }
+  }
+  let(:element3) {
+    { 
+      'id' => 3,
+      'marbles' => [
+        { 'color' => 'purple', 'count' => 73 }
+      ],
+      'owner' => { 'name' => 'Phil', 'age' => '5' }
+    }
+  }
+
+  context "when parsing a JSON string whose root element is an Array" do
+    let(:json_string) { MultiJson.dump([element1, element2, element3]) }
+    let(:io) { StringIO.new(json_string) }
+
+    it "should yield each of the array's elements as the :data of a Parsers::JSON::Record" do
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(io, &b)
+      }.to yield_successive_args(
+        Threatinator::Parsers::JSON::Record.new(element1), 
+        Threatinator::Parsers::JSON::Record.new(element2), 
+        Threatinator::Parsers::JSON::Record.new(element3)
+      )
+    end
+  end
+
+  context "when parsing a JSON string whose root element is a Hash" do
+    let(:json_string) { MultiJson.dump({'foo' => element1, 'bar' => element2, 'bla' => element3}) }
+    let(:io) { StringIO.new(json_string) }
+
+    it "should yield each of the hash's key/value pairs as the :key and :data of a Parsers::JSON::Record" do
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(io, &b)
+      }.to yield_successive_args(
+        Threatinator::Parsers::JSON::Record.new(element1, key: 'foo'), 
+        Threatinator::Parsers::JSON::Record.new(element2, key: 'bar'), 
+        Threatinator::Parsers::JSON::Record.new(element3, key: 'bla')
+      )
+    end
+  end
+
+  context "when parsing something that is not JSON data" do
+    let(:json_string) { "hi there! this is my data feed" }
+    let(:io) { StringIO.new(json_string) }
+
+    it "should raise a exception Threatinator::Exceptions::ParseError" do
+      parser = described_class.new()
+
+      expect { |b|
+        parser.run(io, &b)
+      }.to raise_error(Threatinator::Exceptions::ParseError)
+    end
+  end
+
+  context "when parsing a JSON that is truncated" do
+    let(:json_string) { MultiJson.dump({'foo' => element1, 'bar' => element2, 'bla' => element3})[0..-3] }
+    let(:io) { StringIO.new(json_string) }
+
+    it "should yield as many records as possible before raising a ParseError" do
+      parser = described_class.new()
+
+      expect { |b|
+        expect {
+          parser.run(io, &b)
+        }.to raise_error(Threatinator::Exceptions::ParseError)
+      }.to yield_successive_args(
+        Threatinator::Parsers::JSON::Record.new(element1, key: 'foo'), 
+        Threatinator::Parsers::JSON::Record.new(element2, key: 'bar')
+      )
+    end
+  end
+end