nfcollector 3.2.0

data/spec/attributes_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Nfcollector::Attributes do
+  describe '::parse' do
+    let(:attributes_string) { '>a,Rh,Rp' }
+    subject { described_class.parse(attributes_string) }
+
+    specify do
+      expect(subject).to be_a(Nfcollector::InputDefinition)
+      expect(subject.column_index(:client_ip)).to eq(0)
+      expect(subject.column_index(:host)).to eq(1)
+      expect(subject.column_index(:path)).to eq(2)
+    end
+  end
+end

data/spec/command_parser_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe Nfcollector::CommandParser do
+
+  describe "Instance" do
+    specify "raise if account_id is not an int" do
+      expect {
+        cp = described_class.new("abc", "/file/path", ">a")
+        cp.validate!
+      }.to raise_error(Nfcollector::InvalidCommand, "Account ID is not an Integer")
+    end
+
+    specify "raise if file not found" do
+      allow(File).to receive(:file?).and_return(false)
+      expect {
+        cp = described_class.new(10, "/file/path", ">a")
+        cp.validate!
+      }.to raise_error(Nfcollector::InvalidCommand, "No such file for payload")
+    end
+
+    specify "raise if attributes are not valid" do
+      allow(File).to receive(:file?).and_return(true)
+      allow(
+        Nfcollector::AttributeValidator
+      ).to receive(:validate!).
+        with(">a").and_raise(Nfcollector::UnknownAttribute)
+
+      expect {
+        cp = described_class.new(10, "/file/path", 'X')
+        cp.validate!
+      }.to raise_error(Nfcollector::UnknownAttribute)
+    end
+
+    specify "raise if attributes are missing" do
+      allow(File).to receive(:file?).and_return(true)
+      expect {
+        cp = described_class.new(10, "/file/path", nil)
+        cp.validate!
+      }.to raise_error(Nfcollector::InvalidCommand)
+    end
+
+    specify "raise if attributes are empty" do
+      allow(File).to receive(:file?).and_return(true)
+      expect {
+        cp = described_class.new(10, "/file/path", "")
+        cp.validate!
+      }.to raise_error(Nfcollector::InvalidCommand)
+    end
+
+    specify "not raise if data is correct (string account id)" do
+      allow(File).to receive(:file?).and_return(true)
+      allow(Nfcollector::AttributeValidator).to receive(:validate!).and_return(true)
+      cp = described_class.new("100", "/file/path", Nfcollector::Attributes::REQUIRED.join(','))
+      cp.validate!
+    end
+
+    specify "not raise if data is correct (numeric account id)" do
+      allow(File).to receive(:file?).and_return(true)
+      allow(Nfcollector::AttributeValidator).to receive(:validate!).and_return(true)
+      expect {
+        cp = described_class.new(100, "/file/path", Nfcollector::Attributes::REQUIRED.join(','))
+        cp.validate!
+      }.to_not raise_error
+    end
+  end
+
+  context "Class" do
+    specify "raise if not all tokens are provided" do
+      expect {
+        described_class.parse("10:")
+      }.to raise_error(Nfcollector::InvalidCommand)
+    end
+
+    specify "validate! provided tokens" do
+      cp = stub(:parser)
+      expect(cp).to receive(:validate)
+      expect(described_class).to receive(:new).with(10, '/foo/bar', '>a').and_return(cp)
+      described_class.parse("10:/foo/bar:>a")
+    end
+  end
+end

data/spec/copy_file_writer_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+describe Nfcollector::CopyFileWriter do
+  describe "Adding rows" do
+    # TODO: Test too few columns as well? Will this just result in NULLs?
+    specify "ignore additional data if a row is added that has more columns set in the constructor" do
+      expect {
+        copy = described_class.new(%w(created_at username bytes), 10, 0)
+        copy.add_row([Time.now, "daniel", 1000, "foo"])
+      }.to_not raise_error
+    end
+  end
+
+  describe "Table Name" do
+    # be correct for the given date" do
+    pending "Something needs to happen here but I don't know what"
+  end
+
+  describe "File Name" do
+    specify "that it is correct" do
+      Nfcollector::Configuration.output_dir = "/tmp/"
+      date = stub(:date)
+      copy = described_class.new(%w(created_at username bytes), 10, 0)
+      expect(copy).to receive(:table_name).and_return("mocked_table_name")
+      allow(copy).to receive(:randstr).and_return('randstr')
+      expect(copy.file_name(Date.today)).to eq("/tmp/mocked_table_name_randstr.copy")
+    end
+  end
+
+  describe "Writing" do
+    before do
+      allow_any_instance_of(described_class).to receive(:table_name).and_return('mocked_table_name')
+      Timecop.freeze(Time.new(2011, 7, 11, 0, 0, 0, "+10:00"))
+    end
+
+    after do
+      Timecop.return
+    end
+
+    specify "write the file" do
+      copy = described_class.new(%w(created_at username bytes), 10, 0)
+      copy.add_row([Time.now.utc, "daniel", 1000])
+      io = StringIO.new
+      expect(File).to receive(:open).and_yield(io)
+      allow(copy).to receive(:randstr).and_return('randstr')
+      expect(FileUtils).to receive(:mv).with('/tmp/mocked_table_name_randstr.copy.lock', '/tmp/mocked_table_name_randstr.copy')
+      copy.write
+      expect(io.string).to eq <<-COPY
+-- Created at: 2011-07-10 14:00:00 UTC
+COPY mocked_table_name (created_at,username,bytes) FROM stdin WITH csv;
+"2011-07-10 14:00:00 UTC","daniel","1000"
+      COPY
+    end
+
+    specify "write the file and handle NULLs" do
+      copy = described_class.new(%w(created_at username bytes user_group), 10, 0)
+      copy.add_row([Time.now.utc, 'daniel', '1000', nil])
+      io = StringIO.new
+      expect(File).to receive(:open).and_yield(io)
+      expect(copy).to receive(:randstr).and_return('randstr')
+      expect(FileUtils).to receive(:mv).with('/tmp/mocked_table_name_randstr.copy.lock', '/tmp/mocked_table_name_randstr.copy')
+      copy.write
+      expect(io.string).to eq <<-COPY
+-- Created at: 2011-07-10 14:00:00 UTC
+COPY mocked_table_name (created_at,username,bytes,user_group) FROM stdin WITH csv;
+"2011-07-10 14:00:00 UTC","daniel","1000",""
+      COPY
+    end
+
+    specify "delete the resultant file if there is an exception" do
+      copy = described_class.new(%w(created_at username bytes user_group), 10, 0)
+      copy.add_row([Time.now, 'daniel', '1000', nil])
+      expect(CSV).to receive(:new).and_raise(RuntimeError)
+      allow(File).to receive(:file?).and_return(true)
+      expect(File).to receive(:delete)
+      expect { copy.write }.to raise_error(RuntimeError)
+    end
+
+    specify "raise if there are no rows" do
+      copy = described_class.new(%w(created_at username bytes user_group), 10, 0)
+      expect { copy.write }.to raise_error(Nfcollector::FileEmpty)
+    end
+
+    # TODO: This is not tested effectively
+    specify "return false if a new copy file should be generated (date barrier)" do
+      copy = described_class.new(%w(created_at username bytes), 10, 0)
+      copy.add_row([Time.gm(2011, 7, 11, 0, 0, 0), 'daniel', '1000'])
+      copy.add_row([Time.gm(2011, 7, 12, 0, 0, 0), 'daniel', '1000'])
+      allow(copy).to receive(:randstr).and_return('randstr')
+      expect(FileUtils).to receive(:mv).with('/tmp/mocked_table_name_randstr.copy.lock', '/tmp/mocked_table_name_randstr.copy').twice
+      expect(File).to receive(:open).twice
+      copy.write
+    end
+  end
+end

data/spec/input_definition_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Nfcollector::InputDefinition do
+  describe '#set' do
+    let(:definition) { described_class.new }
+    before { definition.set(1, :username) }
+
+    specify do
+      expect(definition.column_index(:username)).to eq(1)
+    end
+
+    specify 'that an error is raised if there is no index for the column' do
+      expect {
+        definition.column_index(:foo)
+      }.to raise_error(Nfcollector::InputDefinition::MissingDefinition)
+    end
+  end
+end

data/spec/nfcollector/category_partitioner_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe Nfcollector::CategoryPartitioner do
+  let(:partitioner) { described_class.new(2) }
+
+  describe '#partition_id' do
+    subject { partitioner.partition_id(row) }
+
+    context 'both columns available' do
+      let(:row) { %w(1000 100 10) }
+      specify   { expect(subject).to eq('10') }
+    end
+  end
+
+  describe '#add_row' do
+    let(:row) { %w(1000 100 10) }
+    subject   { partitioner }
+
+    before { subject.add_row(row) }
+
+    specify do
+      expect(subject.data).to eq({
+        '10' => [row]
+      })
+    end
+
+    describe 'adding a second row with the same partition id' do
+      let(:row2) { %w(1001 100 10) }
+      before     { subject.add_row(row2) }
+
+      specify do
+        expect(subject.data).to eq({
+          '10' => [row, row2]
+        })
+      end
+    end
+
+    describe 'adding a second row with the a different partition id' do
+      let(:row3) { %w(1001 100 11) }
+      before     { subject.add_row(row3) }
+
+      specify do
+        expect(subject.data).to eq({
+          '10' => [row],
+          '11' => [row3]
+        })
+      end
+    end
+
+  end
+end

data/spec/nfcollector/date_partitioner_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Nfcollector::DatePartitioner do
+  let(:partitioner) { described_class.new(2) }
+
+  describe '#partition_id' do
+    subject { partitioner.partition_id(row) }
+
+    context 'the timestamp in UTC is on the same day' do
+      let(:row) { [ 1000, 100, '2014-06-16 10:29:52 +1000'.to_time ] }
+      specify   { expect(subject.to_s).to eq('2014-06-16') }
+    end
+
+    context 'the timestamp in UTC is on the previous day' do
+      let(:row) { [ 1000, 100, '2014-06-16 9:29:52 +1000'.to_time ] }
+      specify   { expect(subject.to_s).to eq('2014-06-15') }
+    end
+  end
+end

data/spec/nfcollector/input_definition_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Nfcollector::InputDefinition do
+  let(:input) { described_class.new }
+
+  describe '#set' do
+    before  { input.set(0, :username) }
+    specify { expect(input.column_index(:username)).to eq(0) }
+  end
+
+  describe '#has_index_for?' do
+    context 'no column present' do
+      specify { expect(input.has_index_for?(:username)).to be(false) }
+    end
+
+    context 'column is present' do
+      before  { input.set(0, :username) }
+      specify { expect(input.has_index_for?(:username)).to be(true) }
+    end
+
+    context 'mulitple columns present' do
+      before  do
+        input.set(0, :username)
+        input.set(1, :client_ip)
+      end
+
+      specify do
+        expect(input.has_index_for?([:username, :client_ip])).to be(true)
+      end
+    end
+  end
+end

data/spec/nfcollector/mapping/column_transpiler_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Nfcollector::Mapping::ColumnTranspiler do
+  let(:output)    { Nfcollector::Mapping::Output.new(:username) }
+  let(:input_row) { [ '2013-10-10 00:30', 'daniel' ] }
+  subject(:column_transpiler) { described_class.new(output) }
+
+  describe '#go' do
+    context 'not yet built' do
+      specify do
+        expect {
+          subject.go(input_row)
+        }.to raise_error(Nfcollector::Mapping::ColumnTranspiler::NotYetBuilt)
+      end
+    end
+
+    context 'has been built' do
+      let(:input_definition) { Nfcollector::Attributes.parse('t,Un') }
+      before { subject.build(input_definition) }
+      
+      specify do
+        expect(subject.go(input_row)).to eq('daniel')
+      end
+    end
+  end
+end

data/spec/nfcollector/mapping/output_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Nfcollector::Mapping::Output do
+  describe 'class DSL' do
+    describe 'base state' do
+      before do
+        class MyMapperA < Nfcollector::Mapping::Output; end
+      end
+
+      specify { expect(MyMapperA.outputs).to eq([]) }
+    end
+
+    describe 'with outputs defined' do
+      before do
+        class MyMapperB < Nfcollector::Mapping::Output
+          output :created_at
+          output :username
+        end
+      end
+
+      specify do
+        expect(MyMapperB.outputs.size).to eq(2)
+        expect(MyMapperB.outputs.first).to be_a(Nfcollector::Mapping::Output)
+      end
+    end
+
+    describe '#compile' do
+
+    end
+  end
+
+  describe '#initialize' do
+    context 'defaults' do
+      subject { described_class.new(:username) }
+
+      specify do
+        expect(subject.name).to eq(:username)  
+        expect(subject.inputs).to eq([:username])  
+        expect(subject.process_with).to eq(:username)
+      end
+    end
+
+    context 'one using specific input' do
+      subject { described_class.new(:username, inputs: :user) }
+      specify { expect(subject.inputs).to eq([:user]) }
+    end
+
+    context 'several specific inputs' do
+      subject { described_class.new(:username, inputs: [:user, :group]) }
+      specify { expect(subject.inputs).to eq([:user, :group]) }
+    end
+
+    context 'with a custom processing method' do
+      subject { described_class.new(:username, process_with: :do_stuff) }
+      specify { expect(subject.process_with).to eq(:do_stuff) }
+    end
+  end
+
+  describe '#optional' do
+    context 'unset' do
+      specify { expect(described_class.new(:username)).to_not be_optional }
+    end
+    
+    context 'set' do
+      specify { expect(described_class.new(:username, optional: true)).to be_optional }
+    end
+  end
+
+  describe '#compile' do
+    pending
+  end
+
+  describe '#method_missing' do
+    pending
+  end
+end

data/spec/nfcollector/mapping/transpiler_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Nfcollector::Mapping::Transpiler do
+  # TODO: Rename to add_output
+  describe '#add_column' do
+    let(:transpiler)       { described_class.new(input_definition) }
+    let(:input_definition) { Nfcollector::Attributes.parse('>a,Rh,Rp') }
+    subject                { transpiler.add_column(output) }
+
+    context 'input definition has index for column' do
+      let(:output) { Nfcollector::Mapping::Output.new(:client_ip) }
+
+      specify 'that we add an output' do
+        expect { subject }.to change { transpiler.outputs.size }.by(1)
+      end
+
+      specify 'that the added output is valid column transpiler' do
+        subject
+        expect(transpiler.outputs[0]).to be_a(Nfcollector::Mapping::ColumnTranspiler)
+      end
+    end
+
+    context 'input definition does not have index for column' do
+      context 'and output is required' do
+        let(:output) { Nfcollector::Mapping::Output.new(:username) }
+
+        specify 'that we DO NOT add an output' do
+          expect {
+            subject
+          }.to raise_error(Nfcollector::InputDefinition::MissingDefinition)
+        end
+      end
+
+      context 'and output is not required' do
+        let(:output) { Nfcollector::Mapping::Output.new(:username, optional: true) }
+
+        specify 'that we DO NOT add an output' do
+          expect { subject }.to_not change { transpiler.outputs.size }
+        end
+      end
+    end
+  end
+
+  describe '#transpile' do
+    pending
+  end
+end