td 0.10.65 → 0.10.66

data/spec/file_reader/io_filter_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+require 'file_reader/shared_context'
+
+require 'stringio'
+require 'td/file_reader'
+
+include TreasureData
+
+describe 'FileReader io filters' do
+  include_context 'error_proc'
+
+  describe FileReader::DecompressIOFilter do
+    let :lines do
+      [
+        '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"size":95,"time":1353541928}',
+        '{"host":"224.225.147.72","user":"-","method":"GET","path":"/category/electronics","code":200,"size":43,"time":1353541927}',
+        '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"size":79,"time":1353541925}',
+      ]
+    end
+
+    let :io do
+      StringIO.new(lines.join("\n"))
+    end
+
+    let :gzipped_io do
+      require 'zlib'
+
+      io = StringIO.new('', 'w+')
+      gz = Zlib::GzipWriter.new(io)
+      gz.write(lines.join("\n"))
+      gz.close
+      StringIO.new(io.string)
+    end
+
+    it "can't filter with unknown compression" do
+      expect { 
+        FileReader::DecompressIOFilter.filter(io, error, :compress => 'oreore')
+      }.to raise_error(Exception, /unknown compression/)
+    end
+
+    describe 'gzip' do
+      it "can't be wrapped with un-gzipped io" do
+        expect {
+          FileReader::DecompressIOFilter.filter(io, error, :compress => 'gzip')
+        }.to raise_error(Zlib::GzipFile::Error)
+      end
+
+      it 'returns Zlib::GzipReader with :gzip' do
+        wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, :compress => 'gzip')
+        wrapped.should be_an_instance_of(Zlib::GzipReader)
+      end
+
+      it 'returns Zlib::GzipReader with auto detection' do
+        wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, {})
+        wrapped.should be_an_instance_of(Zlib::GzipReader)
+      end
+
+      context 'after initialization' do
+        [{:compress => 'gzip'}, {}].each { |opts|
+          let :reader do
+            wrapped = FileReader::DecompressIOFilter.filter(gzipped_io, error, opts)
+            FileReader::LineReader.new(wrapped, error, {})
+          end
+
+          it 'forward_row returns one line' do
+            reader.forward_row.should == lines[0]
+          end
+
+          it 'feeds all lines' do
+            begin
+              i = 0
+              while line = reader.forward_row
+                line.should == lines[i]
+                i += 1
+              end
+            rescue
+              gzipped_io.eof?.should be_true
+            end
+          end
+        }
+      end
+    end
+
+    describe 'plain' do
+      it 'returns passed io with :plain' do
+        wrapped = FileReader::DecompressIOFilter.filter(io, error, :compress => 'plain')
+        wrapped.should be_an_instance_of(StringIO)
+      end
+
+      it 'returns passed io with auto detection' do
+        wrapped = FileReader::DecompressIOFilter.filter(io, error, {})
+        wrapped.should be_an_instance_of(StringIO)
+      end
+    end
+  end
+end

data/spec/file_reader/line_reader_spec.rb

@@ -0,0 +1,227 @@
+require 'spec_helper'
+require 'file_reader/shared_context'
+
+require 'stringio'
+require 'td/file_reader'
+
+include TreasureData
+
+describe FileReader::LineReader do
+  include_context 'error_proc'
+
+  let :lines do
+    [
+      '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"referer":"http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=Sports+Electronics&oq=Sports+Electronics&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=3994&bih=421","size":95,"agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7","time":1353541928}',
+      '{"host":"224.225.147.72","user":"-","method":"GET","path":"/category/electronics","code":200,"referer":"-","size":43,"agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)","time":1353541927}',
+      '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"referer":"-","size":79,"agent":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","time":1353541925}',
+    ]
+  end
+
+  let :io do
+    StringIO.new(lines.join("\n"))
+  end
+
+  it 'initialize' do
+    if io.respond_to?(:external_encoding)
+      ee = io.external_encoding
+    end
+    FileReader::LineReader.new(io, error, {})
+    if io.respond_to?(:external_encoding)
+      io.external_encoding.should == ee
+    end
+  end
+
+  it 'initialize with specifid encoding' do
+    if io.respond_to?(:external_encoding)
+      ee = io.external_encoding
+    end
+    FileReader::LineReader.new(io, error, {:encoding => 'utf-8'})
+    if io.respond_to?(:external_encoding)
+      io.external_encoding.should_not == ee
+      io.external_encoding.should == Encoding.find('utf-8')
+    end
+  end
+
+  context 'after initialization' do
+    let :reader do
+      FileReader::LineReader.new(io, error, {})
+    end
+
+    it 'forward_row returns one line' do
+      reader.forward_row.should == lines[0]
+    end
+
+    # TODO: integrate with following shared_examples_for
+    it 'feeds all lines' do
+      begin
+        i = 0
+        while line = reader.forward_row
+          line.should == lines[i]
+          i += 1
+        end
+      rescue RSpec::Expectations::ExpectationNotMetError => e
+        fail
+      rescue
+        io.eof?.should be_true
+      end
+    end
+
+    shared_examples_for 'parser iterates all' do |step|
+      step = step || 1
+
+      it 'feeds all' do
+        begin
+          i = 0
+          while line = parser.forward
+            line.should == get_expected.call(i)
+            i += step
+          end
+        rescue RSpec::Expectations::ExpectationNotMetError => e
+          fail
+        rescue
+          io.eof?.should be_true
+        end
+      end
+    end
+
+    describe FileReader::JSONParser do
+      it 'initialize with LineReader' do
+        parser = FileReader::JSONParser.new(reader, error, {})
+        parser.should_not be_nil
+      end
+
+      context 'after initialization' do
+        let :parser do
+          FileReader::JSONParser.new(reader, error, {})
+        end
+
+        it 'forward returns one line' do
+          parser.forward.should == JSON.parse(lines[0])
+        end
+
+        let :get_expected do
+          lambda { |i| JSON.parse(lines[i]) }
+        end
+
+        it_should_behave_like 'parser iterates all'
+
+        context 'with broken line' do
+          let :lines do
+            [
+              '{"host":"128.216.140.97","user":"-","method":"GET","path":"/item/sports/2511","code":200,"referer":"http://www.google.com/search?ie=UTF-8&q=google&sclient=psy-ab&q=Sports+Electronics&oq=Sports+Electronics&aq=f&aqi=g-vL1&aql=&pbx=1&bav=on.2,or.r_gc.r_pw.r_qf.,cf.osb&biw=3994&bih=421","size":95,"agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7","time":1353541928}',
+              '{This is invalid as a JSON}',
+              '{"host":"172.75.186.56","user":"-","method":"GET","path":"/category/jewelry","code":200,"referer":"-","size":79,"agent":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","time":1353541925}',
+            ]
+          end
+
+          let :error_pattern do
+            /^invalid json format/
+          end
+
+          it_should_behave_like 'parser iterates all', 2
+        end
+      end
+    end
+
+    [',', "\t"].each { |pattern|
+      describe FileReader::DelimiterParser do
+        let :lines do
+          [
+            ['hoge', '12345', Time.now.to_s].join(pattern),
+            ['foo',  '34567', Time.now.to_s].join(pattern),
+            ['piyo', '56789', Time.now.to_s].join(pattern),
+          ]
+        end
+
+        it "initialize with LineReader and #{pattern} delimiter" do
+          parser = FileReader::DelimiterParser.new(reader, error, {:delimiter_expr => Regexp.new(pattern)})
+          parser.should_not be_nil
+        end
+
+        context 'after initialization' do
+          let :parser do
+            FileReader::DelimiterParser.new(reader, error, {:delimiter_expr => Regexp.new(pattern)})
+          end
+
+          it 'forward returns one line' do
+            parser.forward.should == lines[0].split(pattern)
+          end
+
+          let :get_expected do
+            lambda { |i| lines[i].split(pattern) }
+          end
+
+          it_should_behave_like 'parser iterates all'
+        end
+      end
+    }
+
+    {
+      FileReader::ApacheParser => [
+        [
+          '58.83.188.60 - - [23/Oct/2011:08:15:46 -0700] "HEAD / HTTP/1.0" 200 277 "-" "-"',
+          '127.0.0.1 - - [23/Oct/2011:08:20:01 -0700] "GET / HTTP/1.0" 200 492 "-" "Wget/1.12 (linux-gnu)"',
+          '68.64.37.100 - - [24/Oct/2011:01:48:54 -0700] "GET /phpMyAdmin/scripts/setup.php HTTP/1.1" 404 480 "-" "ZmEu"'
+        ],
+        [
+          ["58.83.188.60", "-", "23/Oct/2011:08:15:46 -0700", "HEAD", "/", "200", "277", "-", "-"],
+          ["127.0.0.1", "-", "23/Oct/2011:08:20:01 -0700", "GET", "/", "200", "492", "-", "Wget/1.12 (linux-gnu)"],
+          ["68.64.37.100", "-", "24/Oct/2011:01:48:54 -0700", "GET", "/phpMyAdmin/scripts/setup.php", "404", "480", "-", "ZmEu"],
+        ]
+      ],
+      FileReader::SyslogParser => [
+        [
+          'Dec 20 12:41:44 localhost kernel: [4843680.692840] e1000e: eth2 NIC Link is Down',
+          'Dec 20 12:41:44 localhost kernel: [4843680.734466] br0: port 1(eth2) entering disabled state',
+          'Dec 22 10:42:41 localhost kernel: [5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]',
+        ],
+        [
+          ["Dec 20 12:41:44", "localhost", "kernel", nil, "[4843680.692840] e1000e: eth2 NIC Link is Down"],
+          ["Dec 20 12:41:44", "localhost", "kernel", nil, "[4843680.734466] br0: port 1(eth2) entering disabled state"],
+          ["Dec 22 10:42:41", "localhost", "kernel", nil, "[5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]"],
+        ]
+      ]
+    }.each_pair { |parser_class, (input, output)|
+      describe parser_class do
+        let :lines do
+          input
+        end
+
+        it "initialize with LineReader" do
+          parser = parser_class.new(reader, error, {})
+          parser.should_not be_nil
+        end
+
+        context 'after initialization' do
+          let :parser do
+            parser_class.new(reader, error, {})
+          end
+
+          it 'forward returns one line' do
+            parser.forward.should == output[0]
+          end
+
+          let :get_expected do
+            lambda { |i| output[i] }
+          end
+
+          it_should_behave_like 'parser iterates all'
+
+          context 'with broken line' do
+            let :lines do
+              broken = input.dup
+              broken[1] = "Raw text sometimes is broken!"
+              broken
+            end
+
+            let :error_pattern do
+              /^invalid #{parser.instance_variable_get(:@format)} format/
+            end
+
+            it_should_behave_like 'parser iterates all', 2
+          end
+        end
+      end
+    }
+  end
+end

data/spec/file_reader/parsing_reader_spec.rb

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'file_reader/shared_context'
+
+require 'stringio'
+require 'msgpack'
+require 'td/file_reader'
+
+include TreasureData
+
+describe 'FileReader parsing readers' do
+  include_context 'error_proc'
+
+  shared_examples_for 'forward basics' do
+    it 'forward returns one data' do
+      reader.forward.should == dataset[0]
+    end
+
+    it 'feeds all dataset' do
+      begin
+        i = 0
+        while line = reader.forward
+          line.should == dataset[i]
+          i += 1
+        end
+      rescue RSpec::Expectations::ExpectationNotMetError => e
+          fail
+      rescue => e
+        io.eof?.should be_true
+      end
+    end
+  end
+
+  describe FileReader::MessagePackParsingReader do
+    let :dataset do
+      [
+        {'name' => 'k', 'num' => 12345, 'time' => Time.now.to_i},
+        {'name' => 's', 'num' => 34567, 'time' => Time.now.to_i},
+        {'name' => 'n', 'num' => 56789, 'time' => Time.now.to_i},
+      ]
+    end
+
+    let :io do
+      StringIO.new(dataset.map(&:to_msgpack).join(""))
+    end
+
+    it 'initialize' do
+      reader = FileReader::MessagePackParsingReader.new(io, error, {})
+      reader.should_not be_nil
+    end
+
+    context 'after initialization' do
+      let :reader do
+        FileReader::MessagePackParsingReader.new(io, error, {})
+      end
+
+      it_should_behave_like 'forward basics'
+    end
+  end
+
+  test_time = Time.now.to_i
+  {
+    'csv' => [
+      {:delimiter_expr => ',', :quote_char => '"', :encoding => 'utf-8'},
+      [
+        %!k,123,"fo\no",true,#{test_time}!,
+        %!s,456,"Ｔ,Ｄ",false,#{test_time}!,
+        %!n,789,"ba""z",false,#{test_time}!,
+      ],
+      [
+        %W(k 123 fo\no true #{test_time}),
+        %W(s 456 Ｔ,Ｄ false #{test_time}),
+        %W(n 789 ba\"z false #{test_time}),
+      ]
+    ],
+    'tsv' => [
+      {:delimiter_expr => "\t"},
+      [
+        %!k\t123\t"fo\no"\ttrue\t#{test_time}!,
+        %!s\t456\t"b,ar"\tfalse\t#{test_time}!,
+        %!n\t789\t"ba\tz"\tfalse\t#{test_time}!,
+      ],
+      [
+        %W(k 123 fo\no true #{test_time}),
+        %W(s 456 b,ar false #{test_time}),
+        %W(n 789 ba\tz false #{test_time}),
+      ]
+    ]
+  }.each_pair { |format, (opts, input, output)|
+    describe FileReader::SeparatedValueParsingReader do
+      let :dataset do
+        output
+      end
+
+      let :lines do
+        input
+      end
+
+      let :io do
+        StringIO.new(lines.join($/))
+      end
+
+      it "initialize #{format}" do
+        reader = FileReader::SeparatedValueParsingReader.new(io, error, opts)
+        reader.should_not be_nil
+      end
+
+      context "after #{format} initialization" do
+        let :reader do
+          reader = FileReader::SeparatedValueParsingReader.new(io, error, opts)
+        end
+
+        it_should_behave_like 'forward basics'
+
+        context "broken encodings" do
+        end
+      end
+    end
+  }
+end

data/spec/file_reader/shared_context.rb

@@ -0,0 +1,9 @@
+require 'rspec'
+
+shared_context 'error_proc' do
+  let :error do
+    Proc.new { |reason, data|
+      reason.should match(error_pattern)
+    }
+  end
+end