td 0.10.65 → 0.10.66

data/spec/file_reader_spec.rb

@@ -0,0 +1,401 @@
+require 'spec_helper'
+require 'file_reader/shared_context'
+
+require 'stringio'
+require 'td/file_reader'
+
+include TreasureData
+
+describe FileReader do
+  include_context 'error_proc'
+
+  describe 'initialize' do
+    subject { FileReader.new }
+
+    its(:parser_class) { should be_nil }
+    its(:opts) { should be_empty }
+    [:delimiter_expr, :null_expr, :true_expr, :false_expr].each { |key|
+      its(:default_opts) { should have_key(key); }
+    }
+  end
+
+  let :reader do
+    FileReader.new
+  end
+
+  describe 'set_format_template' do
+    it 'can set csv' do
+      reader.set_format_template('csv')
+      reader.instance_variable_get(:@format).should == 'text'
+      reader.opts.should include(:delimiter_expr => /,/)
+    end
+
+    it 'can set tsv' do
+      reader.set_format_template('tsv')
+      reader.instance_variable_get(:@format).should == 'text'
+      reader.opts.should include(:delimiter_expr => /\t/)
+    end
+
+    it 'can set apache' do
+      reader.set_format_template('apache')
+      reader.instance_variable_get(:@format).should == 'apache'
+      reader.opts.should include(:time_column => 'time')
+    end
+
+    it 'can set syslog' do
+      reader.set_format_template('syslog')
+      reader.instance_variable_get(:@format).should == 'syslog'
+      reader.opts.should include(:time_column => 'time')
+    end
+
+    it 'can set msgpack' do
+      reader.set_format_template('msgpack')
+      reader.instance_variable_get(:@format).should == 'msgpack'
+    end
+
+    it 'can set json' do
+      reader.set_format_template('json')
+      reader.instance_variable_get(:@format).should == 'json'
+    end
+
+    it 'raises when set unknown format' do
+      expect {
+        reader.set_format_template('oreore')
+      }.to raise_error(Exception, /Unknown format: oreore/)
+    end
+  end
+
+  describe 'init_optparse' do
+    def parse_opt(argv, &block)
+      op = OptionParser.new
+      reader.init_optparse(op)
+      op.parse!(argv)
+      block.call
+    end
+
+    context '-f option' do
+      ['-f', '--format'].each { |opt|
+        ['csv', 'tsv', 'apache', 'syslog', 'msgpack', 'json'].each { |format|
+          it "#{opt} option with #{format}" do
+            reader.should_receive(:set_format_template).with(format)
+            parse_opt([opt, format]) { }
+          end
+        }
+      }
+    end
+
+    context 'columns names option' do
+      ['-h', '--columns'].each { |opt|
+        it "#{opt} option" do
+          columns = 'A,B,C'
+          parse_opt([opt, columns]) {
+            reader.opts.should include(:column_names => columns.split(','))
+          }
+        end
+      }
+    end
+
+    context 'columns header option' do
+      ['-H', '--column-header'].each { |opt|
+        it "#{opt} option" do
+          parse_opt([opt]) {
+            reader.opts.should include(:column_header => true)
+          }
+        end
+      }
+    end
+
+    context 'delimiter between column option' do
+      ['-d', '--delimiter'].each { |opt|
+        it "#{opt} option" do
+          pattern = '!'
+          parse_opt([opt, pattern]) {
+            reader.opts.should include(:delimiter_expr => Regexp.new(pattern))
+          }
+        end
+      }
+    end
+
+    context 'null expression option' do
+      it "--null REGEX option" do
+        pattern = 'null'
+        parse_opt(['--null', pattern]) {
+          reader.opts.should include(:null_expr => Regexp.new(pattern))
+        }
+      end
+    end
+
+    context 'true expression option' do
+      it "--true REGEX option" do
+        pattern = 'true'
+        parse_opt(['--true', pattern]) {
+          reader.opts.should include(:true_expr => Regexp.new(pattern))
+        }
+      end
+    end
+
+    context 'false expression option' do
+      it "--false REGEX option" do
+        pattern = 'false'
+        parse_opt(['--false', pattern]) {
+          reader.opts.should include(:false_expr => Regexp.new(pattern))
+        }
+      end
+    end
+
+    context 'disable automatic type conversion option' do
+      ['-S', '--all-string'].each { |opt|
+        it "#{opt} option" do
+          parse_opt([opt]) {
+            reader.opts.should include(:all_string => true)
+          }
+        end
+      }
+    end
+
+    context 'name of the time column option' do
+      ['-t', '--time-column'].each { |opt|
+        it "#{opt} option" do
+          name = 'created_at'
+          parse_opt([opt, name]) {
+            reader.opts.should include(:time_column => name)
+          }
+        end
+      }
+    end
+
+    context 'strftime(3) format of the time column option' do
+      ['-T', '--time-format'].each { |opt|
+        it "#{opt} option" do
+          format = '%Y'
+          parse_opt([opt, format]) {
+            reader.opts.should include(:time_format => format)
+          }
+        end
+      }
+    end
+
+    context 'value of the time column option' do
+      {'int' => lambda { |t| t.to_i.to_s }, 'formatted' => lambda { |t| t.to_s }}.each_pair { |value_type, converter|
+        it "--time-value option with #{value_type}" do
+          time = Time.now
+          parse_opt(['--time-value', converter.call(time)]) {
+            reader.opts.should include(:time_value => time.to_i)
+          }
+        end
+      }
+    end
+
+    context 'text encoding option' do
+      ['-e', '--encoding'].each { |opt|
+        it "#{opt} option" do
+          enc = 'utf-8'
+          parse_opt([opt, enc]) {
+            reader.opts.should include(:encoding => enc)
+          }
+        end
+      }
+    end
+
+    context 'compression format option' do
+      ['-C', '--compress'].each { |opt|
+        it "#{opt} option" do
+          format = 'gzip'
+          parse_opt([opt, format]) {
+            reader.opts.should include(:compress => format)
+          }
+        end
+      }
+    end
+  end
+
+  describe 'compose_factory' do
+    it 'returns Proc object' do
+      factory = reader.compose_factory
+      factory.should be_an_instance_of(Proc)
+    end
+
+    # other specs in parse spec
+  end
+
+  describe 'parse' do
+    let :dataset_header do
+      ['name', 'num', 'created_at', 'flag']
+    end
+
+    let :dataset_values do
+      [
+        ['k', 12345, Time.now.to_s, true],
+        ['s', 34567, Time.now.to_s, false],
+        ['n', 56789, Time.now.to_s, true],
+      ]
+    end
+
+    let :dataset do
+      dataset_values.map { |data|
+        Hash[dataset_header.zip(data)]
+      }
+    end
+
+    let :time_column do
+      'created_at'
+    end
+
+    def parse_opt(argv, &block)
+      op = OptionParser.new
+      reader.init_optparse(op)
+      op.parse!(argv)
+      block.call
+    end
+
+    shared_examples_for 'parse --time-value / --time-column cases' do |format, args|
+      it "parse #{format} with --time-value" do
+        @time = Time.now.to_i
+        parse_opt(%W(-f #{format} --time-value #{@time}) + (args || [])) {
+          i = 0
+          reader.parse(io, error) { |record|
+            record.should == dataset[i].merge('time' => @time)
+            i += 1
+          }
+        }
+      end
+
+      it "parse #{format} with --time-column" do
+        parse_opt(%W(-f #{format} --time-column #{time_column}) + (args || [])) {
+          i = 0
+          reader.parse(io, error) { |record|
+            time = record[time_column]
+            time = Time.parse(time).to_i if time.is_a?(String)
+            record.should == dataset[i].merge('time' => time)
+            i += 1
+          }
+        }
+      end
+    end
+
+    shared_examples_for 'parse --columns / --column-header cases' do |format|
+      converter = "to_#{format}".to_sym
+
+      context 'array format' do
+        let :lines do
+          dataset_values.map { |data| data.__send__(converter) }
+        end
+
+        context 'with --column-columns' do
+          it_should_behave_like 'parse --time-value / --time-column cases', format, %W(-h name,num,created_at,flag)
+        end
+
+        context 'with --column-header' do
+          let :lines do
+            [dataset_header.__send__(converter)] + dataset_values.map { |data| data.__send__(converter) }
+          end
+
+          it_should_behave_like 'parse --time-value / --time-column cases', format, %W(-H)
+        end
+      end
+    end
+
+    let :io do
+      StringIO.new(lines.join("\n"))
+    end
+
+    context 'json' do
+      require 'json'
+
+      let :lines do
+        dataset.map(&:to_json)
+      end
+
+      it_should_behave_like 'parse --time-value / --time-column cases', 'json'
+      it_should_behave_like 'parse --columns / --column-header cases', 'json'
+    end
+
+    context 'msgpack' do
+      require 'msgpack'
+
+      let :lines do
+        dataset.map(&:to_msgpack)
+      end
+
+      let :io do
+        StringIO.new(lines.join(""))
+      end
+
+      it_should_behave_like 'parse --time-value / --time-column cases', 'msgpack'
+      it_should_behave_like 'parse --columns / --column-header cases', 'msgpack'
+    end
+
+    [['csv', ','], ['tsv', "\t"]].each { |text_type, pattern|
+      context 'text' do
+        let :lines do
+          dataset_values.map { |data| data.map(&:to_s).join(pattern) }
+        end
+
+        it "raises an exception without --column-header or --columns in #{pattern}" do
+          parse_opt(%W(-f #{text_type})) {
+            expect {
+              reader.parse(io, error)
+            }.to raise_error(Exception, /--column-header or --columns option is required/)
+          }
+        end
+
+        context 'with --column-columns' do
+          it_should_behave_like 'parse --time-value / --time-column cases', text_type, %W(-h name,num,created_at,flag)
+        end
+
+        context 'with --column-header' do
+          let :lines do
+            [dataset_header.join(pattern)] + dataset_values.map { |data| data.map(&:to_s).join(pattern) }
+          end
+
+          it_should_behave_like 'parse --time-value / --time-column cases', text_type, %W(-H)
+        end
+
+        # TODO: Add all_string
+      end
+    }
+
+    {
+      'apache' => [
+        [
+          '58.83.188.60 - - [23/Oct/2011:08:15:46 -0700] "HEAD / HTTP/1.0" 200 277 "-" "-"',
+          '127.0.0.1 - - [23/Oct/2011:08:20:01 -0700] "GET / HTTP/1.0" 200 492 "-" "Wget/1.12 (linux-gnu)"',
+          '68.64.37.100 - - [24/Oct/2011:01:48:54 -0700] "GET /phpMyAdmin/scripts/setup.php HTTP/1.1" 404 480 "-" "ZmEu"'
+        ],
+        [
+          {"host" => "58.83.188.60", "user" => nil, "time" => 1319382946, "method" => "HEAD", "path" => "/", "code" => 200, "size" => 277, "referer" => nil, "agent" => nil},
+          {"host" => "127.0.0.1", "user" => nil, "time" => 1319383201, "method" => "GET", "path" => "/", "code" => 200, "size" => 492, "referer" => nil, "agent" => "Wget/1.12 (linux-gnu)"},
+          {"host" => "68.64.37.100", "user" => nil, "time" => 1319446134, "method" => "GET", "path" => "/phpMyAdmin/scripts/setup.php", "code" => 404, "size" => 480, "referer" => nil, "agent" => "ZmEu"},
+        ]
+      ],
+      'syslog' => [
+        [
+          'Dec 20 12:41:44 localhost kernel: [4843680.692840] e1000e: eth2 NIC Link is Down',
+          'Dec 20 12:41:44 localhost kernel: [4843680.734466] br0: port 1(eth2) entering disabled state',
+          'Dec 22 10:42:41 localhost kernel[10000]: [5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]',
+        ],
+        [
+          {"pid" => nil, "time" => 1355974904, "host" => "localhost", "ident" => "kernel", "message" => "[4843680.692840] e1000e: eth2 NIC Link is Down"},
+          {"pid" => nil, "time" => 1355974904, "host" => "localhost", "ident" => "kernel", "message" => "[4843680.734466] br0: port 1(eth2) entering disabled state"},
+          {"pid" => 10000, "time" => 1356140561, "host" => "localhost", "ident" => "kernel", "message" => "[5009052.220155] zsh[25578]: segfault at 7fe849460260 ip 00007fe8474fd74d sp 00007fffe3bdf0e0 error 4 in libc-2.11.1.so[7fe847486000+17a000]"},
+        ]
+      ]
+    }.each_pair { |format, (input, output)|
+      context format do
+        let :lines do
+          input
+        end
+
+        let :dataset do
+          output
+        end
+
+        let :time_column do
+          'time'
+        end
+
+        it_should_behave_like 'parse --time-value / --time-column cases', format
+      end
+    }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'json'
+
+if ENV['SIMPLE_COV']
+  # SimpleCov
+  # https://github.com/colszowka/simplecov
+  require 'simplecov'
+  SimpleCov.start do 
+    add_filter 'spec/'
+    add_filter 'pkg/'
+    add_filter 'vendor/'
+  end
+end

data/td.gemspec

@@ -20,7 +20,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1.0"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.5.19"
-  gem.add_dependency "td-client", "~> 0.8.40"
+  gem.add_dependency "td-client", "~> 0.8.42"
   gem.add_dependency "td-logger", "~> 0.3.16"
   gem.add_development_dependency "rake", "~> 0.9"
+  gem.add_development_dependency "rspec", "~> 2.10.0"
+  gem.add_development_dependency "simplecov", "~> 0.5.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.65
+  version: 0.10.66
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-27 00:00:00.000000000 Z
+date: 2013-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.40
+        version: 0.8.42
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.40
+        version: 0.8.42
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +123,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.4
 description: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 email: support@treasure-data.com
 executables:
@@ -179,6 +211,13 @@ files:
 - lib/td/distribution.rb
 - lib/td/file_reader.rb
 - lib/td/version.rb
+- spec/file_reader/filter_spec.rb
+- spec/file_reader/io_filter_spec.rb
+- spec/file_reader/line_reader_spec.rb
+- spec/file_reader/parsing_reader_spec.rb
+- spec/file_reader/shared_context.rb
+- spec/file_reader_spec.rb
+- spec/spec_helper.rb
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
@@ -204,4 +243,12 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
-test_files: []
+test_files:
+- spec/file_reader/filter_spec.rb
+- spec/file_reader/io_filter_spec.rb
+- spec/file_reader/line_reader_spec.rb
+- spec/file_reader/parsing_reader_spec.rb
+- spec/file_reader/shared_context.rb
+- spec/file_reader_spec.rb
+- spec/spec_helper.rb
+has_rdoc: false