embulk-parser-query_string 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d212e03f093b093ae862308c637069251feb71a
-  data.tar.gz: 968354927bcd4ab12998fe8c366e79e3613401bc
+  metadata.gz: 577bbbbfd20a0b5ac1a95053773531166aa6abf4
+  data.tar.gz: d371c84d90510138671ff77fcd991d73596f5213
 SHA512:
-  metadata.gz: 5e31e9f7e2675cf161686d05a4517bddeac6be16cc8eeb890c86257bff274077a8735060e8dd0d5a269a061497fec1b30a4e7b930b2f02c740a8f982be0ae12f
-  data.tar.gz: e5e3adfa549e74d627b6afa06acad1b30026811b0db9073cafaa63254156f16774ae00143dc6a9ba630f10fac6c2904d572b3d31adabfeaa4336b1efd5127416
+  metadata.gz: df8f61b4304e53d7e6fa13cbbaa10506db82e911c99b1ed65f4c02047144c3c84b4c7eddf1b88bf17e6b4d878613f35ad6c2ac3fef2f8650901476f2a893f8e5
+  data.tar.gz: 23f03b8da79ef4e96911f2477dbe686198ca97c47a09609e5038d1b0d17b7808f46060c2fe39200f8052082807969be4c469c4b2cbb20274c602b3efba36d3d5

data/.travis.yml

@@ -10,3 +10,12 @@ notifications:
     secure: "QRZ6zQxcRdN2XylIz+b46TbYIv7x5Ko7Y0Mi0aGbf7Wg1xfnS4X2FZKp8gF+RH1qjT6CYsQFUpPBhPVCMx0fP0ik5/gBrdngrj0htludvQ5CTK2oVegM1VC5NSn+gi8bt2F20Ode9fxU5OS1HDzNR9sjPz3W7miMUNDZoYPy77Om/CvmVkBbGyMkWnfwhEvwnL8JpBNpklDvrUUKznNsAcSbKQBMBAglFMds/OaHXCkK142zMbSlHMctch1VDAKhX9/6M3v8v6ZRheX68+VVCz3nTrTF0/lnYh3ig9ey3DNlgkZUqK2gmypnhTE6tpeibHdyVa8qOMNzyAgltu1qIhFHN390WMk9gZXEcvWG3m++PMAIKKtd4varbH1vdCeHOGUV7GRgarcfSMvJ54I9xc5GkUnAhjRKj8xDzYzhkXwApD8eqWb2RgGXUm6G4cKj0MeQ4WBqHr9FOW/4EJcZoztFi0YlP7KE+6xptBJU479KSsv31BwHrMfblObOOj58C7gUfdbTCByxng0u1axxvixL8zJwgHvFND3/4yoXoCCgm0DXBPbi6B5xpt8QmN6K4E5sMGrppTI7VwJqkcwkVD0EEeCedRNiWLTaxlKpLYxYnoXwEnpm9o2WXWa+PjkWeawsnJvV5IAt6oJhaK22VqJN86Macuu+nCdeF1/0TZU="
 jdk:
 - oraclejdk8
+gemfile:
+  - gemfiles/embulk-0.6.10
+  - gemfiles/embulk-0.6.11
+  - gemfiles/embulk-0.6.12
+  - gemfiles/embulk-0.6.13
+  - gemfiles/embulk-0.6.14
+  - gemfiles/embulk-0.6.15
+  - gemfiles/embulk-0.6.16
+  - gemfiles/embulk-latest

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.3 - 2015-07-16
+* [enhancement] Fix bug nil value casting unexpectedly [#16](https://github.com/treasure-data/embulk-parser-query_string/pull/16)
+* [maintenance] Improve test [#15](https://github.com/treasure-data/embulk-parser-query_string/pull/15)
+
 ## 0.1.2 - 2015-07-14
 * [fixed] Fix to ignore empty line same as invalid line [#14](https://github.com/treasure-data/embulk-parser-query_string/pull/14)
 

data/embulk-parser-query_string.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-query_string"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Query String parser plugin for Embulk"
   spec.description   = "Parses Query String files read by other file input plugins."

data/gemfiles/embulk-0.6.10

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.10"

data/gemfiles/embulk-0.6.11

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.11"

data/gemfiles/embulk-0.6.12

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.12"

data/gemfiles/embulk-0.6.13

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.13"

data/gemfiles/embulk-0.6.14

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.14"

data/gemfiles/embulk-0.6.15

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.15"

data/gemfiles/embulk-0.6.16

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.16"

data/gemfiles/embulk-latest

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'

data/lib/embulk/parser/query_string.rb

@@ -82,11 +82,13 @@ module Embulk
           name = column.name
           value = record[name]
 
+          next nil unless value
+
           case column.type
           when :long
-            Integer(value)
+            value.strip.empty? ? nil : Integer(value)
           when :timestamp
-            Time.parse(value)
+            value.strip.empty? ? nil : Time.parse(value)
           else
             value.to_s
           end

data/test/capture_io.rb

@@ -0,0 +1,45 @@
+module CaptureIo
+  def capture(output = :out, &block)
+    _, out = swap_io(output, &block)
+    out
+  end
+
+  def silence(&block)
+    block_result = nil
+    swap_io(:out) do
+      block_result,_ = swap_io(:err, &block)
+    end
+    block_result
+  end
+
+  def swap_io(output = :out, &block)
+    java_import 'java.io.PrintStream'
+    java_import 'java.io.ByteArrayOutputStream'
+    java_import 'java.lang.System'
+
+    ruby_original_stream = output == :out ? $stdout.dup : $stderr.dup
+    java_original_stream = System.send(output) # :out or :err
+    ruby_buf = StringIO.new
+    java_buf = ByteArrayOutputStream.new
+
+    case output
+    when :out
+      $stdout = ruby_buf
+      System.setOut(PrintStream.new(java_buf))
+    when :err
+      $stderr = ruby_buf
+      System.setErr(PrintStream.new(java_buf))
+    end
+
+    [block.call, ruby_buf.string + java_buf.toString]
+  ensure
+    case output
+    when :out
+      $stdout = ruby_original_stream
+      System.setOut(java_original_stream)
+    when :err
+      $stderr = ruby_original_stream
+      System.setErr(java_original_stream)
+    end
+  end
+end

data/test/embulk/guess/test_query_string.rb

@@ -101,10 +101,10 @@ module Embulk
 
       def task
         {
-          parser: {
-            strip_quote: true,
-            strip_whitespace: true,
-            schema: columns,
+          "parser" => {
+            "strip_quote" => true,
+            "strip_whitespace" => true,
+            "schema" => columns,
           }
         }
       end

data/test/embulk/parser/test_query_string_plugin.rb

@@ -74,16 +74,34 @@ module Embulk
       end
 
       class TestProcessLine < self
-        def test_process_line
+        def test_all_keys_exist_in_schema
           mock(page_builder).add(["FOO", 1, Time.parse("2015-07-08T16:25:46")])
           plugin.send(:process_line, line)
         end
 
+        def test_key_nonexist_in_schema
+          nonexist_column = Column.new(nil, "different", :long)
+          nonexist_schema = schema << nonexist_column
+
+          plugin = QueryString.new(DataSource[task], nonexist_schema, page_builder)
+          mock(page_builder).add(["FOO", 1, Time.parse("2015-07-08T16:25:46"), nil])
+          plugin.send(:process_line, line)
+        end
+
+        def test_long_value_for_key_nonexist
+          mock(page_builder).add(["FOO", nil, Time.parse("2015-07-08T16:25:46")])
+          plugin.send(:process_line, value_nonexist_line)
+        end
+
         private
 
         def line
           "foo=FOO&bar=1&baz=2015-07-08T16:25:46"
         end
+
+        def value_nonexist_line
+          "foo=FOO&bar=&baz=2015-07-08T16:25:46"
+        end
       end
 
       def test_transaction

data/test/embulk_run_helper.rb

@@ -0,0 +1,19 @@
+require "capture_io"
+
+module EmbulkRunHelper
+  include CaptureIo
+
+  def embulk_guess(seed_path, dest_path)
+    silence do
+      embulk_exec(%W(guess -g query_string #{seed_path} -o #{dest_path}))
+    end
+  end
+
+  def embulk_run(yaml_path)
+    embulk_exec(%W(run #{yaml_path}))
+  end
+
+  def embulk_exec(cli_options = [])
+    Embulk.run(cli_options)
+  end
+end

data/test/test_embulk_run.rb

@@ -0,0 +1,109 @@
+require "yaml"
+require "tmpdir"
+require "prepare_embulk"
+require "embulk_run_helper"
+
+module Embulk
+  class EmbulkRunTest < Test::Unit::TestCase
+    include EmbulkRunHelper
+
+    def setup
+      @dir = Dir.mktmpdir
+      @target_file = "#{@dir}/target_file.txt"
+      File.open(@target_file, "w") do |f|
+        f.write <<-FILE
+          "foo=FOO&bar=1"
+          "foo=FOO&bar=2"
+          "foo=FOO&bar=3"
+          "foo=FOO&bar=4"
+          "foo=FOO&bar=5"
+          "foo=FOO&bar=6"
+          "foo=FOO&bar=7"
+          "foo=FOO&bar=8"
+          "foo=FOO&bar=9"
+          "foo=FOO&bar=10"
+          "foo=FOO&bar=11"
+          "foo=FOO&bar=12"
+          "foo=FOO&bar=13"
+        FILE
+      end
+    end
+
+    def teardown
+      FileUtils.rm_rf @dir
+    end
+
+    def test_embulk_run
+      config_path = "#{@dir}/config.yml"
+      File.open(config_path, "w") do |f|
+        f.write guessed_content
+      end
+      out = capture do
+        embulk_run(config_path)
+      end
+
+      assert_true(out.include?(<<-CONTENT))
+FOO,1
+FOO,2
+FOO,3
+FOO,4
+FOO,5
+FOO,6
+FOO,7
+FOO,8
+FOO,9
+FOO,10
+FOO,11
+FOO,12
+FOO,13
+      CONTENT
+    end
+
+    def test_embulk_guess
+      seed_path = "#{@dir}/seed.yml"
+      File.open(seed_path, "w") do |f|
+        f.write seed_content
+      end
+      dest_path = "#{@dir}/guessed.yml"
+      embulk_guess(seed_path, dest_path)
+      guessed = IO.read(dest_path)
+
+      assert_equal(guessed_content, guessed)
+    end
+
+    private
+
+    def seed_content
+      <<YAML
+in:
+  type: file
+  path_prefix: #{@dir}/target_file
+  parser:
+    strip_quote: true
+    strip_whitespace: true
+exec: {}
+out: {type: stdout}
+YAML
+    end
+
+    def guessed_content
+      <<YAML
+in:
+  type: file
+  path_prefix: #{@dir}/target_file
+  parser:
+    strip_quote: true
+    strip_whitespace: true
+    charset: UTF-8
+    newline: CRLF
+    type: query_string
+    schema:
+    - {name: foo, type: string}
+    - {name: bar, type: long}
+exec: {}
+out: {type: stdout}
+YAML
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-query_string
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,13 +132,24 @@ files:
 - README.md
 - Rakefile
 - embulk-parser-query_string.gemspec
+- gemfiles/embulk-0.6.10
+- gemfiles/embulk-0.6.11
+- gemfiles/embulk-0.6.12
+- gemfiles/embulk-0.6.13
+- gemfiles/embulk-0.6.14
+- gemfiles/embulk-0.6.15
+- gemfiles/embulk-0.6.16
+- gemfiles/embulk-latest
 - lib/embulk/guess/query_string.rb
 - lib/embulk/parser/query_string.rb
 - partial-config.yml
+- test/capture_io.rb
 - test/embulk/guess/test_query_string.rb
 - test/embulk/parser/test_query_string_plugin.rb
+- test/embulk_run_helper.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/test_embulk_run.rb
 homepage: https://github.com/treasure-data/embulk-parser-query_string
 licenses:
 - Apache2
@@ -164,7 +175,10 @@ signing_key:
 specification_version: 4
 summary: Query String parser plugin for Embulk
 test_files:
+- test/capture_io.rb
 - test/embulk/guess/test_query_string.rb
 - test/embulk/parser/test_query_string_plugin.rb
+- test/embulk_run_helper.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/test_embulk_run.rb