embulk-parser-query_string 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 577bbbbfd20a0b5ac1a95053773531166aa6abf4
-  data.tar.gz: d371c84d90510138671ff77fcd991d73596f5213
+  metadata.gz: 4aacb287d4c071f254e3876d81896df78369f2a9
+  data.tar.gz: 214293101b683e49c23d05b1a85b42119ffdc5c9
 SHA512:
-  metadata.gz: df8f61b4304e53d7e6fa13cbbaa10506db82e911c99b1ed65f4c02047144c3c84b4c7eddf1b88bf17e6b4d878613f35ad6c2ac3fef2f8650901476f2a893f8e5
-  data.tar.gz: 23f03b8da79ef4e96911f2477dbe686198ca97c47a09609e5038d1b0d17b7808f46060c2fe39200f8052082807969be4c469c4b2cbb20274c602b3efba36d3d5
+  metadata.gz: 36517f8ac5bb32b3eb2e41683e305e081d2152ce096ce590a6dc603081da1f2e8e7b919807f15b13fe9dab725281c0443d3b2d09583dc318625a37cb916c248b
+  data.tar.gz: 397c4fe3fa874d46a5eb34a54de7b87254d1757f6661a0d7170a7b8335924099a6dbc94ab3d54f40fbcc3b052ec7f9169a7e263defdc8ec71a9eb6a81eda57be

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.2.0 - 2015-07-29
+
+This version breaks backword compatibility. With this version, if you use config created by embulk-parser-query_string 0.1.3 or earlier, you should replace `schema:` key name with `columns:` in your config file (e.g. `config.yml`) .
+
+* [fixed] Use "column" as key for schema in config file [#22](https://github.com/treasure-data/embulk-parser-query_string/pull/22) [[reported by @muga](https://github.com/treasure-data/embulk-parser-query_string/issues/21). Thanks!!]
+* [enhancement] Display cast error log to human [#19](https://github.com/treasure-data/embulk-parser-query_string/pull/19)
+* [maintenace] Fix same name tests weren't run [#18](https://github.com/treasure-data/embulk-parser-query_string/pull/18)
+
 ## 0.1.3 - 2015-07-16
 * [enhancement] Fix bug nil value casting unexpectedly [#16](https://github.com/treasure-data/embulk-parser-query_string/pull/16)
 * [maintenance] Improve test [#15](https://github.com/treasure-data/embulk-parser-query_string/pull/15)

data/README.md

@@ -65,7 +65,7 @@ in:
     charset: ISO-8859-2
     newline: CRLF
     type: query_string
-    schema:
+    columns:
     - {name: user_id, type: long}
     - {name: some_param, type: string}
 exec: {}

data/embulk-parser-query_string.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-query_string"
-  spec.version       = "0.1.3"
+  spec.version       = "0.2.0"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Query String parser plugin for Embulk"
   spec.description   = "Parses Query String files read by other file input plugins."

data/lib/embulk/guess/query_string.rb

@@ -25,12 +25,12 @@ module Embulk
           end
           result
         end
-        guessed = {type: "query_string", schema: []}
+        guessed = {type: "query_string", columns: []}
         format.each_pair do |key, values|
           if values.any? {|value| value.match(/[^0-9]/) }
-            guessed[:schema] << {name: key, type: :string}
+            guessed[:columns] << {name: key, type: :string}
           else
-            guessed[:schema] << {name: key, type: :long}
+            guessed[:columns] << {name: key, type: :long}
           end
         end
         return {"parser" => guessed}

data/lib/embulk/parser/query_string.rb

@@ -17,7 +17,7 @@ module Embulk
         }
 
         columns = []
-        schema = config.param("schema", :array, default: [])
+        schema = config.param("columns", :array, default: [])
         schema.each do |column|
           name = column["name"]
           type = column["type"].to_sym
@@ -84,13 +84,18 @@ module Embulk
 
           next nil unless value
 
-          case column.type
-          when :long
-            value.strip.empty? ? nil : Integer(value)
-          when :timestamp
-            value.strip.empty? ? nil : Time.parse(value)
-          else
-            value.to_s
+          begin
+            case column.type
+            when :long
+              value.strip.empty? ? nil : Integer(value)
+            when :timestamp
+              value.strip.empty? ? nil : Time.parse(value)
+            else
+              value.to_s
+            end
+          rescue => e
+            Embulk.logger.error "Cast failed '#{value}' as '#{column.type}' (key is '#{column.name}')"
+            raise e
           end
         end
 

data/test/embulk/guess/test_query_string.rb

@@ -8,35 +8,35 @@ module Embulk
       class TestGuessLines < self
         data do
           {
-            same_keys: [sample_lines_with_same_keys, schema_with_same_keys],
-            different_keys: [sample_lines_with_different_keys, schema_with_different_keys],
-            invalid: [sample_lines_with_invalid, schema_with_invalid],
+            same_keys: [sample_lines_with_same_keys, columns_with_same_keys],
+            different_keys: [sample_lines_with_different_keys, columns_with_different_keys],
+            invalid: [sample_lines_with_invalid, columns_with_invalid],
 
           }
         end
 
-        def test_schema(data)
-          sample_lines, schema = data
+        def test_columns(data)
+          sample_lines, columns = data
           actual = QueryString.new.guess_lines(config, sample_lines)
           expected = {
             "parser" => {
               type: "query_string",
-              schema: schema
+              columns: columns
             }
           }
           assert_equal(expected, actual)
         end
 
         data do
-          valid_schema = {
+          valid_columns = {
             "parser" => {
               type: "query_string",
-              schema: schema_with_same_keys,
+              columns: columns_with_same_keys,
             }
           }
 
           {
-            "query_string" => ["query_string", valid_schema],
+            "query_string" => ["query_string", valid_columns],
             "other" => ["other", {}],
           }
         end
@@ -61,7 +61,7 @@ module Embulk
           ]
         end
 
-        def schema_with_same_keys
+        def columns_with_same_keys
           [
             {name: "foo", type: :long},
             {name: "bar", type: :string},
@@ -76,7 +76,7 @@ module Embulk
           ]
         end
 
-        def schema_with_different_keys
+        def columns_with_different_keys
           [
             {name: "foo", type: :long},
             {name: "bar", type: :string},
@@ -94,8 +94,8 @@ module Embulk
           ]
         end
 
-        def schema_with_invalid
-          schema_with_same_keys
+        def columns_with_invalid
+          columns_with_same_keys
         end
       end
 
@@ -104,7 +104,7 @@ module Embulk
           "parser" => {
             "strip_quote" => true,
             "strip_whitespace" => true,
-            "schema" => columns,
+            "columns" => columns,
           }
         }
       end

data/test/embulk/parser/{test_query_string_plugin.rb → test_query_string.rb}

@@ -4,7 +4,7 @@ require "embulk/data_source"
 
 module Embulk
   module Parser
-    class QueryStringPluginTest < Test::Unit::TestCase
+    class QueryStringTest < Test::Unit::TestCase
       class TestParse < self
         def test_without_options
           result = QueryString.parse(line)
@@ -106,7 +106,7 @@ module Embulk
 
       def test_transaction
         expected_task = task.dup
-        expected_task.delete("schema")
+        expected_task.delete("columns")
 
         QueryString.transaction(config) do |actual_task, actual_columns|
           assert_equal(expected_task, actual_task)
@@ -130,7 +130,7 @@ module Embulk
           "strip_quote" => true,
           "strip_whitespace" => true,
           "capture" => "(.*)",
-          "schema" => columns,
+          "columns" => columns,
         }
       end
 

data/test/run-test.rb

@@ -15,4 +15,4 @@ ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
 
 CodeClimate::TestReporter.start
 
-exit Test::Unit::AutoRunner.run(true, test_dir)
+exit Test::Unit::AutoRunner.run(true, test_dir, ARGV + %w(--collector=dir))

data/test/test_embulk_run.rb

@@ -97,7 +97,7 @@ in:
     charset: UTF-8
     newline: CRLF
     type: query_string
-    schema:
+    columns:
     - {name: foo, type: string}
     - {name: bar, type: long}
 exec: {}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-query_string
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-16 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -145,7 +145,7 @@ files:
 - partial-config.yml
 - test/capture_io.rb
 - test/embulk/guess/test_query_string.rb
-- test/embulk/parser/test_query_string_plugin.rb
+- test/embulk/parser/test_query_string.rb
 - test/embulk_run_helper.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
@@ -177,7 +177,7 @@ summary: Query String parser plugin for Embulk
 test_files:
 - test/capture_io.rb
 - test/embulk/guess/test_query_string.rb
-- test/embulk/parser/test_query_string_plugin.rb
+- test/embulk/parser/test_query_string.rb
 - test/embulk_run_helper.rb
 - test/prepare_embulk.rb
 - test/run-test.rb