embulk-parser-json 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a50a899523c149951f3bb4b67f917ed7be074d0
-  data.tar.gz: eff2a6db83ee9af56a4588c1951b28093732094d
+  metadata.gz: b6f1282a901442d18e42cc128c2d3bd655ea6c13
+  data.tar.gz: 6ecce69a72ef4ef3937eb5a6e4c543eeb0589895
 SHA512:
-  metadata.gz: 66d5ab493130ac329c0a2790c2803d294f9860e8949a1c6706cae2b8f7cffca272b2cf5f97bf3468ce24c09d7b2ee5207d17b707d5551214b2f285b318cd5ca3
-  data.tar.gz: 3f7f230abb7b10dc89a6385732ed5108e236328d4950e0a6b8518bdad59edb86f3add74a83e39e9762c69eb12de808fd7af16b6579ea677430d68c36b66e31a6
+  metadata.gz: c26df186cf61cde5cd9b39e5f87c274a88d9760c4de9b441c390eca5fa47ee9a353744f656ee19f5de1b7921f3111b5af8360dbccb1358ea18022b71e48166a6
+  data.tar.gz: 2c7ecd428848dc85478af476cd5009ab4e15777b40429602c89d847c990efe3ea6b09f3b191249de7347dc19516e8f1f762241ca290514ff167aeb5e0d833fdd

data/README.md

@@ -22,6 +22,7 @@ A type name has been changed from **json** to **jsonpath** from v0.0.3.
 parser:
   type: jsonpath
   root: $.response.station
+  stop_on_invalid_record: false
   schema:
     - {name: name, type: string}
     - {name: next, type: string}
@@ -34,9 +35,10 @@ parser:
     - {name: optionals, type: json}
 ```
 
-- **type**: specify this plugin as `json`
-- **root**: root property to start fetching each entries, specify in [jsonpath](http://goessner.net/articles/JsonPath/) style, required
-- **schema**: specify the attribute of table and data type, required
+- **type**: Specify this plugin as `json`
+- **root**: Root property to start fetching each entries, specify in [jsonpath](http://goessner.net/articles/JsonPath/) style, required
+- **schema**: Specify the attribute of table and data type, required
+- **stop_on_invalid_record**: Stop bulk load transaction if a file includes invalid record, false by default
 
 
 ## Example

data/embulk-parser-json.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-json"
-  spec.version       = "0.0.5"
+  spec.version       = "0.0.6"
   spec.authors       = ["Takuma kanari"]
   spec.email         = ["chemtrails.t@gmail.com"]
   spec.summary       = %q{Embulk parser plugin for json with jsonpath}

data/lib/embulk/parser/jsonpath.rb

@@ -9,8 +9,9 @@ module Embulk
 
       def self.transaction(config, &control)
         task = {
-          :schema => config.param("schema", :array),
-          :root => config.param("root", :string)
+          schema: config.param("schema", :array),
+          root: config.param("root", :string),
+          stop_on_invalid_record: config.param("stop_on_invalid_record", :bool, default: false)
         }
         columns = task[:schema].each_with_index.map do |c, i|
           Column.new(i, c["name"], c["type"].to_sym)
@@ -21,9 +22,16 @@ module Embulk
       def run(file_input)
         root = JsonPath.new(@task["root"])
         schema = @task["schema"]
+        stop_on_invalid_record = @task["stop_on_invalid_record"]
         while file = file_input.next_file
-          root.on(JSON.parse(file.read)).flatten.each do |e|
-            @page_builder.add(make_record(schema, e))
+          json_text = file.read
+          begin
+            root.on(JSON.parse(json_text)).flatten.each do |e|
+              @page_builder.add(make_record(schema, e))
+            end
+          rescue JSON::ParserError, DataParseError => e
+            raise "Invalid record: '#{json_text}' (#{e})" if stop_on_invalid_record
+            Embulk.logger.warn "Skipped record (#{e}): '#{json_text}'"
           end
         end
         @page_builder.finish
@@ -59,7 +67,15 @@ module Embulk
                 !!val
               end
             when "timestamp"
-              val.nil? || val.empty? ? nil : Time.strptime(val, c["format"])
+              if val.nil? || val.empty?
+                nil
+              else
+                begin
+                  Time.strptime(val, c["format"])
+                rescue ArgumentError => e
+                  raise DataParseError.new e
+                end
+              end
             else
               raise "Unsupported type #{type}"
           end
@@ -69,6 +85,8 @@ module Embulk
       def kind_of_boolean?(val)
         val.kind_of?(TrueClass) || val.kind_of?(FalseClass)
       end
+
+      class DataParseError < StandardError ; end
     end
   end
 end

metadata

@@ -1,59 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-json
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Takuma kanari
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jsonpath
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  type: :runtime
+  name: jsonpath
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Json parser plugin is Embulk plugin to fetch entries in json format with
-  jsonpath.
+description: Json parser plugin is Embulk plugin to fetch entries in json format with jsonpath.
 email:
 - chemtrails.t@gmail.com
 executables: []
@@ -71,7 +70,7 @@ homepage: https://github.com/takumakanari/embulk-parser-json
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -86,9 +85,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
 specification_version: 4
 summary: Embulk parser plugin for json with jsonpath
 test_files: []