embulk-parser-xml 0.0.5 → 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a19b485eaa8f3a9143e2d1b2eb8a07a6b7fe616
+  data.tar.gz: 859ef953a4d17ca9b24c8060c6950231887949db
+SHA512:
+  metadata.gz: 5c3ada6ad8d47dc867bf62c4bb3f60c4a690a84b69e9e41a048f87d0ba5dc0bd19f0ca92f90fdcc515c4d648821b983a01bc1e22c9aa7d0fde02017fccb556ed
+  data.tar.gz: 621bde05d19a7f3f58199ae681a891a39ec552d9b0a3a5b751f97983a711879e40312128d47b66ad49c0f5f4147722b157f759945534917186212ece394fddab

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.4.0

data/README.md

@@ -10,9 +10,15 @@ Read data from input as xml and fetch each entries to output.
 * **Load all or nothing**: yes
 * **Resume supported**: no
 
+## Types
+
+- **xml**:   Find rows by SAX.
+- **xpath**: Find finds rows by Xpath, so you can process XML by more complex condition than *xml* type.
 
 ## Configuration
 
+### XML
+
 ```yaml
 parser:
   type: xml
@@ -32,10 +38,41 @@ If you need to parse column as timestamp type, *schema* supports 2 optional para
 schema:
   - {name: timestamp_column, type: timestamp, format: "%Y-%m-%d", timezone: "+0000"}
 ```
+
+- **format**: timestamp format to parse, required.
+- **timezone**: timestamp will be parsing in this timezone, `"+0900"` is used by default.
+
+
+### Xpath
+
+```yaml
+parser:
+  type: xpath
+  root: //data/students/student
+  schema:
+    - {path: name, type: string, name: name}
+    - {path: age, type: long, name: age}
+```
+
+- **type**: specify this plugin as `xpath` .
+- **root**: root property to start fetching each entries, specify in Xpath, *'/''* is used by default.
+- **schema**: specify the attribute of table and data type, required.
+- **namespaces**: xml namespaces
+
+
+If you need to parse column as timestamp type, *schema* supports 2 optional parameters:
+
+```yaml
+schema:
+  - {name: timestamp_column, type: timestamp, format: "%Y-%m-%d", timezone: "+0000"}
+```
+
 - **format**: timestamp format to parse, required.
 - **timezone**: timestamp will be parsing in this timezone, `"+0900"` is used by default.
 
-Then you can fetch entries from the following xml:
+
+
+Here is XML for xample:
 
 ```xml
 <data>
@@ -59,4 +96,4 @@ Then you can fetch entries from the following xml:
     </student>
   </students>
 </data>
-```
+```

data/embulk-parser-xml.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-xml"
-  spec.version       = "0.0.5"
+  spec.version       = "0.0.7"
   spec.authors       = ["Takuma kanari"]
   spec.email         = ["chemtrails.t@gmail.com"]
   spec.summary       = %q{Embulk parser plugin for XML}
@@ -16,7 +16,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "nokogiri", "~> 1.6"
+  spec.add_dependency "nokogiri", "~> 1.4.0"
   spec.add_development_dependency "bundler", "~> 1.0"
+  spec.add_development_dependency 'embulk', ['>= 0.8.8']
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/lib/embulk/parser/xml.rb

@@ -20,7 +20,7 @@ module Embulk
         end
         task = {
           :schema => schema_serialized,
-          :root_to_route => config.param("root", :string).split("/")
+          :root => config.param("root", :string).split("/")
         }
         columns = schema.each_with_index.map do |c, i|
           Column.new(i, c["name"], c["type"].to_sym)
@@ -29,15 +29,11 @@ module Embulk
       end
 
       def run(file_input)
-        on_new_record = lambda {|record|
-          @page_builder.add(record)
-        }
-        doc = RecordBinder.new(@task["root_to_route"],
-                               @task["schema"], on_new_record)
+        doc = RecordBinder.new(@task["root"], @task["schema"], @page_builder)
         parser = Nokogiri::XML::SAX::Parser.new(doc)
         while file = file_input.next_file
           data = file.read
-          if !data.nil? && !data.empty?
+          unless data.nil?
             doc.clear
             parser.parse(data)
           end
@@ -48,10 +44,10 @@ module Embulk
 
     class RecordBinder  < Nokogiri::XML::SAX::Document
 
-      def initialize(route, schema, on_new_record)
+      def initialize(route, schema, page_builder)
         @route = route
         @schema = schema
-        @on_new_record = on_new_record
+        @page_builder = page_builder
         clear
         super()
       end
@@ -89,7 +85,7 @@ module Embulk
         if @enter
           if name == @route.last
             @enter = false
-            @on_new_record.call(@current_data.map{|k, v| v})
+            @page_builder.add(@current_data.map{|k, v| v})
             @current_data = new_map_by_schema
           elsif !@current_element_name.nil? && @schema.key?(name)
             @current_data[name] = convert(@current_data[name], @schema[name])
@@ -131,6 +127,5 @@ module Embulk
         end
       end
     end
-
   end
 end

data/lib/embulk/parser/xpath.rb

@@ -0,0 +1,60 @@
+require "nokogiri"
+
+module Embulk
+  module Parser
+
+    class XPathParserPlugin < ParserPlugin
+      Plugin.register_parser("xpath", self)
+
+      def self.transaction(config, &control)
+        schema = config.param("schema", :array)
+        task = {
+          :schema => schema,
+          :root => config.param("root", :string, default: "/"),
+          :namespaces => config.param("namespaces", :hash, default: {})
+        }
+        columns = schema.each_with_index.map do |c, i|
+          path, name = c["path"], c["name"]
+          Column.new(i, name.nil? ? path : name, c["type"].to_sym)
+        end
+        yield(task, columns)
+      end
+
+      def run(file_input)
+        while file = file_input.next_file
+          data = file.read
+          if !data.nil? && !data.empty?
+            Nokogiri::XML(data).xpath(@task["root"], @task["namespaces"]).each do |item|
+              dest = @task["schema"].inject([]) do |memo, s|
+                es = item.xpath(s["path"], @namespaces)
+                memo << convert(es.empty? ? nil : es.text, s["type"])
+                memo
+              end
+              @page_builder.add(dest)
+            end
+          end
+        end
+        @page_builder.finish
+      end
+
+      private
+      def convert(val, type)
+        v = val.nil? ? "" : val
+        case type
+          when "string"
+            v
+          when "long"
+            v.to_i
+          when "double"
+            v.to_f
+          when "boolean"
+            ["yes", "true", "1"].include?(v.downcase)
+          when "timestamp"
+            v.empty? ? nil : Time.strptime(v, c["format"])
+          else
+            raise "Unsupported type '#{type}'"
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,64 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-xml
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.0.7
 platform: ruby
 authors:
 - Takuma kanari
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-27 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :runtime
-  prerelease: false
+        version: 1.4.0
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.4.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: embulk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: XML parser plugin is Embulk plugin to fetch entries in xml format.
 email:
 - chemtrails.t@gmail.com
@@ -66,36 +73,37 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - embulk-parser-xml.gemspec
 - lib/embulk/parser/xml.rb
+- lib/embulk/parser/xpath.rb
 homepage: https://github.com/takumakanari/embulk-parser-xml
 licenses:
 - MIT
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.23
-signing_key: 
-specification_version: 3
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
 summary: Embulk parser plugin for XML
 test_files: []