embulk-parser-xml 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da39918e4e87ef06f5a8a3f1c321062ef00f10b4
-  data.tar.gz: e440ca6333be465a791abeb0b0998e02f7251440
+  metadata.gz: 31b1d69fab1c3bfcf1e676508a8f7e8c644db85e
+  data.tar.gz: 86d1162369def6860f2471c008b0842469db303a
 SHA512:
-  metadata.gz: 010d16da3d32201b038fe0f219eae33ae19115bf2d465e904c30035b5d1b093197b8067b9dc11593c4e8ab6bfed8360ff640342e7626c508c74a9d14ce746be8
-  data.tar.gz: 89467d469d2d7547b03c04102a8c4a1216318daad0a463d03b7ccb3749e013acfcfb632d89e42ec7654ddd1825d089ac28f48c0158d5a325a513942cd963f50e
+  metadata.gz: 2f81442fd1695ccb607a1f056a190432a3ed94b146a4650cbcf2d9d6ebe10e716a2068e41860ee18a275c36223b1bec51ce3bca4718700229079d452d069ae64
+  data.tar.gz: 6e8c4e50183802026bddab9eea6b298ff2f06af6ac8a773cbf0ae3afda50feaa095127163052c5aac648cd1364ef6d899f96208ef2b10f83674feba04c6c5422

data/embulk-parser-xml.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-xml"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["Takuma kanari"]
   spec.email         = ["chemtrails.t@gmail.com"]
   spec.summary       = %q{Embulk parser plugin for XML}
@@ -16,6 +16,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "nokogiri", "~> 1.6"
   spec.add_development_dependency "bundler", "~> 1.0"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/lib/embulk/parser/xml.rb

@@ -1,4 +1,4 @@
-require "rexml/document"
+require "nokogiri"
 
 module Embulk
   module Parser
@@ -7,54 +7,111 @@ module Embulk
       Plugin.register_parser("xml", self)
 
       def self.transaction(config, &control)
+        schema = config.param("schema", :array)
+        schema_serialized = schema.inject({}) do |memo, s|
+          memo[s["name"]] = s["type"]
+          memo
+        end
         task = {
-          :schema => config.param("schema", :array),
-          :root => config.param("root", :string)
+          :schema => schema_serialized,
+          :root_to_route => config.param("root", :string).split("/")
         }
-        columns = task[:schema].each_with_index.map do |c, i|
+        columns = schema.each_with_index.map do |c, i|
           Column.new(i, c["name"], c["type"].to_sym)
         end
         yield(task, columns)
       end
 
       def run(file_input)
-        schema = @task["schema"]
-        root = @task["root"]
+        on_new_record = lambda {|record|
+          @page_builder.add(record)
+        }
+        doc = RecordBinder.new(@task["root_to_route"],
+                               @task["schema"], on_new_record)
+        parser = Nokogiri::XML::SAX::Parser.new(doc)
         while file = file_input.next_file
-          REXML::Document.new(file.read).elements.each(root) do |e|
-            dest = {}
-            e.elements.each do |d|
-              dest[d.name] = d.text
-            end
-            @page_builder.add(make_record(schema, dest))
-          end
+          parser.parse(file.read)
+          doc.clear
         end
         @page_builder.finish
       end
+    end
 
-      private
+    class RecordBinder  < Nokogiri::XML::SAX::Document
 
-      def make_record(schema, e)
-        schema.map do |c|
-          name = c["name"]
-          val = e[name]
-
-          v = val.nil? ? "" : val
-          type = c["type"]
-          case type
-            when "string"
-              v
-            when "long"
-              v.to_i
-            when "double"
-              v.to_f
-            when "boolean"
-              ["yes", "true", "1"].include?(v.downcase)
-            when "timestamp"
-              v.empty? ? nil : Time.strptime(v, c["format"])
+      def initialize(route, schema, on_new_record)
+        @route = route
+        @schema = schema
+        @on_new_record = on_new_record
+        clear
+        super()
+      end
+
+      def clear
+        @find_route_idx = 0
+        @enter = false
+        @current_element_name = nil
+        @current_data = new_map_by_schema
+      end
+
+      def start_element(name, attributes = [])
+        if !@enter
+          if name == @route[@find_route_idx]
+            if @find_route_idx == @route.size - 1
+              @enter = true
             else
-              raise "Unsupported type #{type}"
+              @find_route_idx += 1
+            end
           end
+        else
+          @current_element_name = (@schema[name].nil?) ? nil : name
+        end
+      end
+
+      def characters(string)
+        return if !@enter || string.strip.size == 0 || @current_element_name.nil?
+        val = @current_data[@current_element_name]
+        val = "" if val.nil?
+        val += string
+        @current_data[@current_element_name] = val
+      end
+
+      def end_element(name, attributes = [])
+        if @enter
+          if name == @route.last
+            @enter = false
+            @on_new_record.call(@current_data.map{|k, v| v})
+            @current_data = new_map_by_schema
+          elsif !@current_element_name.nil?
+            @current_data[name] = convert(@current_data[name], @schema[name])
+          end
+        end
+      end
+
+      private
+
+      def new_map_by_schema
+        @schema.keys.inject({}) do |memo, k|
+          memo[k] = nil
+          memo
+        end
+      end
+
+      def convert(val, type)
+        v = val.nil? ? "" : val
+        case type
+          when "string"
+            v
+          when "long"
+            v.to_i
+          when "double"
+            v.to_f
+          when "boolean"
+            ["yes", "true", "1"].include?(v.downcase)
+          when "timestamp"
+            v.empty? ? nil : Time.strptime(v, c["format"])
+          else
+            raise "Unsupported type #{type}"
         end
       end
     end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-xml
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Takuma kanari
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-14 00:00:00.000000000 Z
+date: 2015-04-18 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement