embulk-parser-xpath 0.0.1

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,22 @@
+PATH
+  remote: .
+  specs:
+    embulk-parser-xpath (0.0.1)
+      nokogiri (~> 1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    nokogiri (1.6.7.1-java)
+    rake (10.4.2)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  embulk-parser-xpath!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/embulk-parser-xpath.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-parser-xpath"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Tatsunori Matoba"]
+  spec.email         = ["matobat@gmail.com"]
+  spec.summary       = %q{Embulk parser plugin for XML with XPath support}
+  spec.description   = %q{XPath parser plugin is Embulk plugin to fetch entries in xml format use XPath.}
+  spec.homepage      = "https://github.com/matobat/embulk-parser-xpath"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri", "~> 1.6"
+  spec.add_development_dependency "bundler", "~> 1.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/embulk/parser/xpath.rb

@@ -0,0 +1,115 @@
+require "nokogiri"
+
+module Embulk
+  module Parser
+
+    class XPathParserPlugin < ParserPlugin
+      Plugin.register_parser("xpath", self)
+
+      def self.transaction(config, &control)
+        schema = config.param("schema", :array)
+        schema_serialized = schema.inject({}) do |memo, s|
+          memo[s["name"]] = s["type"]
+          memo
+        end
+        task = {
+          :schema => schema_serialized,
+          :root => config.param("root", :string),
+          :namespaces => config.param("namespaces", :hash)
+        }
+        columns = schema.each_with_index.map do |c, i|
+          Column.new(i, c["name"], c["type"].to_sym)
+        end
+        yield(task, columns)
+      end
+
+      def run(file_input)
+        on_new_record = lambda {|record|
+          @page_builder.add(record)
+        }
+        doc = RecordBinder.new(@task["root"],
+                               @task["schema"], @task["namespaces"], on_new_record)
+
+        while file = file_input.next_file
+          data = file.read
+          if !data.nil? && !data.empty?
+            doc.clear
+            doc.parse(data)
+          end
+        end
+        @page_builder.finish
+      end
+    end
+
+    class RecordBinder 
+
+      def initialize(root, schema, namespaces, on_new_record)
+        @root = root
+        @schema = schema
+        @on_new_record = on_new_record
+        @namespaces = namespaces
+        clear
+        super()
+      end
+
+      def clear
+        @find_route_idx = 0
+        @enter = false
+        @current_element_name = nil
+        @current_data = new_map_by_schema
+      end
+
+      def parse(data)
+        doc = Nokogiri::XML(data)
+
+        items = doc.xpath(@root, @namespaces)
+        items.each do |item|
+          @current_data = new_map_by_schema
+         
+          @schema.each do |key,value|
+            doc.xpath(key, @namespaces).each do |item|
+              @current_data[key] = item.text
+            end
+          end
+          
+          @on_new_record.call(@current_data.map{|k, v| v})
+        end
+      end
+
+      def characters(string)
+        return if !@enter || string.strip.size == 0 || @current_element_name.nil?
+        val = @current_data[@current_element_name]
+        val = "" if val.nil?
+        val += string
+        @current_data[@current_element_name] = val
+      end
+
+      private
+
+      def new_map_by_schema
+        @schema.keys.inject({}) do |memo, k|
+          memo[k] = nil
+          memo
+        end
+      end
+
+      def convert(val, type)
+        v = val.nil? ? "" : val
+        case type
+          when "string"
+            v
+          when "long"
+            v.to_i
+          when "double"
+            v.to_f
+          when "boolean"
+            ["yes", "true", "1"].include?(v.downcase)
+          when "timestamp"
+            v.empty? ? nil : Time.strptime(v, c["format"])
+          else
+            raise "Unsupported type '#{type}'"
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification 
+name: embulk-parser-xpath
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Tatsunori Matoba
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2016-01-05 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 6
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 1
+        - 0
+        version: "1.0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 35
+        segments: 
+        - 10
+        - 0
+        version: "10.0"
+  type: :development
+  version_requirements: *id003
+description: XPath parser plugin is Embulk plugin to fetch entries in xml format use XPath.
+email: 
+- matobat@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- Rakefile
+- embulk-parser-xpath.gemspec
+- lib/embulk/parser/xpath.rb
+has_rdoc: true
+homepage: https://github.com/matobat/embulk-parser-xpath
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Embulk parser plugin for XML with XPath support
+test_files: []
+