embulk-input-http 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d8ea58d778613a1a60214badcea8ea098f9dab77
+  data.tar.gz: c5eb80607d79558874a009a2dc47f12696053c7f
+SHA512:
+  metadata.gz: 98ab84c0d458317898947ded60b4edfd6b4c132b1aaea930a485d76b14e408b742863fac125172bb46d86d54e9e6b6ea6c3aa6229832054b0dbd3233361ffe90
+  data.tar.gz: 984d9997c12a42438546d8b4acf28053fd2d3c6b5620fde4cbbf07718a6cdce631a7e0ffac8b5156005a284856c884c6c28f1f66bc7d8b3e24fca4922093d4ff

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,117 @@
+# Embulk::Input::Http
+
+Input HTTP plugin for [Embulk](https://github.com/embulk/embulk).
+Read content via HTTP and parse/iterate json(or xml) data.
+
+## Installation
+
+Run this command with your embulk binary.
+
+```ruby
+$ embulk gem install embulk-input-http
+```
+
+## Usage
+
+Specify in your config.yml file
+
+```yaml
+in:
+  type: http
+  url: http://express.heartrails.com/api/json
+  params:
+    - {name: method, value: getStations}
+    - {name: x, value: 135.0}
+    - {name: y, value: 35.0}
+  schema:
+    - {name: name, type: string}
+    - {name: next, type: string}
+    - {name: prev, type: string}
+    - {name: distance, type: string}
+    - {name: x, type: double}
+    - {name: y, type: double}
+    - {name: line, type: string}
+    - {name: postal, type: string}
+  iterate: {type: json, path: $.response.station}
+  method: get
+```
+
+- type: specify this plugin as `http`
+- url: base url something like api (required)
+- schema: specify the attribute of table and data type (required)
+- iterate: data type and path to find root data, json/xml is supported for now (required)
+- method: http method, get is used by default (optional)
+- params: pair of name/value to specify query parameter (optional)
+
+
+### Iterate data
+
+You can specify 2 types to parse result from HTTP api in *iterate* section.
+
+
+#### json
+
+For this type, you need to specify *path* as  [jsonpath](http://goessner.net/articles/JsonPath/).
+
+for example:
+
+```json
+{
+    "result" : "success",
+    "students" : [
+      { "name" : "John", "age" : 10 },
+      { "name" : "Paul", "age" : 16 },
+      { "name" : "George", "age" : 17 },
+      { "name" : "Ringo", "age" : 18 }
+    ]
+}
+```
+
+You can iterate "students" node by the following condifuration:
+
+    iterate: {type: json, path: $.students}
+
+#### xml
+
+You can parse also xml by specifing **path/to/node** style to *path*.
+
+for example:
+
+
+```xml
+<data>
+  <result>true</result>
+  <students>
+    <student>
+      <name>John</name>
+      <age>10</name>
+    <student>
+    <student>
+      <name>Paul</name>
+      <age>16</name>
+    <student>
+    <student>
+      <name>George</name>
+      <age>17</name>
+    <student>
+    <student>
+      <name>Ringo</name>
+      <age>18</name>
+    <student>
+  </students>
+```
+
+Configuration as below to iterate student node:
+
+    iterate: {type: xml, path: data/students/student}
+
+
+## TODO
+
+- BasicAuth
+- HTTP-proxy
+- Breace-expansion style parameter, such as curl
+
+## Patch
+
+Welcome!

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/embulk-input-http.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-http"
+  spec.version       = "0.0.2"
+  spec.authors       = ["Takuma kanari"]
+  spec.email         = ["chemtrails.t@gmail.com"]
+  spec.summary       = %q{Embulk plugin for http input}
+  spec.description   = %q{fetch data via http}
+  spec.homepage      = "https://github.com/takumakanari/embulk-input-http"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "jsonpath", "~> 0.5"
+  spec.add_development_dependency "bundler", "~> 1.0"
+  spec.add_development_dependency "rake", "~> 0.9.2"
+end

data/example/json-example.yml

@@ -0,0 +1,21 @@
+exec: {}
+in:
+  type: http
+  url: http://express.heartrails.com/api/json
+  params:
+    - {name: method, value: getStations}
+    - {name: x, value: 135.0}
+    - {name: y, value: 35.0}
+  schema:
+    - {name: name, type: string}
+    - {name: next, type: string}
+    - {name: prev, type: string}
+    - {name: distance, type: string}
+    - {name: x, type: double}
+    - {name: y, type: double}
+    - {name: line, type: string}
+    - {name: postal, type: string}
+  method: get
+  iterate: {type: json, path: $.response.station}
+out: {type: stdout}
+

data/lib/embulk/input/http.rb

@@ -0,0 +1,164 @@
+require "net/http"
+require "uri"
+
+module Embulk
+  module Input
+
+    class HttpInputPlugin < InputPlugin
+      Plugin.register_input("http", self)
+
+      def self.transaction(config, &control)
+        url = config.param("url", :string)
+        schema = config.param("schema", :array)
+        method = config.param("method", :string, default: "get")
+        params = config.param("params", :array, default: [])
+        iterate = config.param("iterate", :hash)
+        open_timeout = config.param("open_timeout", :float, default: 2.0)
+        read_timeout = config.param("read_timeout", :float, default: 10.0)
+
+        data_type = iterate["type"]
+        unless ["json", "xml"].include?(data_type)
+          raise "Unknown data_type #{data_type}, only supported for json or xml"
+        end
+
+        columns = schema.each_with_index.map do |c, i|
+          Column.new(i, c["name"], c["type"].to_sym)
+        end
+
+        task = {
+          :url => url,
+          :method => method,
+          :params => params,
+          :schema => schema,
+          :iterate => iterate,
+          :open_timeout => open_timeout,
+          :read_timeout => read_timeout
+        }
+
+        report = yield(task, columns, 1)
+        config.merge(report["done"].flatten.compact)
+        {}
+      end
+
+      def run
+        schema = @task["schema"]
+        iterate = @task["iterate"]
+
+        data = fetch.body
+        data_type = iterate["type"]
+
+        case data_type
+        when "json"
+          iter = IterJson.new(data, iterate["path"])
+        when "xml"
+          iter = IterXML.new(data, iterate["path"])
+        else
+          raise "Unsupported data_type #{data_type}"
+        end
+
+        rows = 0
+        iter.each do |e|
+          rows += 1
+          @page_builder.add(schema.map{|c|
+            name = c["name"]
+            type = c["type"]
+            val = e[name].nil? ? "" : e[name]
+            case type
+            when "string"
+              val
+            when "long"
+              val.to_i
+            when "double"
+              val.to_f
+            when "boolean"
+              ["yes", "true", "1"].include?(val)
+            when "timestamp"
+              (val.nil? || val.empty?) ? nil : Time.strptime(val, c["format"])
+            else
+              raise "Unsupported type #{type}"
+            end
+          })
+        end
+        @page_builder.finish
+
+        {:rows => rows}
+      end
+
+      private
+
+      def fetch
+        uri = URI.parse(@task["url"])
+        method = @task["method"]
+        qs = URI.encode_www_form(@task["params"].map {|p|
+            [p["name"], p["value"]]
+        })
+        puts "#{method.upcase} #{uri}?#{qs}"
+
+        res = Net::HTTP.start(uri.host, uri.port) do |client|
+          client.open_timeout = @task["open_timeout"]
+          client.read_timeout = @task["read_timeout"]
+          case method.downcase
+          when "get"
+            client.get([uri.path, qs].join("?"))
+          when "post"
+            client.post(uri.path, qs)
+          else
+            raise "Unsupported method #{method}"
+          end
+        end
+
+        case res
+        when Net::HTTPSuccess
+          res
+        else
+          raise "Request is not successful, code=#{res.code}, value=#{res.body}"
+        end
+      end
+
+      class Iter
+        def initialize(data, path)
+          @data = data
+          @path = path
+        end
+
+        def each
+          raise NotImplementedError("each")
+        end
+      end
+
+      class IterXML < Iter
+        def initialize(data, path)
+          require "rexml/document"
+          super
+          @doc = REXML::Document.new(@data)
+        end
+
+        def each
+          @doc.elements.each(@path) do |e|
+            ret = {}
+            e.elements.each do |d|
+              ret[d.name] = d.text
+            end
+            yield ret
+          end
+        end
+      end
+
+      class IterJson < Iter
+        def initialize(data, path)
+          require "jsonpath"
+          super
+          @jsonpath = JsonPath.new(@path)
+        end
+
+        def each
+          @jsonpath.on(@data).flatten.each do |e|
+            raise "data is must be hash, but #{e.class}" unless e.instance_of?(Hash)
+            yield e
+          end
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-http
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Takuma kanari
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: fetch data via http
+email:
+- chemtrails.t@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-http.gemspec
+- example/json-example.yml
+- lib/embulk/input/http.rb
+homepage: https://github.com/takumakanari/embulk-input-http
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Embulk plugin for http input
+test_files: []