embulk-input-http 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8ea58d778613a1a60214badcea8ea098f9dab77
-  data.tar.gz: c5eb80607d79558874a009a2dc47f12696053c7f
+  metadata.gz: 0bc5ae1c890470e5d30b21ced647bd9e85e8a272
+  data.tar.gz: 77c89043f7c45fae6750eb84152053f10ae691a1
 SHA512:
-  metadata.gz: 98ab84c0d458317898947ded60b4edfd6b4c132b1aaea930a485d76b14e408b742863fac125172bb46d86d54e9e6b6ea6c3aa6229832054b0dbd3233361ffe90
-  data.tar.gz: 984d9997c12a42438546d8b4acf28053fd2d3c6b5620fde4cbbf07718a6cdce631a7e0ffac8b5156005a284856c884c6c28f1f66bc7d8b3e24fca4922093d4ff
+  metadata.gz: be0f73d6d3e4f33d7dcba51208e32446b5c31c4b4415bc6b415285234ec2982ffbc131a565914605b8d8331a371fd0daf143f4cf23f24a01d604be8dce1999fc
+  data.tar.gz: 44c57e268a4a9e4d0ecc7c8e06f836e62b7f69f2c760306665aba9e714eb7e522b025daabf9c23b34029ba9b5820094bd32c0170203ae78daaa3bb8f4eec57e0

data/README.md

@@ -22,14 +22,14 @@ in:
   params:
     - {name: method, value: getStations}
     - {name: x, value: 135.0}
-    - {name: y, value: 35.0}
+    - {name: y, value: "{30..35}.0", expand: true}
   schema:
     - {name: name, type: string}
     - {name: next, type: string}
     - {name: prev, type: string}
     - {name: distance, type: string}
-    - {name: x, type: double}
-    - {name: y, type: double}
+    - {name: lat, type: double, path: x}
+    - {name: lng, type: double, path: y}
     - {name: line, type: string}
     - {name: postal, type: string}
   iterate: {type: json, path: $.response.station}
@@ -42,7 +42,8 @@ in:
 - iterate: data type and path to find root data, json/xml is supported for now (required)
 - method: http method, get is used by default (optional)
 - params: pair of name/value to specify query parameter (optional)
-
+- open_timeout: timeout to open connection (optional, 5 is used by default)
+- read_timeout: timeout to read content via http (optional, 10 is used by default)
 
 ### Iterate data
 
@@ -71,6 +72,30 @@ You can iterate "students" node by the following condifuration:
 
     iterate: {type: json, path: $.students}
 
+You can specify jsonpath to also *path* in schema section:
+
+```yaml
+schema:
+  - {name: firstName, type: string, path: "names[0]"}
+  - {name: lastName, type: string, path: "names[1]"}
+iterate: {type: json, path: $.students}
+```
+
+Then you can make record from more complicated json like as follows:
+
+```json
+{
+    "result" : "success",
+    "students" : [
+      { "names" : ["John", "Lennon"], "age" : 10 },
+      { "names" : ["Paul", "Maccartney"], "age" : 10 }
+    ]
+}
+```
+
+In this case, names[0] will be firstName of schema and names[1] will be lastName.
+
+
 #### xml
 
 You can parse also xml by specifing **path/to/node** style to *path*.
@@ -105,12 +130,23 @@ Configuration as below to iterate student node:
 
     iterate: {type: xml, path: data/students/student}
 
+### Brace expansion style in params
 
-## TODO
+In *params* section, you can specify also multilple params by using **brace expansion style**.
 
+```yaml
+params
+  - {name: id, value "{1..5}", expand: true}
+  - {name: name, value "{John,Paul,George,Ringo}", expand: true}
+```
+
+To use this style, you need to set true to parameter *expand*, then all patterns of query will be called in a defferent request.
+
+
+## TODO
+- Split input/formatter
 - BasicAuth
 - HTTP-proxy
-- Breace-expansion style parameter, such as curl
 
 ## Patch
 

data/embulk-input-http.gemspec

@@ -4,11 +4,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-http"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["Takuma kanari"]
   spec.email         = ["chemtrails.t@gmail.com"]
   spec.summary       = %q{Embulk plugin for http input}
-  spec.description   = %q{fetch data via http}
+  spec.description   = %q{Fetch data via http}
   spec.homepage      = "https://github.com/takumakanari/embulk-input-http"
   spec.license       = "MIT"
 
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "jsonpath", "~> 0.5"
+  spec.add_dependency "bracecomp", "~> 0.1", ">= 0.1.2"
   spec.add_development_dependency "bundler", "~> 1.0"
   spec.add_development_dependency "rake", "~> 0.9.2"
 end

data/example/json-example-expand.yml

@@ -0,0 +1,21 @@
+exec: {}
+in:
+  type: http
+  url: http://express.heartrails.com/api/json
+  params:
+    - {name: method, value: getStations}
+    - {name: x, value: "{130..135}.0", expand: true}
+    - {name: y, value: "{30..35}.0", expand: true}
+  schema:
+    - {name: name, type: string}
+    - {name: next, type: string}
+    - {name: prev, type: string}
+    - {name: distance, type: string}
+    - {name: lat, type: double, path: x}
+    - {name: lng, type: double, path: y}
+    - {name: line, type: string}
+    - {name: postal, type: string}
+  method: get
+  iterate: {type: json, path: $.response.station}
+out: {type: stdout}
+

data/example/json-example.yml

@@ -11,8 +11,8 @@ in:
     - {name: next, type: string}
     - {name: prev, type: string}
     - {name: distance, type: string}
-    - {name: x, type: double}
-    - {name: y, type: double}
+    - {name: lat, type: double, path: x}
+    - {name: lng, type: double, path: y}
     - {name: line, type: string}
     - {name: postal, type: string}
   method: get

data/lib/embulk/input/http.rb

@@ -1,5 +1,6 @@
 require "net/http"
 require "uri"
+require "bracecomp"
 
 module Embulk
   module Input
@@ -8,155 +9,186 @@ module Embulk
       Plugin.register_input("http", self)
 
       def self.transaction(config, &control)
-        url = config.param("url", :string)
-        schema = config.param("schema", :array)
-        method = config.param("method", :string, default: "get")
+        task = {
+          :url => config.param("url", :string),
+          :method => config.param("method", :string, default: "get"),
+          :schema => config.param("schema", :array),
+          :iterate => config.param("iterate", :hash),
+          :open_timeout => config.param("open_timeout", :float, default: 2.0),
+          :read_timeout => config.param("read_timeout", :float, default: 10.0),
+          :done => config.param("done", :array, default: [])
+        }
         params = config.param("params", :array, default: [])
-        iterate = config.param("iterate", :hash)
-        open_timeout = config.param("open_timeout", :float, default: 2.0)
-        read_timeout = config.param("read_timeout", :float, default: 10.0)
+        params_unexpand, params_expand = configure_queries(params)
 
-        data_type = iterate["type"]
+        data_type = task[:iterate]["type"]
         unless ["json", "xml"].include?(data_type)
           raise "Unknown data_type #{data_type}, only supported for json or xml"
         end
 
-        columns = schema.each_with_index.map do |c, i|
+        columns = task[:schema].each_with_index.map do |c, i|
           Column.new(i, c["name"], c["type"].to_sym)
         end
 
-        task = {
-          :url => url,
-          :method => method,
-          :params => params,
-          :schema => schema,
-          :iterate => iterate,
-          :open_timeout => open_timeout,
-          :read_timeout => read_timeout
-        }
+        task[:params] = params_unexpand
+        task[:params_expand] = params_expand - task[:done]
+        num_of_threads = task[:params_expand].empty? ? 1 : task[:params_expand].size
 
-        report = yield(task, columns, 1)
-        config.merge(report["done"].flatten.compact)
-        {}
+        report = yield(task, columns, num_of_threads)
+        {"done" => report.map{|r| r["done"]}.compact}
+      end
+
+      def self.configure_queries(params)
+        base = params.select{|p| !p["expand"]}.map do |p|
+          [p["name"], p["value"]]
+        end
+        expands = params.select{|p| p["expand"] }.map do |p|
+          p["value"].expand.map do |v|
+            [p["name"], v]
+          end
+        end
+        if expands.size > 0
+            dest = expands.first.product(*(expands.slice(1, expands.size - 1)))
+            dest.sort!{|a, b| "#{a[0]}=#{a[1]}" <=> "#{b[0]}=#{b[1]}"}
+        else
+            dest = []
+        end
+        [base, dest]
       end
 
       def run
         schema = @task["schema"]
         iterate = @task["iterate"]
+        url = @task["url"]
+        method = @task["method"]
 
-        data = fetch.body
+        params_expand = @task["params_expand"][@index] || []
+        query = URI.encode_www_form(@task["params"] + params_expand)
+        puts "#{@index}: #{method.upcase} #{url}?#{query}"
+
+        data = fetch(url, method, query).body
         data_type = iterate["type"]
 
         case data_type
-        when "json"
-          iter = IterJson.new(data, iterate["path"])
-        when "xml"
-          iter = IterXML.new(data, iterate["path"])
-        else
-          raise "Unsupported data_type #{data_type}"
+          when "json"
+            iter = IterJson.new(schema, data, iterate)
+          when "xml"
+            iter = IterXML.new(schema, data, iterate)
+          else
+            raise "Unsupported data_type #{data_type}"
         end
 
-        rows = 0
-        iter.each do |e|
-          rows += 1
-          @page_builder.add(schema.map{|c|
-            name = c["name"]
-            type = c["type"]
-            val = e[name].nil? ? "" : e[name]
-            case type
-            when "string"
-              val
-            when "long"
-              val.to_i
-            when "double"
-              val.to_f
-            when "boolean"
-              ["yes", "true", "1"].include?(val)
-            when "timestamp"
-              (val.nil? || val.empty?) ? nil : Time.strptime(val, c["format"])
-            else
-              raise "Unsupported type #{type}"
-            end
-          })
+        iter.each do |record|
+          @page_builder.add(record)
         end
         @page_builder.finish
 
-        {:rows => rows}
+        {:done => params_expand}
       end
 
       private
 
-      def fetch
-        uri = URI.parse(@task["url"])
-        method = @task["method"]
-        qs = URI.encode_www_form(@task["params"].map {|p|
-            [p["name"], p["value"]]
-        })
-        puts "#{method.upcase} #{uri}?#{qs}"
+      def fetch(url, method, query)
+        uri = URI.parse(url)
 
         res = Net::HTTP.start(uri.host, uri.port) do |client|
           client.open_timeout = @task["open_timeout"]
           client.read_timeout = @task["read_timeout"]
           case method.downcase
-          when "get"
-            client.get([uri.path, qs].join("?"))
-          when "post"
-            client.post(uri.path, qs)
-          else
-            raise "Unsupported method #{method}"
+            when "get"
+              client.get([uri.path, query].join("?"))
+            when "post"
+              client.post(uri.path, query)
+            else
+              raise "Unsupported method #{method}"
           end
         end
 
         case res
-        when Net::HTTPSuccess
-          res
-        else
-          raise "Request is not successful, code=#{res.code}, value=#{res.body}"
+          when Net::HTTPSuccess
+            res
+          else
+            raise "Request is not successful, code=#{res.code}, value=#{res.body}"
         end
       end
 
       class Iter
-        def initialize(data, path)
+        def initialize(schema, data, config)
+          @schema = schema
           @data = data
-          @path = path
+          @config = config
         end
 
         def each
-          raise NotImplementedError("each")
+          raise NotImplementedError.new("each")
+        end
+
+        private
+
+        def make_record(e)
+          @schema.map do |c|
+            name = c["name"]
+            path = c["path"]
+            val = path.nil? ? e[name] : find_by_path(e, path)
+
+            v = val.nil? ? "" : val
+            type = c["type"]
+            case type
+              when "string"
+                v
+              when "long"
+                v.to_i
+              when "double"
+                v.to_f
+              when "boolean"
+                ["yes", "true", "1"].include?(v)
+              when "timestamp"
+                v.empty? ? nil : Time.strptime(v, c["format"])
+              else
+                raise "Unsupported type #{type}"
+            end
+          end
+        end
+
+        def find_by_path(e, path)
+          raise NotImplementedError.new("Find by path is unsupported")
         end
       end
 
       class IterXML < Iter
-        def initialize(data, path)
+        def initialize(schema, data, config)
           require "rexml/document"
           super
           @doc = REXML::Document.new(@data)
         end
 
         def each
-          @doc.elements.each(@path) do |e|
-            ret = {}
+          @doc.elements.each(@config["path"]) do |e|
+            dest = {}
             e.elements.each do |d|
-              ret[d.name] = d.text
+              dest[d.name] = d.text
             end
-            yield ret
+            yield make_record(dest)
           end
         end
       end
 
       class IterJson < Iter
-        def initialize(data, path)
+        def initialize(schema, data, config)
           require "jsonpath"
           super
-          @jsonpath = JsonPath.new(@path)
+          @jsonpath = JsonPath.new(@config["path"])
         end
 
         def each
           @jsonpath.on(@data).flatten.each do |e|
-            raise "data is must be hash, but #{e.class}" unless e.instance_of?(Hash)
-            yield e
+            yield make_record(e)
           end
         end
+
+        def find_by_path(e, path)
+          JsonPath.on(e, path).first
+        end
       end
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-http
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Takuma kanari
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-08 00:00:00.000000000 Z
+date: 2015-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jsonpath
@@ -24,6 +24,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: bracecomp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +72,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.2
-description: fetch data via http
+description: Fetch data via http
 email:
 - chemtrails.t@gmail.com
 executables: []
@@ -65,6 +85,7 @@ files:
 - README.md
 - Rakefile
 - embulk-input-http.gemspec
+- example/json-example-expand.yml
 - example/json-example.yml
 - lib/embulk/input/http.rb
 homepage: https://github.com/takumakanari/embulk-input-http