embulk-input-spotx 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b6cc25940e7346519766d95f9ce9d30dfa3e9c08
+  data.tar.gz: 519189017d5aab5267ff4367f6f967930dadf6b2
+SHA512:
+  metadata.gz: 8eebb03ec9f0f91222d7fbc98495b112f15acb1f43baf68d78f34a057fb9effbf09fa22bc3d85a4ec947d26a1d669a7a8e176c89b14eeb89301bcc80ffced317
+  data.tar.gz: 280aa257e4ccb08e49954944de3d939f997fe4f0c881a43010b485445f0b029581cb82d3170498219064514a7cb86a248decae5572ac811f546899d0f2be5895

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.1.13.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Spotx API data input plugin for Embulk
+
+Loads records from Spotx API
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: no
+* **Cleanup supported**: no
+* **Guess supported**: no
+
+## Configuration
+
+- **endpoint**: Spotx API endpoint to get data from (string, required)
+- **client_id**: Spotx API client ID (string, required)
+- **client_secret**: Spotx API client secret (string, required)
+- **refresh_token**: Spotx API refresh token (string, required)
+
+## Example
+
+```yaml
+in:
+  type: spotx
+  endpoint: https://api.spotxchange.com/1.0/Publisher($ID)/Channels/AdvertiserReport?date_range=last_7_days
+  client_id: $client_id
+  client_secret: $client_secret
+  refresh_token: $refresh_token
+  parser:
+    type: json
+```
+
+
+## Build
+
+```
+$ rake build
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-input-spotx.gemspec

@@ -0,0 +1,22 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-spotx"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Ming Liu"]
+  spec.summary       = "Spotx API data input plugin for Embulk"
+  spec.description   = "Loads records from Spotx API."
+  spec.email         = ["liuming@lmws.net"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/liuming/embulk-input-spotx"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'http', '~> 3.0.0'
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'embulk', ['>= 0.8.39']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/input/spotx.rb

@@ -0,0 +1,91 @@
+require 'http'
+
+module Embulk
+  module Input
+
+    class Spotx < InputPlugin
+      Plugin.register_input("spotx", self)
+
+      def self.transaction(config, &control)
+        task = {
+          "endpoint" => config.param("endpoint", :string),
+          "client_id" => config.param("client_id", :string),
+          "client_secret" => config.param("client_secret", :string),
+          "refresh_token" => config.param("refresh_token", :string),
+          "headers" => config.param("headers", :hash, default: {}),
+        }
+
+        columns = [
+          Column.new(0, "record", :json),
+        ]
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        _task_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      # TODO
+      # def self.guess(config)
+      #   sample_records = [
+      #     {"example"=>"a", "column"=>1, "value"=>0.1},
+      #     {"example"=>"a", "column"=>2, "value"=>0.2},
+      #   ]
+      #   columns = Guess::SchemaGuess.from_hash_records(sample_records)
+      #   return {"columns" => columns}
+      # end
+
+      def init
+        @endpoint = task["endpoint"]
+        @client_id = task["client_id"]
+        @client_secret = task["client_secret"]
+        @refresh_token = task["refresh_token"]
+        @headers = task["headers"]
+      end
+
+      def refresh_access_token
+        uri = 'https://publisher-api.spotxchange.com/1.0/token'
+        request_body = {
+          client_id: @client_id,
+          client_secret: @client_secret,
+          refresh_token: @refresh_token,
+          grant_type: 'refresh_token',
+        }
+        response = ::HTTP.post(uri, form: request_body)
+        response_body = JSON.parse(response.body.to_s)
+        response_body['value']['data']['access_token']
+      end
+
+      def request_data
+        auth = {"Authorization" => "Bearer #{refresh_access_token}"}
+        response = ::HTTP.get(@endpoint, headers: @headers.merge(auth))
+        body = ''
+        response.body.each do |chunk|
+          chunk.gsub!('{"value":{"data":[', '')
+          chunk.gsub!(']}}', '')
+          chunk.gsub!('},{', "}\n{")
+          body += chunk
+        end
+        return body
+      end
+
+      def run
+        content = request_data
+        content.split("\n").map do |line|
+          row = JSON.parse(line)
+          page_builder.add([row])
+          row
+        end
+        page_builder.finish
+
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-spotx
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ming Liu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: embulk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.39
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.39
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Loads records from Spotx API.
+email:
+- liuming@lmws.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-spotx.gemspec
+- lib/embulk/input/spotx.rb
+homepage: https://github.com/liuming/embulk-input-spotx
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Spotx API data input plugin for Embulk
+test_files: []