embulk-output-bigobject 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d77b13380f7135a1f30aead13e12a33c16dddb2
-  data.tar.gz: df4f041e3abcccb388a81a8a2829f6e0fba8a71c
+  metadata.gz: 229379a513faaf6478955285b0640d23858f0ea0
+  data.tar.gz: 38a19d01b384b5dbd4018d878f89b5cd6fd4bde1
 SHA512:
-  metadata.gz: e11cbaee25140bb287efa8e3a2e75231774f1d32613c45365490ff907466a6d3acef8e2a84942ab29989354a6abcfe2ac7165b86b07a709423ff50c2d11ebfdf
-  data.tar.gz: 9ba0d2a421df4c802b93a1d8ef437e66174bb615f9586a70e87fe6e757526389ac1193e0acca42a6cc1f3f4f83fe3bed0660d8fc19042611560aeb1d6767eb2a
+  metadata.gz: 557932358583d6e91d3da7bba271fd35bd98a6a96f43ce47fc77eec3fc87192da65cd71baa09e2d6d080635391784ce6390008bdd9b0379137ce0db87fff6230
+  data.tar.gz: 83718189456159337507f5ed47d1121d0d76ec0460f782b030358ac9d302ab189ddd7a970e412ea17becbed32cfebd82299d4fd9044c459b20178b898329a211

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# Bigobject output plugin for Embulk
+
+BigObject output plugins for Embulk loads records to BigObject.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, default: 9090)
+- **table**: database table name (string, required)
+
+## Example
+
+```yaml
+out:
+  type: bigobject
+  host: localhost
+  table: mytest
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-output-bigobject.gemspec

@@ -0,0 +1,20 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-bigobject"
+  spec.version       = "0.2.1"
+  spec.authors       = ["randyviola"]
+  spec.summary       = "Bigobject output plugin for Embulk"
+  spec.description   = "Dumps records to Bigobject."
+  spec.email         = ["randyh0329@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/randyh0329/embulk-output-bigobject"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'embulk', ['~> 0.7.10']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/output/bigobject.rb

@@ -0,0 +1,153 @@
+module Embulk
+  module Output
+
+    class Bigobject < OutputPlugin
+      require 'logger'
+      require 'net/http'
+      require 'json'
+
+      Plugin.register_output("bigobject", self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+          "host" => config.param("host", :string),                     # string, required
+          "port" => config.param("port", :integer, default: 9090),  # integer, optional
+          "table" => config.param("table", :string),        # string, required
+        }
+        @@log = Logger.new(STDOUT)
+        @@tblexist = false
+        @@ttlcounter = 0
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        # non-resumable output:
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def init
+        # initialization code:
+
+        @host = task["host"]
+        @port = task["port"]
+        @table = task["table"]
+
+        @uri = URI.parse("http://#@host:#@port/cmd")
+        @http = Net::HTTP.new(@uri.host, @uri.port)
+        @counter = 0
+        
+        if !istbexist(@table)
+          if !createtbl(@table)
+            abort
+          end
+        end
+      end
+
+      def close
+        @http.finish
+      end
+
+      def add(page)
+        # output code:
+        
+        data = Array.new
+        values = Array.new
+        
+        page.each do |records|
+          values = []
+          records.each do |row| values << row.to_json end
+          data.push("(#{values.join(",")})")
+        end
+        @counter += data.length
+        @@ttlcounter += data.length
+
+        rsp = send_stmt("INSERT INTO #@table VALUES #{data.join(",")}")
+        if rsp["Status"] == 0 then
+          @@log.info "add #{data.length} to BigObject - #@counter | #@@ttlcounter"
+        else
+          @@log.error "add #{data.length} to BigObject failed!!"
+          abort
+        end
+
+      end
+
+      def finish
+      end
+
+      def abort
+        raise "Please Check BigObject"
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+
+private
+        def send_stmt(stmt)
+          begin
+            params = {"stmt" => "#{stmt}"}
+            json_headers = {"Content-Type" => "application/json", "Accept" => "application/json"}
+            response = @http.post(@uri.path, params.to_json, json_headers)
+            return JSON.parse(response.body)
+          rescue Exception => e
+            @@log.error e
+            abort
+          end
+          
+        end
+  
+        def istbexist(tbl)
+          if !@@tblexist then
+            stmt = "desc #{tbl}"
+            rsp = send_stmt(stmt)
+            rsp["Status"] == 0 ? @@tblexist = true : false
+          end
+          return @@tblexist
+        end
+  
+        def createtbl(tbl)
+          rsp = send_stmt("CREATE TABLE #{tbl}  (#{change2botype(schema).join(",")})")
+          rsp["Status"] == 0 ? @@tblexist = true : false
+          return @@tblexist
+        end
+  
+        def change2botype(schema)
+          colntype = Array.new
+          schema.each do |s|
+            botype = ""
+            case s[2]
+              when /long/
+                botype = :INT64
+              when /boolean/
+                botype = :INT8
+              when /string/
+                botype = :STRING
+              when /double/
+                botype = :DOUBLE
+              when /timestamp/
+                if s[3].include? "%H" 
+                  botype = :DATETIME32
+                else
+                  botype = :DATE32
+                end
+            end
+            
+            colntype << "'#{s[1]}' #{botype}"
+          end
+          return colntype
+        end 
+
+    end
+
+  end
+end

data/lib/embulk/output/test.rb

@@ -0,0 +1,61 @@
+require 'net/http'
+require 'json'
+require 'logger'
+
+
+params = {"stmt" => "desc testcsv"}
+json_headers = {"Content-Type" => "application/json",
+                "Accept" => "application/json"}
+
+uri = URI.parse('http://192.168.99.100:9090/cmd')
+http = Net::HTTP.new(uri.host, uri.port)
+
+response = http.post(uri.path, params.to_json, json_headers)
+
+@rsp = JSON.parse(response.body)
+
+def checktbl
+ 	
+ return @rsp["Status"] == 0 ? "YES" : "NO"
+end
+
+puts checktbl
+
+# values = Array.new
+# values2 = Array.new
+# values2 << 1
+# values2 << "2"
+# values.push("(#{values2.join(",")})")
+# values.push("id1")
+# values.push(2)
+# values.push("id2")
+# str = "INSERT INTO #{values.join(",")}"
+# log = Logger.new(STDOUT)
+# puts str
+
+#   def init
+#     @http = Net::HTTP::Persistent.new 'embulk-plugin-bigobject'
+#     @http.headers['Content-Type'] = {"Content-Type" => "application/json", "Accept" => "application/json"}
+#     @uri = URI.parse('http://192.168.99.100:9090/cmd')
+#     puts "initialized"
+#   end
+
+#   def send
+#   	#uri = URI.parse('http://192.168.99.100:9090/cmd')
+#     post = Net::HTTP::Post.new @uri.request_uri
+#     post.body = {"stmt" => "show tables"}.to_json
+#     response = @http.request @uri, post
+#     puts JSON.parse(response.body)
+#   end
+
+#  def shutdown
+#     @http.shutdown
+#     puts "shutdown"
+#   end
+
+# init
+# send
+# shutdown
+
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigobject
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - randyviola
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-22 00:00:00.000000000 Z
+date: 2016-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: embulk
@@ -58,8 +58,15 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
-homepage: 
+files:
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-bigobject.gemspec
+- lib/embulk/output/bigobject.rb
+- lib/embulk/output/test.rb
+homepage: https://github.com/randyh0329/embulk-output-bigobject
 licenses:
 - MIT
 metadata: {}