embulk-output-bigobject 0.2.3 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af597b0c7c4f99f4382a8b0e5c573dbaebd82d87
-  data.tar.gz: 47aef4b338f3f9ffaa081ed8d71f909e1b16e10b
+  metadata.gz: 03d0eda74a099f6b37f90a96d23cfa4eea9da2b4
+  data.tar.gz: c54cab398511de5290c0a36cf46709cfe76967ee
 SHA512:
-  metadata.gz: ebfb60eecf467b032f3adf08f8f4f85809be7899b5862817e758ef1cd99c6473df4d3299d9c9700a765cb2243c38f406f95e56b845231715d58f472d6ca5f29c
-  data.tar.gz: 7cb964fb2baa2c3ef0221beff2ca634e5f7107fbbeb3b187dd828c6eea5fad51cc3203a27bc4de4d574501fb0171616df4f55f5793bd080c3255694ebb82a344
+  metadata.gz: 29126af1038a32a65246b50679fe005d2c1dde141d5de96b5a7bffad1a12b3e36692fc38849e5499b8f0bbeec44f9c53e234b2d680ca104dad0d42bf55153d0e
+  data.tar.gz: eaf4267a3554f840ca863ba3fc4c9aae13ea573cc727b02e096b6fe875ce58c9207f8d72894b7d6d8701083cc7ff532be00af214f41fb36bf0b93760e4d492d5

data/embulk-output-bigobject.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigobject"
-  spec.version       = "0.2.3"
+  spec.version       = "0.2.8"
   spec.authors       = ["randyviola"]
   spec.summary       = "Bigobject output plugin for Embulk"
   spec.description   = "Dumps records to Bigobject."
@@ -13,7 +13,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_dependency 'rest-client', ['~> 1.8.0']
+  spec.add_development_dependency 'rest-client', ['~> 1.8.0']
   spec.add_development_dependency 'embulk', ['~> 0.7.10']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/output/bigobject.rb

@@ -2,8 +2,8 @@ module Embulk
   module Output
 
     class Bigobject < OutputPlugin
-      require 'logger'
-      require 'net/http'
+
+      require 'rest-client'
       require 'json'
 
       Plugin.register_output("bigobject", self)
@@ -15,12 +15,28 @@ module Embulk
           "port" => config.param("port", :integer, default: 9090),  # integer, optional
           "table" => config.param("table", :string),        # string, required
         }
-        @@log = Logger.new(STDOUT)
-        @@tblexist = false
+        
+        @@bo_url = "http://#{task['host']}:#{task['port']}/cmd".freeze
         @@ttlcounter = 0
+        
         # resumable output:
         # resume(task, schema, count, &control)
 
+        # Create Table if not exists
+        response = send_stmt2borest("desc #{task['table']}") # check table
+        if response["Status"] == 0 then # table exist
+              Embulk.logger.debug { "#{response}" }
+        elsif response["Status"] == -11 then # table not exist
+          response = send_stmt2borest("#{create_botable_stmt("#{task['table']}",schema)}")
+            if response["Status"] != 0 then 
+              Embulk.logger.error { "#{response}" }
+              raise "Create table #{task['table']} in BigObject Failed"
+            end
+        else # should not be here
+          Embulk.logger.error { "#{response}" }
+          raise "Please check table #{task['table']} in BigObject First"  
+        end
+
         # non-resumable output:
         task_reports = yield(task)
         next_config_diff = {}
@@ -36,24 +52,11 @@ module Embulk
 
       def init
         # initialization code:
-
-        @host = task["host"]
-        @port = task["port"]
         @table = task["table"]
-
-        @uri = URI.parse("http://#@host:#@port/cmd")
-        @http = Net::HTTP.new(@uri.host, @uri.port)
-        @counter = 0
-        
-        if !istbexist(@table)
-          if !createtbl(@table)
-            abort
-          end
-        end
+        @counter = 0 
       end
 
       def close
-        @http.finish
       end
 
       def add(page)
@@ -67,14 +70,16 @@ module Embulk
           records.each do |row| values << "#{row}".to_json end
           data.push("(#{values.join(",")})")
         end
+
         @counter += data.length
         @@ttlcounter += data.length
-
-        rsp = send_stmt("INSERT INTO #@table VALUES #{data.join(",")}")
+        
+        Embulk.logger.trace { "INSERT INTO #@table VALUES #{data.join(",")}" }
+        rsp = self.class.send_stmt2borest("INSERT INTO #@table VALUES #{data.join(",")}")
         if rsp["Status"] == 0 then
-          @@log.info "add #{data.length} to BigObject - #@counter | #@@ttlcounter"
+           Embulk.logger.debug { "add #{data.length} to BigObject - #@counter | #@@ttlcounter" }
         else
-          @@log.error "add #{data.length} to BigObject failed!!"
+           Embulk.logger.error { "add #{data.length} to BigObject failed!!" }
           abort
         end
 
@@ -88,64 +93,45 @@ module Embulk
       end
 
       def commit
-        task_report = {}
+        task_report = {
+          "records" => @counter
+        }
         return task_report
       end
 
-private
-        def send_stmt(stmt)
-          begin
-            params = {"stmt" => "#{stmt}"}
-            json_headers = {"Content-Type" => "application/json", "Accept" => "application/json"}
-            response = @http.post(@uri.path, params.to_json, json_headers)
-            return JSON.parse(response.body)
-          rescue Exception => e
-            @@log.error e
-            abort
-          end
-          
+      def self.send_stmt2borest(stmt)
+        begin
+          response = RestClient.post @@bo_url, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json
+          JSON.parse(response.body)
+        rescue Exception => e
+          Embulk.logger.error { e }
         end
-  
-        def istbexist(tbl)
-          if !@@tblexist then
-            stmt = "desc #{tbl}"
-            rsp = send_stmt(stmt)
-            rsp["Status"] == 0 ? @@tblexist = true : false
+      end
+
+      def self.create_botable_stmt(tbl,schema)
+        bo_table_schema = schema.map {|column| "#{column.name} #{to_bigobject_column_type(column.type.to_s, column.format.to_s)}" }.join(',')
+        "CREATE TABLE #{tbl} (#{bo_table_schema})"
+      end
+
+      def self.to_bigobject_column_type(type, format)
+        case type
+        when 'long'
+          botype = :INT64
+        when 'boolean'
+          botype = :INT8
+        when 'string'
+          botype = :STRING
+        when 'double'
+          botype = :DOUBLE
+        when 'timestamp'
+          if format.include? "%H" 
+            botype = :DATETIME32
+          else
+            botype = :DATE32
           end
-          return @@tblexist
-        end
-  
-        def createtbl(tbl)
-          rsp = send_stmt("CREATE TABLE #{tbl}  (#{change2botype(schema).join(",")})")
-          rsp["Status"] == 0 ? @@tblexist = true : false
-          return @@tblexist
         end
-  
-        def change2botype(schema)
-          colntype = Array.new
-          schema.each do |s|
-            botype = ""
-            case s[2]
-              when /long/
-                botype = :INT64
-              when /boolean/
-                botype = :INT8
-              when /string/
-                botype = :STRING
-              when /double/
-                botype = :DOUBLE
-              when /timestamp/
-                if s[3].include? "%H" 
-                  botype = :DATETIME32
-                else
-                  botype = :DATE32
-                end
-            end
-            
-            colntype << "'#{s[1]}' #{botype}"
-          end
-          return colntype
-        end 
+        botype
+      end
 
     end
 

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigobject
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.8
 platform: ruby
 authors:
 - randyviola
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-05 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: embulk
   requirement: !ruby/object:Gem::Requirement