embulk-output-bigobject 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03d0eda74a099f6b37f90a96d23cfa4eea9da2b4
-  data.tar.gz: c54cab398511de5290c0a36cf46709cfe76967ee
+  metadata.gz: 43abe5dc1730fe8a446ddeed77a4de07f514c89c
+  data.tar.gz: df2bee768096c9abdfe944e6137e934aa2b66e96
 SHA512:
-  metadata.gz: 29126af1038a32a65246b50679fe005d2c1dde141d5de96b5a7bffad1a12b3e36692fc38849e5499b8f0bbeec44f9c53e234b2d680ca104dad0d42bf55153d0e
-  data.tar.gz: eaf4267a3554f840ca863ba3fc4c9aae13ea573cc727b02e096b6fe875ce58c9207f8d72894b7d6d8701083cc7ff532be00af214f41fb36bf0b93760e4d492d5
+  metadata.gz: 4cfa4216e75df1bb68990700e529a3477d6a5f0c1eecdfbb91df1b70e9e4681c73525999019031269776b3c94679321ef0079952b879d27b2b0625d7dceefa5f
+  data.tar.gz: 8aa257ce70a52b46b3cf3f4cb6e2cda9d734678cf7636e03582a84ad918e2bbe08f17438c4504f89983669aab47d794b87b2808a47430fd0e59a288f721edef3

data/README.md

@@ -17,8 +17,9 @@ embulk gem install embulk-output-bigobject
 
 ## Configuration
 
-- **host**: database host name (string, required)
-- **port**: database port number (integer, default: 9090)
+- **host**: database host name (string, default: localhost)
+- **restport**: database port number (integer, default: 9090)
+- **ncport**: database port number (integer, default: 9090)
 - **table**: database table name (string, required)
 
 ## Example
@@ -28,6 +29,11 @@ out:
   type: bigobject
   host: localhost
   table: mytest
+  column_options:
+    - {name: "col1", type: 'INT64', is_key: true}
+    - {name: "col2", type: 'BYTE', is_key: true}
+    - {name: "col3", type: 'DATE32'}
+    - {name: "col4", type: 'STRING(16)'}
 ```
 
 

data/embulk-output-bigobject.gemspec

@@ -1,11 +1,11 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigobject"
-  spec.version       = "0.2.8"
-  spec.authors       = ["randyviola"]
+  spec.version       = "0.3.0"
+  spec.authors       = ["cchuang"]
   spec.summary       = "Bigobject output plugin for Embulk"
   spec.description   = "Dumps records to Bigobject."
-  spec.email         = ["randyh0329@gmail.com"]
+  spec.email         = ["cchuang.tw@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/bigobject-inc/embulk-output-bigobject"
 
@@ -13,9 +13,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'rest-client', ['~> 1.8.0']
-  spec.add_development_dependency 'rest-client', ['~> 1.8.0']
-  spec.add_development_dependency 'embulk', ['~> 0.7.10']
-  spec.add_development_dependency 'bundler', ['~> 1.0']
-  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_dependency 'rest-client', ['>= 1.8.0']
+  #spec.add_development_dependency 'rest-client', ['>= 1.8.0']
+  spec.add_development_dependency 'embulk', ['>= 0.8.9']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.5.0']
 end

data/lib/embulk/output/bigobject.rb

@@ -1,37 +1,55 @@
 module Embulk
+
   module Output
 
     class Bigobject < OutputPlugin
 
       require 'rest-client'
       require 'json'
+	  require 'socket'
 
       Plugin.register_output("bigobject", self)
 
-      def self.transaction(config, schema, count, &control)
+      def self.configure(config, schema, count)
         # configuration code:
         task = {
-          "host" => config.param("host", :string),                     # string, required
-          "port" => config.param("port", :integer, default: 9090),  # integer, optional
-          "table" => config.param("table", :string),        # string, required
+          "host"           => config.param("host",           :string,  :default => "localhost"), # string, optional
+          "restport"       => config.param("restport",       :integer, :default => 9090),  # integer, optional
+          "ncport"         => config.param("ncport",         :integer, :default => 9091),  # integer, optional
+          "table"          => config.param("table",          :string),        # string, required
+		  "column_options" => config.param("column_options", :array,   :default => []),   
         }
         
-        @@bo_url = "http://#{task['host']}:#{task['port']}/cmd".freeze
-        @@ttlcounter = 0
+		task
+	  end 
+
+	  def self.column_options_map(column_options) 
+		(column_options || {}).map do |column_option| 
+		  [column_option['name'], column_option] 
+		end.to_h 
+	  end
+
+      def self.transaction(config, schema, count, &control)
+		task = self.configure(config, schema, count)
+		task['co_map'] = self.column_options_map task["column_options"]
+        task['rest_uri'] = "http://#{task['host']}:#{task['restport']}/cmd".freeze
+		task['ttl_counter'] = 0
         
+        Embulk.logger.debug { "Transaction #{count}" }
         # resumable output:
         # resume(task, schema, count, &control)
 
-        # Create Table if not exists
-        response = send_stmt2borest("desc #{task['table']}") # check table
-        if response["Status"] == 0 then # table exist
-              Embulk.logger.debug { "#{response}" }
-        elsif response["Status"] == -11 then # table not exist
-          response = send_stmt2borest("#{create_botable_stmt("#{task['table']}",schema)}")
-            if response["Status"] != 0 then 
-              Embulk.logger.error { "#{response}" }
-              raise "Create table #{task['table']} in BigObject Failed"
-            end
+        # Create-Table if it does not exist
+        response = rest_exec(task['rest_uri'], "desc #{task['table']}") # check table
+        if response["Status"] == 0 then # the table exists
+          Embulk.logger.debug { "#{response}" }
+        elsif response["Status"] == -11 then # the table does not exist
+          response = rest_exec(task['rest_uri'], "#{create_botable_stmt("#{task['table']}",schema, task['co_map'])}")
+          if response["Status"] != 0 then 
+            Embulk.logger.error { "#{response}" }
+            raise "Create table #{task['table']} in BigObject Failed"
+          end
+          Embulk.logger.info { "embulk-output-bigobject: Create table #{task['table']}" }
         else # should not be here
           Embulk.logger.error { "#{response}" }
           raise "Please check table #{task['table']} in BigObject First"  
@@ -43,6 +61,22 @@ module Embulk
         return next_config_diff
       end
 
+	  def safe_io_puts(buff)
+		@@io ||= create_shared_io
+		@@mutext ||= Mutex.new
+        @@mutext.synchronize do 
+		  @@io.puts buff
+		end
+	  end
+
+	  def create_shared_io
+		io = TCPSocket.new @task['host'], @task['ncport']
+		#@@io = File.new "embulkout.dump", "w"
+		io.write "csv\x01"
+		io.puts @task['table']
+		io
+	  end
+
       #def self.resume(task, schema, count, &control)
       #  task_reports = yield(task)
       #
@@ -50,7 +84,10 @@ module Embulk
       #  return next_config_diff
       #end
 
-      def init
+      def initialize(task, schema, index)
+		super
+
+        # Embulk.logger.debug { "Initialize #{index}" }
         # initialization code:
         @table = task["table"]
         @counter = 0 
@@ -62,26 +99,20 @@ module Embulk
       def add(page)
         # output code:
         
-        data = Array.new
+        #data = Array.new
         values = Array.new
+		count = 0
         
         page.each do |records|
           values = []
           records.each do |row| values << "#{row}".to_json end
-          data.push("(#{values.join(",")})")
+          #data.push("(#{values.join(",")})")
+		  safe_io_puts "#{values.join(",")}"
+		  count += 1
         end
 
-        @counter += data.length
-        @@ttlcounter += data.length
-        
-        Embulk.logger.trace { "INSERT INTO #@table VALUES #{data.join(",")}" }
-        rsp = self.class.send_stmt2borest("INSERT INTO #@table VALUES #{data.join(",")}")
-        if rsp["Status"] == 0 then
-           Embulk.logger.debug { "add #{data.length} to BigObject - #@counter | #@@ttlcounter" }
-        else
-           Embulk.logger.error { "add #{data.length} to BigObject failed!!" }
-          abort
-        end
+        @counter += count
+        @task['ttl_counter'] += count
 
       end
 
@@ -99,21 +130,43 @@ module Embulk
         return task_report
       end
 
-      def self.send_stmt2borest(stmt)
+      def self.rest_exec(uri, stmt)
         begin
-          response = RestClient.post @@bo_url, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json
+          response = RestClient.post uri, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json, :timeout => 2
           JSON.parse(response.body)
-        rescue Exception => e
-          Embulk.logger.error { e }
+        rescue RestClient::Exception => e
+          #Embulk.logger.error { "RestClient: #{e.http_code}, #{e.message}, response: #{e.response}" }
+          Embulk.logger.warn { "Timeout: statement: #{stmt}" }
+          begin
+            response = RestClient.post uri, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json, :timeout => 4
+            JSON.parse(response.body)
+          rescue RestClient::Exception => e2
+            Embulk.logger.error { "RestClient: #{e2.http_code}, #{e2.message}, response: #{e2.response}" }
+          end
+	    rescue JSON::Exception => e
+          Embulk.logger.error { "JSON: #{e.message}" }
         end
       end
 
-      def self.create_botable_stmt(tbl,schema)
-        bo_table_schema = schema.map {|column| "#{column.name} #{to_bigobject_column_type(column.type.to_s, column.format.to_s)}" }.join(',')
-        "CREATE TABLE #{tbl} (#{bo_table_schema})"
+      def self.create_botable_stmt(tbl,schema, cos)
+		Embulk.logger.debug {"schema: #{schema}"}
+        bo_table_schema = schema.map {|column| "#{column.name} #{to_bigobject_column_type(column.type.to_s, column.format.to_s, cos[column.name])}" }.join(',')
+		keys = Array.new
+		cos.each do |k, co|
+		  keys.push co["name"] if co["is_key"] 
+		end
+		if keys.length == 0
+          "CREATE TABLE #{tbl} (#{bo_table_schema})"
+		else
+          "CREATE TABLE #{tbl} (#{bo_table_schema} KEY(#{keys.join(',')}))"
+		end
       end
 
-      def self.to_bigobject_column_type(type, format)
+      def self.to_bigobject_column_type(type, format, co)
+		co = co || {}
+		Embulk.logger.debug {"type: #{type}, format #{format}, option #{co}"}
+		return co["type"] if co["type"]
+
         case type
         when 'long'
           botype = :INT64

metadata

@@ -1,88 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigobject
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
-- randyviola
+- cchuang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2016-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.8.0
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.8.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: embulk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.8.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.8.9
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.10.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.10.6
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 10.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 10.5.0
 description: Dumps records to Bigobject.
 email:
-- randyh0329@gmail.com
+- cchuang.tw@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,17 +89,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Bigobject output plugin for Embulk