embulk-output-bigobject 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2d77b13380f7135a1f30aead13e12a33c16dddb2
4
- data.tar.gz: df4f041e3abcccb388a81a8a2829f6e0fba8a71c
3
+ metadata.gz: 229379a513faaf6478955285b0640d23858f0ea0
4
+ data.tar.gz: 38a19d01b384b5dbd4018d878f89b5cd6fd4bde1
5
5
  SHA512:
6
- metadata.gz: e11cbaee25140bb287efa8e3a2e75231774f1d32613c45365490ff907466a6d3acef8e2a84942ab29989354a6abcfe2ac7165b86b07a709423ff50c2d11ebfdf
7
- data.tar.gz: 9ba0d2a421df4c802b93a1d8ef437e66174bb615f9586a70e87fe6e757526389ac1193e0acca42a6cc1f3f4f83fe3bed0660d8fc19042611560aeb1d6767eb2a
6
+ metadata.gz: 557932358583d6e91d3da7bba271fd35bd98a6a96f43ce47fc77eec3fc87192da65cd71baa09e2d6d080635391784ce6390008bdd9b0379137ce0db87fff6230
7
+ data.tar.gz: 83718189456159337507f5ed47d1121d0d76ec0460f782b030358ac9d302ab189ddd7a970e412ea17becbed32cfebd82299d4fd9044c459b20178b898329a211
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org/'
2
+ gemspec
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,32 @@
1
+ # Bigobject output plugin for Embulk
2
+
3
+ BigObject output plugins for Embulk loads records to BigObject.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: output
8
+ * **Load all or nothing**: no
9
+ * **Resume supported**: no
10
+ * **Cleanup supported**: yes
11
+
12
+ ## Configuration
13
+
14
+ - **host**: database host name (string, required)
15
+ - **port**: database port number (integer, default: 9090)
16
+ - **table**: database table name (string, required)
17
+
18
+ ## Example
19
+
20
+ ```yaml
21
+ out:
22
+ type: bigobject
23
+ host: localhost
24
+ table: mytest
25
+ ```
26
+
27
+
28
+ ## Build
29
+
30
+ ```
31
+ $ rake
32
+ ```
@@ -0,0 +1,3 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task default: :build
@@ -0,0 +1,20 @@
1
+
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "embulk-output-bigobject"
4
+ spec.version = "0.2.1"
5
+ spec.authors = ["randyviola"]
6
+ spec.summary = "Bigobject output plugin for Embulk"
7
+ spec.description = "Dumps records to Bigobject."
8
+ spec.email = ["randyh0329@gmail.com"]
9
+ spec.licenses = ["MIT"]
10
+ spec.homepage = "https://github.com/randyh0329/embulk-output-bigobject"
11
+
12
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
+ spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
+ spec.require_paths = ["lib"]
15
+
16
+ #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
17
+ spec.add_development_dependency 'embulk', ['~> 0.7.10']
18
+ spec.add_development_dependency 'bundler', ['~> 1.0']
19
+ spec.add_development_dependency 'rake', ['>= 10.0']
20
+ end
@@ -0,0 +1,153 @@
1
+ module Embulk
2
+ module Output
3
+
4
+ class Bigobject < OutputPlugin
5
+ require 'logger'
6
+ require 'net/http'
7
+ require 'json'
8
+
9
+ Plugin.register_output("bigobject", self)
10
+
11
+ def self.transaction(config, schema, count, &control)
12
+ # configuration code:
13
+ task = {
14
+ "host" => config.param("host", :string), # string, required
15
+ "port" => config.param("port", :integer, default: 9090), # integer, optional
16
+ "table" => config.param("table", :string), # string, required
17
+ }
18
+ @@log = Logger.new(STDOUT)
19
+ @@tblexist = false
20
+ @@ttlcounter = 0
21
+ # resumable output:
22
+ # resume(task, schema, count, &control)
23
+
24
+ # non-resumable output:
25
+ task_reports = yield(task)
26
+ next_config_diff = {}
27
+ return next_config_diff
28
+ end
29
+
30
+ #def self.resume(task, schema, count, &control)
31
+ # task_reports = yield(task)
32
+ #
33
+ # next_config_diff = {}
34
+ # return next_config_diff
35
+ #end
36
+
37
+ def init
38
+ # initialization code:
39
+
40
+ @host = task["host"]
41
+ @port = task["port"]
42
+ @table = task["table"]
43
+
44
+ @uri = URI.parse("http://#@host:#@port/cmd")
45
+ @http = Net::HTTP.new(@uri.host, @uri.port)
46
+ @counter = 0
47
+
48
+ if !istbexist(@table)
49
+ if !createtbl(@table)
50
+ abort
51
+ end
52
+ end
53
+ end
54
+
55
+ def close
56
+ @http.finish
57
+ end
58
+
59
+ def add(page)
60
+ # output code:
61
+
62
+ data = Array.new
63
+ values = Array.new
64
+
65
+ page.each do |records|
66
+ values = []
67
+ records.each do |row| values << row.to_json end
68
+ data.push("(#{values.join(",")})")
69
+ end
70
+ @counter += data.length
71
+ @@ttlcounter += data.length
72
+
73
+ rsp = send_stmt("INSERT INTO #@table VALUES #{data.join(",")}")
74
+ if rsp["Status"] == 0 then
75
+ @@log.info "add #{data.length} to BigObject - #@counter | #@@ttlcounter"
76
+ else
77
+ @@log.error "add #{data.length} to BigObject failed!!"
78
+ abort
79
+ end
80
+
81
+ end
82
+
83
+ def finish
84
+ end
85
+
86
+ def abort
87
+ raise "Please Check BigObject"
88
+ end
89
+
90
+ def commit
91
+ task_report = {}
92
+ return task_report
93
+ end
94
+
95
+ private
96
+ def send_stmt(stmt)
97
+ begin
98
+ params = {"stmt" => "#{stmt}"}
99
+ json_headers = {"Content-Type" => "application/json", "Accept" => "application/json"}
100
+ response = @http.post(@uri.path, params.to_json, json_headers)
101
+ return JSON.parse(response.body)
102
+ rescue Exception => e
103
+ @@log.error e
104
+ abort
105
+ end
106
+
107
+ end
108
+
109
+ def istbexist(tbl)
110
+ if !@@tblexist then
111
+ stmt = "desc #{tbl}"
112
+ rsp = send_stmt(stmt)
113
+ rsp["Status"] == 0 ? @@tblexist = true : false
114
+ end
115
+ return @@tblexist
116
+ end
117
+
118
+ def createtbl(tbl)
119
+ rsp = send_stmt("CREATE TABLE #{tbl} (#{change2botype(schema).join(",")})")
120
+ rsp["Status"] == 0 ? @@tblexist = true : false
121
+ return @@tblexist
122
+ end
123
+
124
+ def change2botype(schema)
125
+ colntype = Array.new
126
+ schema.each do |s|
127
+ botype = ""
128
+ case s[2]
129
+ when /long/
130
+ botype = :INT64
131
+ when /boolean/
132
+ botype = :INT8
133
+ when /string/
134
+ botype = :STRING
135
+ when /double/
136
+ botype = :DOUBLE
137
+ when /timestamp/
138
+ if s[3].include? "%H"
139
+ botype = :DATETIME32
140
+ else
141
+ botype = :DATE32
142
+ end
143
+ end
144
+
145
+ colntype << "'#{s[1]}' #{botype}"
146
+ end
147
+ return colntype
148
+ end
149
+
150
+ end
151
+
152
+ end
153
+ end
@@ -0,0 +1,61 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require 'logger'
4
+
5
+
6
+ params = {"stmt" => "desc testcsv"}
7
+ json_headers = {"Content-Type" => "application/json",
8
+ "Accept" => "application/json"}
9
+
10
+ uri = URI.parse('http://192.168.99.100:9090/cmd')
11
+ http = Net::HTTP.new(uri.host, uri.port)
12
+
13
+ response = http.post(uri.path, params.to_json, json_headers)
14
+
15
+ @rsp = JSON.parse(response.body)
16
+
17
+ def checktbl
18
+
19
+ return @rsp["Status"] == 0 ? "YES" : "NO"
20
+ end
21
+
22
+ puts checktbl
23
+
24
+ # values = Array.new
25
+ # values2 = Array.new
26
+ # values2 << 1
27
+ # values2 << "2"
28
+ # values.push("(#{values2.join(",")})")
29
+ # values.push("id1")
30
+ # values.push(2)
31
+ # values.push("id2")
32
+ # str = "INSERT INTO #{values.join(",")}"
33
+ # log = Logger.new(STDOUT)
34
+ # puts str
35
+
36
+ # def init
37
+ # @http = Net::HTTP::Persistent.new 'embulk-plugin-bigobject'
38
+ # @http.headers['Content-Type'] = {"Content-Type" => "application/json", "Accept" => "application/json"}
39
+ # @uri = URI.parse('http://192.168.99.100:9090/cmd')
40
+ # puts "initialized"
41
+ # end
42
+
43
+ # def send
44
+ # #uri = URI.parse('http://192.168.99.100:9090/cmd')
45
+ # post = Net::HTTP::Post.new @uri.request_uri
46
+ # post.body = {"stmt" => "show tables"}.to_json
47
+ # response = @http.request @uri, post
48
+ # puts JSON.parse(response.body)
49
+ # end
50
+
51
+ # def shutdown
52
+ # @http.shutdown
53
+ # puts "shutdown"
54
+ # end
55
+
56
+ # init
57
+ # send
58
+ # shutdown
59
+
60
+
61
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-bigobject
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - randyviola
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-22 00:00:00.000000000 Z
11
+ date: 2016-01-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: embulk
@@ -58,8 +58,15 @@ email:
58
58
  executables: []
59
59
  extensions: []
60
60
  extra_rdoc_files: []
61
- files: []
62
- homepage:
61
+ files:
62
+ - Gemfile
63
+ - LICENSE.txt
64
+ - README.md
65
+ - Rakefile
66
+ - embulk-output-bigobject.gemspec
67
+ - lib/embulk/output/bigobject.rb
68
+ - lib/embulk/output/test.rb
69
+ homepage: https://github.com/randyh0329/embulk-output-bigobject
63
70
  licenses:
64
71
  - MIT
65
72
  metadata: {}