fluent-plugin-td 0.9.7 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +6 -0
- data/Rakefile +1 -0
- data/VERSION +1 -1
- data/fluent-plugin-td.gemspec +7 -3
- data/lib/fluent/plugin/out_tdlog.rb +60 -83
- metadata +22 -4
data/ChangeLog
ADDED
data/Rakefile
CHANGED
@@ -13,6 +13,7 @@ begin
|
|
13
13
|
gemspec.has_rdoc = false
|
14
14
|
gemspec.require_paths = ["lib"]
|
15
15
|
gemspec.add_dependency "fluent", "~> 0.9.7"
|
16
|
+
gemspec.add_dependency "td-client", "~> 0.8.0"
|
16
17
|
gemspec.test_files = Dir["test/**/*.rb"]
|
17
18
|
gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
|
18
19
|
%w[example.conf VERSION AUTHORS Rakefile fluent-plugin-td.gemspec]
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.9.
|
1
|
+
0.9.8
|
data/fluent-plugin-td.gemspec
CHANGED
@@ -5,13 +5,14 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{fluent-plugin-td}
|
8
|
-
s.version = "0.9.
|
8
|
+
s.version = "0.9.8"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Sadayuki Furuhashi"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-09-03}
|
13
13
|
s.extra_rdoc_files = [
|
14
|
-
"
|
14
|
+
"ChangeLog",
|
15
|
+
"README.rdoc"
|
15
16
|
]
|
16
17
|
s.files = [
|
17
18
|
"AUTHORS",
|
@@ -32,11 +33,14 @@ Gem::Specification.new do |s|
|
|
32
33
|
|
33
34
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
34
35
|
s.add_runtime_dependency(%q<fluent>, ["~> 0.9.7"])
|
36
|
+
s.add_runtime_dependency(%q<td-client>, ["~> 0.8.0"])
|
35
37
|
else
|
36
38
|
s.add_dependency(%q<fluent>, ["~> 0.9.7"])
|
39
|
+
s.add_dependency(%q<td-client>, ["~> 0.8.0"])
|
37
40
|
end
|
38
41
|
else
|
39
42
|
s.add_dependency(%q<fluent>, ["~> 0.9.7"])
|
43
|
+
s.add_dependency(%q<td-client>, ["~> 0.8.0"])
|
40
44
|
end
|
41
45
|
end
|
42
46
|
|
@@ -16,11 +16,15 @@ class TreasureDataLogOutput < BufferedOutput
|
|
16
16
|
require 'json'
|
17
17
|
require 'cgi' # CGI.escape
|
18
18
|
require 'time' # Time#rfc2822
|
19
|
+
require 'td-client'
|
19
20
|
super
|
20
21
|
@tmpdir = '/tmp/fluent/tdlog'
|
21
22
|
@apikey = nil
|
22
23
|
@key = nil
|
24
|
+
@key_num_limit = 5120 # TODO
|
25
|
+
@record_size_limit = 32*1024*1024 # TODO
|
23
26
|
@table_list = []
|
27
|
+
@auto_create_table = false
|
24
28
|
end
|
25
29
|
|
26
30
|
def configure(conf)
|
@@ -48,10 +52,16 @@ class TreasureDataLogOutput < BufferedOutput
|
|
48
52
|
raise ConfigError, "'database' and 'table' parameter are required on tdlog output"
|
49
53
|
end
|
50
54
|
|
51
|
-
|
55
|
+
if conf['auto_create_table']
|
56
|
+
@auto_create_table = true
|
57
|
+
end
|
58
|
+
end
|
52
59
|
|
53
|
-
|
54
|
-
|
60
|
+
def start
|
61
|
+
super
|
62
|
+
@client = TreasureData::Client.new(@apikey)
|
63
|
+
unless @auto_create_table
|
64
|
+
check_table_exists(@key)
|
55
65
|
end
|
56
66
|
end
|
57
67
|
|
@@ -67,18 +77,8 @@ class TreasureDataLogOutput < BufferedOutput
|
|
67
77
|
key = "#{database}.#{table}"
|
68
78
|
end
|
69
79
|
|
70
|
-
|
71
|
-
|
72
|
-
begin
|
73
|
-
@table_list = get_table_list
|
74
|
-
rescue
|
75
|
-
$log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
|
76
|
-
$log.debug_backtrace $!
|
77
|
-
end
|
78
|
-
unless @table_list.include?(key)
|
79
|
-
database, table = key.split('.',2)
|
80
|
-
raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
|
81
|
-
end
|
80
|
+
unless @auto_create_table
|
81
|
+
check_table_exists(key)
|
82
82
|
end
|
83
83
|
|
84
84
|
super(tag, es, chain, key)
|
@@ -90,10 +90,23 @@ class TreasureDataLogOutput < BufferedOutput
|
|
90
90
|
|
91
91
|
def format_stream(tag, es)
|
92
92
|
out = ''
|
93
|
+
off = out.bytesize
|
93
94
|
es.each {|event|
|
94
95
|
record = event.record
|
95
96
|
record['time'] = event.time
|
97
|
+
|
98
|
+
if record.size > @key_num_limit
|
99
|
+
raise "Too many number of keys (#{record.size} keys)" # TODO include summary of the record
|
100
|
+
end
|
101
|
+
|
96
102
|
record.to_msgpack(out)
|
103
|
+
|
104
|
+
noff = out.bytesize
|
105
|
+
sz = noff - off
|
106
|
+
if sz > @record_size_limit
|
107
|
+
raise "Size of a record too large (#{sz} bytes)" # TODO include summary of the record
|
108
|
+
end
|
109
|
+
off = noff
|
97
110
|
}
|
98
111
|
out
|
99
112
|
end
|
@@ -122,87 +135,51 @@ class TreasureDataLogOutput < BufferedOutput
|
|
122
135
|
end
|
123
136
|
|
124
137
|
def upload(database, table, io, size)
|
125
|
-
http, header = new_http
|
126
|
-
header['Content-Length'] = size.to_s
|
127
|
-
header['Content-Type'] = 'application/octet-stream'
|
128
|
-
|
129
|
-
url = "/v3/table/import/#{e database}/#{e table}/msgpack.gz"
|
130
|
-
|
131
|
-
req = Net::HTTP::Put.new(url, header)
|
132
|
-
if req.respond_to?(:body_stream=)
|
133
|
-
req.body_stream = io
|
134
|
-
else # Ruby 1.8
|
135
|
-
req.body = io.read
|
136
|
-
end
|
137
|
-
|
138
138
|
$log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
|
139
139
|
|
140
|
-
|
140
|
+
begin
|
141
|
+
@client.import(database, table, "msgpack.gz", io, size)
|
142
|
+
rescue TreasureData::NotFoundError
|
143
|
+
unless @auto_create_table
|
144
|
+
raise $!
|
145
|
+
end
|
146
|
+
$log.info "Creating table #{database}.#{table} on TreasureData"
|
147
|
+
begin
|
148
|
+
@client.create_log_table(database, table)
|
149
|
+
rescue TreasureData::NotFoundError
|
150
|
+
@client.create_database(database)
|
151
|
+
@client.create_log_table(database, table)
|
152
|
+
end
|
153
|
+
io.pos = 0
|
154
|
+
retry
|
155
|
+
end
|
156
|
+
end
|
141
157
|
|
142
|
-
|
143
|
-
|
158
|
+
def check_table_exists(key)
|
159
|
+
unless @table_list.include?(key)
|
160
|
+
begin
|
161
|
+
@table_list = get_table_list
|
162
|
+
rescue
|
163
|
+
$log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
|
164
|
+
$log.debug_backtrace $!
|
165
|
+
end
|
166
|
+
unless @table_list.include?(key)
|
167
|
+
database, table = key.split('.',2)
|
168
|
+
raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
|
169
|
+
end
|
144
170
|
end
|
145
171
|
end
|
146
172
|
|
147
173
|
def get_table_list
|
148
174
|
$log.info "updating table list from Treasure Data"
|
149
175
|
list = []
|
150
|
-
|
151
|
-
|
152
|
-
list << "#{db}.#{
|
176
|
+
@client.databases.each {|db|
|
177
|
+
db.tables.each {|tbl|
|
178
|
+
list << "#{db.name}.#{tbl.name}"
|
153
179
|
}
|
154
180
|
}
|
155
181
|
list
|
156
182
|
end
|
157
|
-
|
158
|
-
def api_list_database
|
159
|
-
body = get("/v3/database/list")
|
160
|
-
js = JSON.load(body)
|
161
|
-
return js["databases"].map {|m| m['name'] }
|
162
|
-
end
|
163
|
-
|
164
|
-
def api_list_table(db)
|
165
|
-
body = get("/v3/table/list/#{e db}")
|
166
|
-
js = JSON.load(body)
|
167
|
-
return js["tables"].map {|m| m['name'] }
|
168
|
-
end
|
169
|
-
|
170
|
-
def get(path)
|
171
|
-
http, header = new_http
|
172
|
-
|
173
|
-
request = Net::HTTP::Get.new(path, header)
|
174
|
-
|
175
|
-
response = http.request(request)
|
176
|
-
|
177
|
-
if response.code[0] != ?2
|
178
|
-
raise "Treasure Data API failed: #{response.body}"
|
179
|
-
end
|
180
|
-
|
181
|
-
return response.body
|
182
|
-
end
|
183
|
-
|
184
|
-
def new_http
|
185
|
-
http = Net::HTTP.new(HOST, PORT)
|
186
|
-
if USE_SSL
|
187
|
-
http.use_ssl = true
|
188
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
189
|
-
store = OpenSSL::X509::Store.new
|
190
|
-
http.cert_store = store
|
191
|
-
end
|
192
|
-
|
193
|
-
# TODO read_timeout
|
194
|
-
#http.read_timeout = options[:read_timeout]
|
195
|
-
|
196
|
-
header = {}
|
197
|
-
header['Authorization'] = "TD1 #{@apikey}"
|
198
|
-
header['Date'] = Time.now.rfc2822
|
199
|
-
|
200
|
-
return http, header
|
201
|
-
end
|
202
|
-
|
203
|
-
def e(s)
|
204
|
-
CGI.escape(s.to_s)
|
205
|
-
end
|
206
183
|
end
|
207
184
|
|
208
185
|
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 43
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 9
|
9
|
-
-
|
10
|
-
version: 0.9.
|
9
|
+
- 8
|
10
|
+
version: 0.9.8
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sadayuki Furuhashi
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-09-03 00:00:00 +09:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -34,6 +34,22 @@ dependencies:
|
|
34
34
|
version: 0.9.7
|
35
35
|
type: :runtime
|
36
36
|
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: td-client
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 63
|
46
|
+
segments:
|
47
|
+
- 0
|
48
|
+
- 8
|
49
|
+
- 0
|
50
|
+
version: 0.8.0
|
51
|
+
type: :runtime
|
52
|
+
version_requirements: *id002
|
37
53
|
description:
|
38
54
|
email:
|
39
55
|
executables: []
|
@@ -41,6 +57,7 @@ executables: []
|
|
41
57
|
extensions: []
|
42
58
|
|
43
59
|
extra_rdoc_files:
|
60
|
+
- ChangeLog
|
44
61
|
- README.rdoc
|
45
62
|
files:
|
46
63
|
- AUTHORS
|
@@ -49,6 +66,7 @@ files:
|
|
49
66
|
- example.conf
|
50
67
|
- fluent-plugin-td.gemspec
|
51
68
|
- lib/fluent/plugin/out_tdlog.rb
|
69
|
+
- ChangeLog
|
52
70
|
- README.rdoc
|
53
71
|
has_rdoc: true
|
54
72
|
homepage:
|