fluent-plugin-td 0.9.7 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog ADDED
@@ -0,0 +1,6 @@
1
+
2
+ Release 0.9.8 - 2011/09/03
3
+
4
+ * supports auto_create_table option
5
+ * uses td-client gem
6
+
data/Rakefile CHANGED
@@ -13,6 +13,7 @@ begin
13
13
  gemspec.has_rdoc = false
14
14
  gemspec.require_paths = ["lib"]
15
15
  gemspec.add_dependency "fluent", "~> 0.9.7"
16
+ gemspec.add_dependency "td-client", "~> 0.8.0"
16
17
  gemspec.test_files = Dir["test/**/*.rb"]
17
18
  gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
18
19
  %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-td.gemspec]
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.9.7
1
+ 0.9.8
@@ -5,13 +5,14 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{fluent-plugin-td}
8
- s.version = "0.9.7"
8
+ s.version = "0.9.8"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Sadayuki Furuhashi"]
12
- s.date = %q{2011-08-06}
12
+ s.date = %q{2011-09-03}
13
13
  s.extra_rdoc_files = [
14
- "README.rdoc"
14
+ "ChangeLog",
15
+ "README.rdoc"
15
16
  ]
16
17
  s.files = [
17
18
  "AUTHORS",
@@ -32,11 +33,14 @@ Gem::Specification.new do |s|
32
33
 
33
34
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
34
35
  s.add_runtime_dependency(%q<fluent>, ["~> 0.9.7"])
36
+ s.add_runtime_dependency(%q<td-client>, ["~> 0.8.0"])
35
37
  else
36
38
  s.add_dependency(%q<fluent>, ["~> 0.9.7"])
39
+ s.add_dependency(%q<td-client>, ["~> 0.8.0"])
37
40
  end
38
41
  else
39
42
  s.add_dependency(%q<fluent>, ["~> 0.9.7"])
43
+ s.add_dependency(%q<td-client>, ["~> 0.8.0"])
40
44
  end
41
45
  end
42
46
 
@@ -16,11 +16,15 @@ class TreasureDataLogOutput < BufferedOutput
16
16
  require 'json'
17
17
  require 'cgi' # CGI.escape
18
18
  require 'time' # Time#rfc2822
19
+ require 'td-client'
19
20
  super
20
21
  @tmpdir = '/tmp/fluent/tdlog'
21
22
  @apikey = nil
22
23
  @key = nil
24
+ @key_num_limit = 5120 # TODO
25
+ @record_size_limit = 32*1024*1024 # TODO
23
26
  @table_list = []
27
+ @auto_create_table = false
24
28
  end
25
29
 
26
30
  def configure(conf)
@@ -48,10 +52,16 @@ class TreasureDataLogOutput < BufferedOutput
48
52
  raise ConfigError, "'database' and 'table' parameter are required on tdlog output"
49
53
  end
50
54
 
51
- @table_list = get_table_list
55
+ if conf['auto_create_table']
56
+ @auto_create_table = true
57
+ end
58
+ end
52
59
 
53
- if @key && !@table_list.include?(@key)
54
- raise ConfigError, "Table #{@key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
60
+ def start
61
+ super
62
+ @client = TreasureData::Client.new(@apikey)
63
+ unless @auto_create_table
64
+ check_table_exists(@key)
55
65
  end
56
66
  end
57
67
 
@@ -67,18 +77,8 @@ class TreasureDataLogOutput < BufferedOutput
67
77
  key = "#{database}.#{table}"
68
78
  end
69
79
 
70
- # check the table exists
71
- unless @table_list.include?(key)
72
- begin
73
- @table_list = get_table_list
74
- rescue
75
- $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
76
- $log.debug_backtrace $!
77
- end
78
- unless @table_list.include?(key)
79
- database, table = key.split('.',2)
80
- raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
81
- end
80
+ unless @auto_create_table
81
+ check_table_exists(key)
82
82
  end
83
83
 
84
84
  super(tag, es, chain, key)
@@ -90,10 +90,23 @@ class TreasureDataLogOutput < BufferedOutput
90
90
 
91
91
  def format_stream(tag, es)
92
92
  out = ''
93
+ off = out.bytesize
93
94
  es.each {|event|
94
95
  record = event.record
95
96
  record['time'] = event.time
97
+
98
+ if record.size > @key_num_limit
99
+ raise "Too many number of keys (#{record.size} keys)" # TODO include summary of the record
100
+ end
101
+
96
102
  record.to_msgpack(out)
103
+
104
+ noff = out.bytesize
105
+ sz = noff - off
106
+ if sz > @record_size_limit
107
+ raise "Size of a record too large (#{sz} bytes)" # TODO include summary of the record
108
+ end
109
+ off = noff
97
110
  }
98
111
  out
99
112
  end
@@ -122,87 +135,51 @@ class TreasureDataLogOutput < BufferedOutput
122
135
  end
123
136
 
124
137
  def upload(database, table, io, size)
125
- http, header = new_http
126
- header['Content-Length'] = size.to_s
127
- header['Content-Type'] = 'application/octet-stream'
128
-
129
- url = "/v3/table/import/#{e database}/#{e table}/msgpack.gz"
130
-
131
- req = Net::HTTP::Put.new(url, header)
132
- if req.respond_to?(:body_stream=)
133
- req.body_stream = io
134
- else # Ruby 1.8
135
- req.body = io.read
136
- end
137
-
138
138
  $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
139
139
 
140
- response = http.request(req)
140
+ begin
141
+ @client.import(database, table, "msgpack.gz", io, size)
142
+ rescue TreasureData::NotFoundError
143
+ unless @auto_create_table
144
+ raise $!
145
+ end
146
+ $log.info "Creating table #{database}.#{table} on TreasureData"
147
+ begin
148
+ @client.create_log_table(database, table)
149
+ rescue TreasureData::NotFoundError
150
+ @client.create_database(database)
151
+ @client.create_log_table(database, table)
152
+ end
153
+ io.pos = 0
154
+ retry
155
+ end
156
+ end
141
157
 
142
- if response.code[0] != ?2
143
- raise "Treasure Data upload failed: #{response.body}"
158
+ def check_table_exists(key)
159
+ unless @table_list.include?(key)
160
+ begin
161
+ @table_list = get_table_list
162
+ rescue
163
+ $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
164
+ $log.debug_backtrace $!
165
+ end
166
+ unless @table_list.include?(key)
167
+ database, table = key.split('.',2)
168
+ raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
169
+ end
144
170
  end
145
171
  end
146
172
 
147
173
  def get_table_list
148
174
  $log.info "updating table list from Treasure Data"
149
175
  list = []
150
- api_list_database.each {|db|
151
- api_list_table(db).each {|t|
152
- list << "#{db}.#{t}"
176
+ @client.databases.each {|db|
177
+ db.tables.each {|tbl|
178
+ list << "#{db.name}.#{tbl.name}"
153
179
  }
154
180
  }
155
181
  list
156
182
  end
157
-
158
- def api_list_database
159
- body = get("/v3/database/list")
160
- js = JSON.load(body)
161
- return js["databases"].map {|m| m['name'] }
162
- end
163
-
164
- def api_list_table(db)
165
- body = get("/v3/table/list/#{e db}")
166
- js = JSON.load(body)
167
- return js["tables"].map {|m| m['name'] }
168
- end
169
-
170
- def get(path)
171
- http, header = new_http
172
-
173
- request = Net::HTTP::Get.new(path, header)
174
-
175
- response = http.request(request)
176
-
177
- if response.code[0] != ?2
178
- raise "Treasure Data API failed: #{response.body}"
179
- end
180
-
181
- return response.body
182
- end
183
-
184
- def new_http
185
- http = Net::HTTP.new(HOST, PORT)
186
- if USE_SSL
187
- http.use_ssl = true
188
- http.verify_mode = OpenSSL::SSL::VERIFY_PEER
189
- store = OpenSSL::X509::Store.new
190
- http.cert_store = store
191
- end
192
-
193
- # TODO read_timeout
194
- #http.read_timeout = options[:read_timeout]
195
-
196
- header = {}
197
- header['Authorization'] = "TD1 #{@apikey}"
198
- header['Date'] = Time.now.rfc2822
199
-
200
- return http, header
201
- end
202
-
203
- def e(s)
204
- CGI.escape(s.to_s)
205
- end
206
183
  end
207
184
 
208
185
 
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-td
3
3
  version: !ruby/object:Gem::Version
4
- hash: 53
4
+ hash: 43
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
8
  - 9
9
- - 7
10
- version: 0.9.7
9
+ - 8
10
+ version: 0.9.8
11
11
  platform: ruby
12
12
  authors:
13
13
  - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2011-08-06 00:00:00 +09:00
18
+ date: 2011-09-03 00:00:00 +09:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -34,6 +34,22 @@ dependencies:
34
34
  version: 0.9.7
35
35
  type: :runtime
36
36
  version_requirements: *id001
37
+ - !ruby/object:Gem::Dependency
38
+ name: td-client
39
+ prerelease: false
40
+ requirement: &id002 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ hash: 63
46
+ segments:
47
+ - 0
48
+ - 8
49
+ - 0
50
+ version: 0.8.0
51
+ type: :runtime
52
+ version_requirements: *id002
37
53
  description:
38
54
  email:
39
55
  executables: []
@@ -41,6 +57,7 @@ executables: []
41
57
  extensions: []
42
58
 
43
59
  extra_rdoc_files:
60
+ - ChangeLog
44
61
  - README.rdoc
45
62
  files:
46
63
  - AUTHORS
@@ -49,6 +66,7 @@ files:
49
66
  - example.conf
50
67
  - fluent-plugin-td.gemspec
51
68
  - lib/fluent/plugin/out_tdlog.rb
69
+ - ChangeLog
52
70
  - README.rdoc
53
71
  has_rdoc: true
54
72
  homepage: