fluent-plugin-sql 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +2 -2
- data/VERSION +1 -1
- data/fluent-plugin-sql.gemspec +2 -1
- data/lib/fluent/plugin/in_sql.rb +19 -13
- data/lib/fluent/plugin/out_sql.rb +232 -0
- metadata +34 -32
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9ca23328ec98228710390c94c0f3f3a1d762e3c4
|
4
|
+
data.tar.gz: 0ff6ef9512db112b7938b2af612a2ffe59ddd59e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 302054c46157cc5438085b75e612a8353d6859b5f997ca66984b2064231e25587cdaee438050e645c345ad33c3a2377a21a85f3317f6d25c1fdceaf181e8f29f
|
7
|
+
data.tar.gz: d567b461ca80f21adbd7c07e33a4deb801a20ea54ccc955e6091e84230ce51c7925b41520aa8640788079921d197bf1f33727a9b4f736dad2158a921cefc8537
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# SQL input plugin for Fluentd event collector
|
1
|
+
# SQL input plugin for [Fluentd](http://fluentd.org) event collector
|
2
2
|
|
3
3
|
## Overview
|
4
4
|
|
@@ -23,7 +23,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
|
|
23
23
|
host rdb_host
|
24
24
|
database rdb_database
|
25
25
|
adapter mysql2_or_postgresql_etc
|
26
|
-
|
26
|
+
username myusername
|
27
27
|
password mypassword
|
28
28
|
|
29
29
|
tag_prefix my.rdb # optional, but recommended
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
data/fluent-plugin-sql.gemspec
CHANGED
@@ -18,7 +18,8 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.license = "Apache 2.0"
|
19
19
|
|
20
20
|
gem.add_dependency "fluentd", "~> 0.10.0"
|
21
|
-
gem.add_dependency 'activerecord',
|
21
|
+
gem.add_dependency 'activerecord', "~> 4.0.2"
|
22
|
+
gem.add_dependency 'activerecord-import', "~> 0.4.1"
|
22
23
|
gem.add_dependency 'mysql2', ['~> 0.3.12']
|
23
24
|
gem.add_dependency 'pg', ['~> 0.16.0']
|
24
25
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
data/lib/fluent/plugin/in_sql.rb
CHANGED
@@ -28,12 +28,17 @@ module Fluent
|
|
28
28
|
config_param :database, :string
|
29
29
|
config_param :username, :string, :default => nil
|
30
30
|
config_param :password, :string, :default => nil
|
31
|
+
config_param :socket, :string, :default => nil
|
31
32
|
|
32
33
|
config_param :state_file, :string, :default => nil
|
33
34
|
config_param :tag_prefix, :string, :default => nil
|
34
35
|
config_param :select_interval, :time, :default => 60
|
35
36
|
config_param :select_limit, :time, :default => 500
|
36
37
|
|
38
|
+
unless method_defined?(:log)
|
39
|
+
define_method(:log) { $log }
|
40
|
+
end
|
41
|
+
|
37
42
|
class TableElement
|
38
43
|
include Configurable
|
39
44
|
|
@@ -44,11 +49,6 @@ module Fluent
|
|
44
49
|
|
45
50
|
def configure(conf)
|
46
51
|
super
|
47
|
-
|
48
|
-
unless @state_file
|
49
|
-
$log.warn "'state_file PATH' parameter is not set to a 'sql' source."
|
50
|
-
$log.warn "this parameter is highly recommended to save the last rows to resume tailing."
|
51
|
-
end
|
52
52
|
end
|
53
53
|
|
54
54
|
def init(tag_prefix, base_model)
|
@@ -129,6 +129,11 @@ module Fluent
|
|
129
129
|
def configure(conf)
|
130
130
|
super
|
131
131
|
|
132
|
+
unless @state_file
|
133
|
+
$log.warn "'state_file PATH' parameter is not set to a 'sql' source."
|
134
|
+
$log.warn "this parameter is highly recommended to save the last rows to resume tailing."
|
135
|
+
end
|
136
|
+
|
132
137
|
@tables = conf.elements.select {|e|
|
133
138
|
e.name == 'table'
|
134
139
|
}.map {|e|
|
@@ -154,6 +159,7 @@ module Fluent
|
|
154
159
|
:database => @database,
|
155
160
|
:username => @username,
|
156
161
|
:password => @password,
|
162
|
+
:socket => @socket,
|
157
163
|
}
|
158
164
|
|
159
165
|
# creates subclass of ActiveRecord::Base so that it can have different
|
@@ -166,7 +172,7 @@ module Fluent
|
|
166
172
|
# ActiveRecord requires the base_model to have a name. Here sets name
|
167
173
|
# of an anonymous class by assigning it to a constant. In Ruby, class has
|
168
174
|
# a name of a constant assigned first
|
169
|
-
SQLInput.const_set("BaseModel_#{rand(1<<31)}", @base_model)
|
175
|
+
SQLInput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
|
170
176
|
|
171
177
|
# Now base_model can have independent configuration from ActiveRecord::Base
|
172
178
|
@base_model.establish_connection(config)
|
@@ -193,11 +199,11 @@ module Fluent
|
|
193
199
|
@tables.reject! do |te|
|
194
200
|
begin
|
195
201
|
te.init(@tag_prefix, @base_model)
|
196
|
-
|
202
|
+
log.info "Selecting '#{te.table}' table"
|
197
203
|
false
|
198
|
-
rescue
|
199
|
-
|
200
|
-
|
204
|
+
rescue => e
|
205
|
+
log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
|
206
|
+
log.warn_backtrace e.backtrace
|
201
207
|
true
|
202
208
|
end
|
203
209
|
end
|
@@ -219,9 +225,9 @@ module Fluent
|
|
219
225
|
last_record = @state_store.last_records[t.table]
|
220
226
|
@state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
|
221
227
|
@state_store.update!
|
222
|
-
rescue
|
223
|
-
|
224
|
-
|
228
|
+
rescue => e
|
229
|
+
log.error "unexpected error", :error => e.message, :error_class => e.class
|
230
|
+
log.error_backtrace e.backtrace
|
225
231
|
end
|
226
232
|
end
|
227
233
|
end
|
@@ -0,0 +1,232 @@
|
|
1
|
+
module Fluent
|
2
|
+
class SQLOutput < BufferedOutput
|
3
|
+
Plugin.register_output('sql', self)
|
4
|
+
|
5
|
+
include SetTimeKeyMixin
|
6
|
+
include SetTagKeyMixin
|
7
|
+
|
8
|
+
config_param :host, :string
|
9
|
+
config_param :port, :integer, :default => nil
|
10
|
+
config_param :adapter, :string
|
11
|
+
config_param :username, :string, :default => nil
|
12
|
+
config_param :password, :string, :default => nil
|
13
|
+
config_param :database, :string
|
14
|
+
config_param :socket, :string, :default => nil
|
15
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
16
|
+
|
17
|
+
attr_accessor :tables
|
18
|
+
|
19
|
+
unless method_defined?(:log)
|
20
|
+
define_method(:log) { $log }
|
21
|
+
end
|
22
|
+
|
23
|
+
# TODO: Merge SQLInput's TableElement
|
24
|
+
class TableElement
|
25
|
+
include Configurable
|
26
|
+
|
27
|
+
config_param :table, :string
|
28
|
+
config_param :column_mapping, :string
|
29
|
+
config_param :num_retries, :integer, :default => 5
|
30
|
+
|
31
|
+
attr_reader :model
|
32
|
+
attr_reader :pattern
|
33
|
+
|
34
|
+
def initialize(pattern, log)
|
35
|
+
super()
|
36
|
+
@pattern = MatchPattern.create(pattern)
|
37
|
+
@log = log
|
38
|
+
end
|
39
|
+
|
40
|
+
def configure(conf)
|
41
|
+
super
|
42
|
+
|
43
|
+
@mapping = parse_column_mapping(@column_mapping)
|
44
|
+
@format_proc = Proc.new { |record|
|
45
|
+
new_record = {}
|
46
|
+
@mapping.each { |k, c|
|
47
|
+
new_record[c] = record[k]
|
48
|
+
}
|
49
|
+
new_record
|
50
|
+
}
|
51
|
+
end
|
52
|
+
|
53
|
+
def init(base_model)
|
54
|
+
# See SQLInput for more details of following code
|
55
|
+
table_name = @table
|
56
|
+
@model = Class.new(base_model) do
|
57
|
+
self.table_name = table_name
|
58
|
+
self.inheritance_column = '_never_use_output_'
|
59
|
+
end
|
60
|
+
|
61
|
+
class_name = table_name.singularize.camelize
|
62
|
+
base_model.const_set(class_name, @model)
|
63
|
+
model_name = ActiveModel::Name.new(@model, nil, class_name)
|
64
|
+
@model.define_singleton_method(:model_name) { model_name }
|
65
|
+
|
66
|
+
# TODO: check column_names and table schema
|
67
|
+
columns = @model.columns.map { |column| column.name }.sort
|
68
|
+
end
|
69
|
+
|
70
|
+
def import(chunk)
|
71
|
+
records = []
|
72
|
+
chunk.msgpack_each { |tag, time, data|
|
73
|
+
begin
|
74
|
+
# format process should be moved to emit / format after supports error stream.
|
75
|
+
records << @model.new(@format_proc.call(data))
|
76
|
+
rescue => e
|
77
|
+
args = {:error => e.message, :error_class => e.class, :table => @table, :record => Yajl.dump(data)}
|
78
|
+
@log.warn "Failed to create the model. Ignore a record:", args
|
79
|
+
end
|
80
|
+
}
|
81
|
+
begin
|
82
|
+
@model.import(records)
|
83
|
+
rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
|
84
|
+
# ignore other exceptions to use Fluentd retry mechanizm
|
85
|
+
@log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
|
86
|
+
one_by_one_import(records)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def one_by_one_import(records)
|
91
|
+
records.each { |record|
|
92
|
+
retries = 0
|
93
|
+
begin
|
94
|
+
@model.import([record])
|
95
|
+
rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
|
96
|
+
@log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
|
97
|
+
rescue => e
|
98
|
+
retries += 1
|
99
|
+
if retries > @num_retries
|
100
|
+
@log.error "Can't recover undeterministic error. Dump a record", :error => e.message, :error_class => e.class, :record => record
|
101
|
+
next
|
102
|
+
end
|
103
|
+
|
104
|
+
@log.warn "Failed to import a record: retry number = #{retries}", :error => e.message, :error_class => e.class
|
105
|
+
sleep 0.5
|
106
|
+
retry
|
107
|
+
end
|
108
|
+
}
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
def parse_column_mapping(column_mapping_conf)
|
114
|
+
mapping = {}
|
115
|
+
column_mapping_conf.split(',').each { |column_map|
|
116
|
+
key, column = column_map.strip.split(':', 2)
|
117
|
+
column = key if column.nil?
|
118
|
+
mapping[key] = column
|
119
|
+
}
|
120
|
+
mapping
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def initialize
|
125
|
+
super
|
126
|
+
require 'active_record'
|
127
|
+
require 'activerecord-import'
|
128
|
+
end
|
129
|
+
|
130
|
+
def configure(conf)
|
131
|
+
super
|
132
|
+
|
133
|
+
if remove_tag_prefix = conf['remove_tag_prefix']
|
134
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
135
|
+
end
|
136
|
+
|
137
|
+
@tables = []
|
138
|
+
@default_table = nil
|
139
|
+
conf.elements.select { |e|
|
140
|
+
e.name == 'table'
|
141
|
+
}.each { |e|
|
142
|
+
te = TableElement.new(e.arg, log)
|
143
|
+
te.configure(e)
|
144
|
+
if e.arg.empty?
|
145
|
+
$log.warn "Detect duplicate default table definition" if @default_table
|
146
|
+
@default_table = te
|
147
|
+
else
|
148
|
+
@tables << te
|
149
|
+
end
|
150
|
+
}
|
151
|
+
@only_default = @tables.empty?
|
152
|
+
|
153
|
+
if @default_table.nil?
|
154
|
+
raise ConfigError, "There is no default table. <table> is required in sql output"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def start
|
159
|
+
super
|
160
|
+
|
161
|
+
config = {
|
162
|
+
:adapter => @adapter,
|
163
|
+
:host => @host,
|
164
|
+
:port => @port,
|
165
|
+
:database => @database,
|
166
|
+
:username => @username,
|
167
|
+
:password => @password,
|
168
|
+
:socket => @socket,
|
169
|
+
}
|
170
|
+
|
171
|
+
@base_model = Class.new(ActiveRecord::Base) do
|
172
|
+
self.abstract_class = true
|
173
|
+
end
|
174
|
+
|
175
|
+
SQLOutput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
|
176
|
+
@base_model.establish_connection(config)
|
177
|
+
|
178
|
+
# ignore tables if TableElement#init failed
|
179
|
+
@tables.reject! do |te|
|
180
|
+
init_table(te, @base_model)
|
181
|
+
end
|
182
|
+
init_table(@default_table, @base_model)
|
183
|
+
end
|
184
|
+
|
185
|
+
def shutdown
|
186
|
+
super
|
187
|
+
end
|
188
|
+
|
189
|
+
def emit(tag, es, chain)
|
190
|
+
if @only_default
|
191
|
+
super(tag, es, chain)
|
192
|
+
else
|
193
|
+
super(tag, es, chain, format_tag(tag))
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
def format(tag, time, record)
|
198
|
+
[tag, time, record].to_msgpack
|
199
|
+
end
|
200
|
+
|
201
|
+
def write(chunk)
|
202
|
+
@tables.each { |table|
|
203
|
+
if table.pattern.match(chunk.key)
|
204
|
+
return table.import(chunk)
|
205
|
+
end
|
206
|
+
}
|
207
|
+
@default_table.import(chunk)
|
208
|
+
end
|
209
|
+
|
210
|
+
private
|
211
|
+
|
212
|
+
def init_table(te, base_model)
|
213
|
+
begin
|
214
|
+
te.init(base_model)
|
215
|
+
log.info "Selecting '#{te.table}' table"
|
216
|
+
false
|
217
|
+
rescue => e
|
218
|
+
log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
|
219
|
+
log.warn_backtrace e.backtrace
|
220
|
+
true
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def format_tag(tag)
|
225
|
+
if @remove_tag_prefix
|
226
|
+
tag.gsub(@remove_tag_prefix, '')
|
227
|
+
else
|
228
|
+
tag
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
metadata
CHANGED
@@ -1,94 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.3.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Sadayuki Furuhashi
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2014-08-28 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- - ~>
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: 0.10.0
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- - ~>
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: 0.10.0
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: activerecord
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - "~>"
|
36
32
|
- !ruby/object:Gem::Version
|
37
|
-
version:
|
33
|
+
version: 4.0.2
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - "~>"
|
44
39
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
40
|
+
version: 4.0.2
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: activerecord-import
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.4.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.4.1
|
46
55
|
- !ruby/object:Gem::Dependency
|
47
56
|
name: mysql2
|
48
57
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
58
|
requirements:
|
51
|
-
- - ~>
|
59
|
+
- - "~>"
|
52
60
|
- !ruby/object:Gem::Version
|
53
61
|
version: 0.3.12
|
54
62
|
type: :runtime
|
55
63
|
prerelease: false
|
56
64
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
65
|
requirements:
|
59
|
-
- - ~>
|
66
|
+
- - "~>"
|
60
67
|
- !ruby/object:Gem::Version
|
61
68
|
version: 0.3.12
|
62
69
|
- !ruby/object:Gem::Dependency
|
63
70
|
name: pg
|
64
71
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
72
|
requirements:
|
67
|
-
- - ~>
|
73
|
+
- - "~>"
|
68
74
|
- !ruby/object:Gem::Version
|
69
75
|
version: 0.16.0
|
70
76
|
type: :runtime
|
71
77
|
prerelease: false
|
72
78
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
79
|
requirements:
|
75
|
-
- - ~>
|
80
|
+
- - "~>"
|
76
81
|
- !ruby/object:Gem::Version
|
77
82
|
version: 0.16.0
|
78
83
|
- !ruby/object:Gem::Dependency
|
79
84
|
name: rake
|
80
85
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
86
|
requirements:
|
83
|
-
- -
|
87
|
+
- - ">="
|
84
88
|
- !ruby/object:Gem::Version
|
85
89
|
version: 0.9.2
|
86
90
|
type: :development
|
87
91
|
prerelease: false
|
88
92
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
93
|
requirements:
|
91
|
-
- -
|
94
|
+
- - ">="
|
92
95
|
- !ruby/object:Gem::Version
|
93
96
|
version: 0.9.2
|
94
97
|
description: SQL input/output plugin for Fluentd event collector
|
@@ -103,30 +106,29 @@ files:
|
|
103
106
|
- VERSION
|
104
107
|
- fluent-plugin-sql.gemspec
|
105
108
|
- lib/fluent/plugin/in_sql.rb
|
109
|
+
- lib/fluent/plugin/out_sql.rb
|
106
110
|
homepage: https://github.com/frsyuki/fluent-plugin-sql
|
107
111
|
licenses:
|
108
112
|
- Apache 2.0
|
113
|
+
metadata: {}
|
109
114
|
post_install_message:
|
110
115
|
rdoc_options: []
|
111
116
|
require_paths:
|
112
117
|
- lib
|
113
118
|
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
-
none: false
|
115
119
|
requirements:
|
116
|
-
- -
|
120
|
+
- - ">="
|
117
121
|
- !ruby/object:Gem::Version
|
118
122
|
version: '0'
|
119
123
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
-
none: false
|
121
124
|
requirements:
|
122
|
-
- -
|
125
|
+
- - ">="
|
123
126
|
- !ruby/object:Gem::Version
|
124
127
|
version: '0'
|
125
128
|
requirements: []
|
126
129
|
rubyforge_project:
|
127
|
-
rubygems_version:
|
130
|
+
rubygems_version: 2.2.2
|
128
131
|
signing_key:
|
129
|
-
specification_version:
|
132
|
+
specification_version: 4
|
130
133
|
summary: SQL input/output plugin for Fluentd event collector
|
131
134
|
test_files: []
|
132
|
-
has_rdoc: false
|