fluent-plugin-mysql-appender 0.4.8 → 0.4.9
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 424553e560bb98d0602e2f00c13e6301d72967b5
|
4
|
+
data.tar.gz: 189980b5c302be2016f59cc4834d92ce016b31d8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b53d691fb0fd6c040b91d2d2f6dd49220beb381efcb4007e71507e70d40788143ffc49d7d52db06aa517d8fd0d8c597e3f4bf9d144472b23212b108a149cbde2
|
7
|
+
data.tar.gz: dd33875581bd3784ff933f87f9f602d4da29087411b4c164739ee768bf51f398e9c3036d41bcd6e9e89302571ea98e665106232937faa4fe7b2fcf4ec0401e51
|
@@ -37,51 +37,67 @@ module Fluent
|
|
37
37
|
if !File.exist?(@yaml_path)
|
38
38
|
raise Fluent::ConfigError, "mysql_appender_multi: No such file in 'yaml_path'."
|
39
39
|
end
|
40
|
+
|
41
|
+
if @tag.nil?
|
42
|
+
raise Fluent::ConfigError, "mysql_appender_multi: missing 'tag' parameter. Please add following line into config like 'tag appender.${name}.${event}.${primary_key}'"
|
43
|
+
end
|
40
44
|
end
|
41
45
|
|
42
46
|
def start
|
43
|
-
|
47
|
+
begin
|
48
|
+
@threads = []
|
49
|
+
@mutex = Mutex.new
|
50
|
+
YAML.load_file(@yaml_path).each do |config|
|
51
|
+
@threads << Thread.new {
|
52
|
+
poll(config)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
$log.error "mysql_appender_multi: stop working due to empty configuration" if @threads.empty?
|
56
|
+
rescue => e
|
57
|
+
$log.error "error: #{e.message}"
|
58
|
+
$log.error e.backtrace.join("\n")
|
59
|
+
end
|
44
60
|
end
|
45
61
|
|
46
62
|
def shutdown
|
47
|
-
|
63
|
+
@threads.each do |thread|
|
64
|
+
Thread.kill(thread)
|
65
|
+
end
|
48
66
|
end
|
49
67
|
|
50
|
-
|
51
|
-
|
52
|
-
def run
|
68
|
+
def poll(config)
|
53
69
|
begin
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
61
|
-
|
70
|
+
tag = format_tag(config)
|
71
|
+
delay = Config.time_value(config['delay'] || 0)
|
72
|
+
@mutex.synchronize {
|
73
|
+
$log.info "mysql_appender_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
|
74
|
+
}
|
75
|
+
last_id = get_lastid(config)
|
62
76
|
loop do
|
77
|
+
rows_count = 0
|
63
78
|
start_time = Time.now
|
64
79
|
db = get_connection
|
65
|
-
|
66
|
-
|
67
|
-
db.query(get_query(config)).each do |row|
|
80
|
+
db.query(get_query(config, last_id)).each do |row|
|
81
|
+
if !config['entry_time'].nil? then
|
68
82
|
entry_time = get_time(row[config['entry_time']])
|
69
|
-
if (start_time -
|
83
|
+
if (start_time - delay) > entry_time then
|
70
84
|
if config['time_column'].nil? then
|
71
85
|
td_time = Engine.now
|
72
86
|
else
|
73
87
|
td_time = get_time(row[config['time_column']]).to_i
|
74
88
|
end
|
75
89
|
row.each {|k, v| row[k] = v.to_s if v.is_a?(Time) || v.is_a?(Date) || v.is_a?(BigDecimal)}
|
76
|
-
|
77
|
-
router.emit(config['tag'], td_time, row)
|
90
|
+
router.emit(tag, td_time, row)
|
78
91
|
rows_count += 1
|
79
|
-
|
92
|
+
last_id = row[config['primary_key']]
|
80
93
|
end
|
81
94
|
end
|
82
|
-
$log.info "mysql_appender_multi: finished execution :tag=>#{config['tag']} :rows_count=>#{rows_count} :last_id=>#{config['last_id']}"
|
83
95
|
end
|
84
96
|
db.close
|
97
|
+
elapsed_time = sprintf("%0.02f", Time.now - start_time)
|
98
|
+
@mutex.synchronize {
|
99
|
+
$log.info "mysql_appender_multi: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :last_id=>#{last_id} :elapsed_time=>#{elapsed_time} sec"
|
100
|
+
}
|
85
101
|
sleep @interval
|
86
102
|
end
|
87
103
|
rescue => e
|
@@ -129,8 +145,8 @@ module Fluent
|
|
129
145
|
end
|
130
146
|
end
|
131
147
|
|
132
|
-
def get_query(config)
|
133
|
-
"SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{
|
148
|
+
def get_query(config, last_id)
|
149
|
+
"SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{last_id} order by #{config['primary_key']} asc limit #{config['limit']}"
|
134
150
|
end
|
135
151
|
|
136
152
|
def format_tag(config)
|
data/setup_sample_db.sql
CHANGED
@@ -12,18 +12,3 @@ insert into test_tbl1 (column1, column2) value ('3','4');
|
|
12
12
|
insert into test_tbl1 (column1, column2) value ('5','6');
|
13
13
|
insert into test_tbl1 (column1, column2) value ('7','8');
|
14
14
|
|
15
|
-
create table test_tbl2 (
|
16
|
-
id int auto_increment,
|
17
|
-
column1 varchar(10),
|
18
|
-
column2 varchar(10),
|
19
|
-
created_at timestamp default current_timestamp,
|
20
|
-
primary key(id)
|
21
|
-
);
|
22
|
-
|
23
|
-
insert into test_tbl2 (column1, column2) value ('A','E');
|
24
|
-
insert into test_tbl2 (column1, column2) value ('B','F');
|
25
|
-
insert into test_tbl2 (column1, column2) value ('C','G');
|
26
|
-
insert into test_tbl2 (column1, column2) value ('D','H');
|
27
|
-
|
28
|
-
|
29
|
-
|
@@ -38,20 +38,7 @@ class MysqlAppenderMultiInputTest < Test::Unit::TestCase
|
|
38
38
|
- created_at
|
39
39
|
last_id: -1
|
40
40
|
entry_time: created_at
|
41
|
-
delay:
|
42
|
-
td_database: sample_datasets
|
43
|
-
- table_name: test_tbl2
|
44
|
-
primary_key: id
|
45
|
-
time_column: created_at
|
46
|
-
limit: 1000
|
47
|
-
columns:
|
48
|
-
- id
|
49
|
-
- column1
|
50
|
-
- column2
|
51
|
-
- created_at
|
52
|
-
last_id: -1
|
53
|
-
entry_time: created_at
|
54
|
-
delay: 5m
|
41
|
+
delay: 3h
|
55
42
|
td_database: sample_datasets
|
56
43
|
EOS
|
57
44
|
conf = YAML.load(str)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-appender
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TERASAKI Tsuyoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|