fluent-plugin-mysql-appender 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-mysql-appender.gemspec +3 -1
- data/lib/fluent/plugin/in_mysql_appender_multi.rb +42 -2
- metadata +29 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05ca7cbe8c0b8975c51ba14c2e9f34dcb5e8125c
|
4
|
+
data.tar.gz: 2bff063d49ccb630916f3d515410e93307eec2e3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05c5048cd4aed2b09bea444c224dc92e5816ff611d4f5d335c6573e08599d4186ee762043f4a1db0d33096b2335a6f2903c53d29977933e170e264f798c538ae
|
7
|
+
data.tar.gz: 72f2b0388fa398595de8463626fbd67c6058e0522c1d8b437a0f92f3a27aa22e56cb9b7c684f91e3efb267a6bb5747e1f63fc131106670a6115670efea97a104
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "fluent-plugin-mysql-appender"
|
4
|
-
spec.version = "0.4.
|
4
|
+
spec.version = "0.4.1"
|
5
5
|
spec.authors = ["TERASAKI Tsuyoshi"]
|
6
6
|
spec.email = ["tsuyoshi_terasaki@realworld.jp"]
|
7
7
|
|
@@ -17,6 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
|
18
18
|
spec.add_runtime_dependency "fluentd"
|
19
19
|
spec.add_runtime_dependency "mysql2"
|
20
|
+
spec.add_runtime_dependency "td"
|
21
|
+
spec.add_runtime_dependency "td-client"
|
20
22
|
|
21
23
|
spec.add_development_dependency "rake"
|
22
24
|
spec.add_development_dependency "test-unit"
|
@@ -11,6 +11,8 @@ module Fluent
|
|
11
11
|
require 'mysql2'
|
12
12
|
require 'time'
|
13
13
|
require 'yaml'
|
14
|
+
require 'td'
|
15
|
+
require 'td-client'
|
14
16
|
super
|
15
17
|
end
|
16
18
|
|
@@ -71,7 +73,7 @@ module Fluent
|
|
71
73
|
$log.info "mysql_replicator_multi: polling start. :tag=>#{tag} :delay=>#{delay}"
|
72
74
|
}
|
73
75
|
con = get_connection()
|
74
|
-
last_id = config
|
76
|
+
last_id = get_lastid(config)
|
75
77
|
loop do
|
76
78
|
rows_count = 0
|
77
79
|
start_time = Time.now
|
@@ -107,6 +109,44 @@ module Fluent
|
|
107
109
|
end
|
108
110
|
end
|
109
111
|
|
112
|
+
def get_lastid(config)
|
113
|
+
begin
|
114
|
+
if !ENV.key?('TD_APIKEY') || !ENV.key?('TD_ENDPOINT') then
|
115
|
+
return -1
|
116
|
+
end
|
117
|
+
cln = TreasureData::Client.new(ENV['TD_APIKEY'],{:endpoint => "https://" + ENV['TD_ENDPOINT']})
|
118
|
+
table_exists = false
|
119
|
+
cln.databases.each { |db|
|
120
|
+
db.tables.each { |tbl|
|
121
|
+
if tbl.db_name == config['td_database'] && tbl.table_name == config['table_name'] then
|
122
|
+
table_exists = true
|
123
|
+
break
|
124
|
+
end
|
125
|
+
}
|
126
|
+
}
|
127
|
+
if table_exists then
|
128
|
+
query = "SELECT MAX(#{config['primary_key']}) FROM #{config['table_name']}"
|
129
|
+
job = cln.query(config['td_database'], query, nil, nil, nil , {:type => :presto})
|
130
|
+
until job.finished?
|
131
|
+
sleep 2
|
132
|
+
job.update_progress!
|
133
|
+
end
|
134
|
+
job.update_status! # get latest info
|
135
|
+
job.result_each { |row|
|
136
|
+
$log.info "mysql_replicator_multi: #{config['td_database']}.#{config['table_name']}'s last_id is #{row.first} "
|
137
|
+
return row.first
|
138
|
+
}
|
139
|
+
else
|
140
|
+
$log.info "mysql_replicator_multi: #{config['td_database']}.#{config['table_name']} is not found. "
|
141
|
+
return -1
|
142
|
+
end
|
143
|
+
rescue => e
|
144
|
+
$log.warn "mysql_appender_multi: failed to get lastid. #{config}"
|
145
|
+
$log.error "error: #{e.message}"
|
146
|
+
$log.error e.backtrace.join("\n")
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
110
150
|
def get_query(config, last_id)
|
111
151
|
"SELECT #{config['columns'].join(",")} FROM #{config['table_name']} where #{config['primary_key']} > #{last_id} order by #{config['primary_key']} asc limit #{config['limit']}"
|
112
152
|
end
|
@@ -141,7 +181,7 @@ module Fluent
|
|
141
181
|
:stream => true,
|
142
182
|
:cache_rows => false
|
143
183
|
})
|
144
|
-
rescue
|
184
|
+
rescue Mysql2::Error => e
|
145
185
|
$log.warn "mysql_appender_multi: #{e}"
|
146
186
|
sleep @interval
|
147
187
|
retry
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-appender
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TERASAKI Tsuyoshi
|
@@ -38,6 +38,34 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: td
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: td-client
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
41
69
|
- !ruby/object:Gem::Dependency
|
42
70
|
name: rake
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|