fluent-plugin-mysql-appender 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/fluent-plugin-mysql-appender.gemspec +12 -7
- data/lib/fluent/plugin/mysql/appender.rb +130 -0
- data/lib/fluent/plugin/mysql/appender/version.rb +9 -0
- metadata +34 -35
- data/lib/fluent/plugin/in_mysql_appender.rb +0 -122
- data/test/fluent/plugin/in_mysql_appender_test.rb +0 -8
- data/test/test_helper.rb +0 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a58e4a794e52d951be2a80bc19277fc1272da17b
|
4
|
+
data.tar.gz: b5f596a5a4b08abe49ec27bacbf6a860dd2fbfd7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: abfd5cfdc4878e92cb5529b7c1efdc3d6505d414f072577dbc008596e7a9fc8199bb9e836a66423f008950d33ac62efe16258ff0f671f523a7b4eeb31a6eaf12
|
7
|
+
data.tar.gz: 8376106e8272e6a382dd98b77e987bcebe55a3f362388dbf099e202318e7f55115f29a1a87d5c7cf41fbecbef119c08b3d59c837ea477ab746a708554ca1c203
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "fluent/plugin/mysql/appender"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -1,7 +1,11 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'fluent/plugin/mysql/appender/version'
|
5
|
+
|
2
6
|
Gem::Specification.new do |spec|
|
3
7
|
spec.name = "fluent-plugin-mysql-appender"
|
4
|
-
spec.version =
|
8
|
+
spec.version = Fluent::Plugin::Mysql::Appender::VERSION
|
5
9
|
spec.authors = ["tsuyoshi_terasaki"]
|
6
10
|
spec.email = ["tsuyoshi_terasaki@realworld.jp"]
|
7
11
|
|
@@ -10,16 +14,17 @@ Gem::Specification.new do |spec|
|
|
10
14
|
spec.homepage = "https://github.com/rw-hub/fluent-plugin-mysql-appender"
|
11
15
|
spec.license = "MIT"
|
12
16
|
|
13
|
-
spec.files = `git ls-files`.split(
|
14
|
-
|
15
|
-
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
16
22
|
spec.require_paths = ["lib"]
|
17
23
|
|
18
|
-
spec.add_runtime_dependency "fluentd"
|
19
|
-
spec.add_runtime_dependency "mysql2"
|
20
|
-
|
21
24
|
spec.add_development_dependency "bundler", "~> 1.13"
|
22
25
|
spec.add_development_dependency "rake", "~> 10.0"
|
23
26
|
spec.add_development_dependency "minitest", "~> 5.0"
|
24
27
|
|
28
|
+
spec.add_runtime_dependency "fluentd"
|
29
|
+
spec.add_runtime_dependency "mysql2"
|
25
30
|
end
|
@@ -0,0 +1,130 @@
|
|
1
|
+
require "fluent/plugin/mysql/appender/version"
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
module Plugin
|
5
|
+
module Mysql
|
6
|
+
module Appender
|
7
|
+
class MysqlAppenderInput < Fluent::Input
|
8
|
+
Plugin.register_input('mysql_appender', self)
|
9
|
+
|
10
|
+
# Define `router` method to support v0.10.57 or earlier
|
11
|
+
unless method_defined?(:router)
|
12
|
+
define_method("router") { Engine }
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
require 'mysql2'
|
17
|
+
super
|
18
|
+
end
|
19
|
+
|
20
|
+
config_param :host, :string, :default => 'localhost'
|
21
|
+
config_param :port, :integer, :default => 3306
|
22
|
+
config_param :username, :string, :default => 'root'
|
23
|
+
config_param :password, :string, :default => nil, :secret => true
|
24
|
+
config_param :database, :string, :default => nil
|
25
|
+
config_param :encoding, :string, :default => 'utf8'
|
26
|
+
config_param :query, :string
|
27
|
+
config_param :primary_key, :string, :default => 'id'
|
28
|
+
config_param :interval, :string, :default => '1m'
|
29
|
+
config_param :tag, :string, :default => nil
|
30
|
+
|
31
|
+
def configure(conf)
|
32
|
+
super
|
33
|
+
@interval = Config.time_value(@interval)
|
34
|
+
|
35
|
+
if @tag.nil?
|
36
|
+
raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
|
37
|
+
end
|
38
|
+
|
39
|
+
$log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
|
40
|
+
end
|
41
|
+
|
42
|
+
def start
|
43
|
+
@thread = Thread.new(&method(:run))
|
44
|
+
end
|
45
|
+
|
46
|
+
def shutdown
|
47
|
+
Thread.kill(@thread)
|
48
|
+
end
|
49
|
+
|
50
|
+
def run
|
51
|
+
begin
|
52
|
+
poll
|
53
|
+
rescue StandardError => e
|
54
|
+
$log.error "mysql_appender: failed to execute query."
|
55
|
+
$log.error "error: #{e.message}"
|
56
|
+
$log.error e.backtrace.join("\n")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def poll
|
61
|
+
con = get_connection()
|
62
|
+
max_id = -1
|
63
|
+
loop do
|
64
|
+
rows_count = 0
|
65
|
+
start_time = Time.now
|
66
|
+
if max_id == -1
|
67
|
+
select_query = @query + " order by #{primary_key} asc"
|
68
|
+
else
|
69
|
+
select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
|
70
|
+
end
|
71
|
+
rows, con = query(select_query, con)
|
72
|
+
rows.each_with_index do |row, index|
|
73
|
+
tag = format_tag(@tag, {:event => :insert})
|
74
|
+
router.emit(tag, Engine.now, row)
|
75
|
+
rows_count += 1
|
76
|
+
if index == rows.size - 1
|
77
|
+
max_id = row[@primary_key]
|
78
|
+
end
|
79
|
+
end
|
80
|
+
con.close
|
81
|
+
elapsed_time = sprintf("%0.02f", Time.now - start_time)
|
82
|
+
$log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
|
83
|
+
sleep @interval
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def format_tag(tag, param)
|
88
|
+
pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
|
89
|
+
tag.gsub(/(\${[a-z_]+})/) do
|
90
|
+
$log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
|
91
|
+
pattern[$1]
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def query(query, con = nil)
|
96
|
+
begin
|
97
|
+
con = con.nil? ? get_connection : con
|
98
|
+
con = con.ping ? con : get_connection
|
99
|
+
return con.query(query), con
|
100
|
+
rescue Exception => e
|
101
|
+
$log.warn "mysql_appender: #{e}"
|
102
|
+
sleep @interval
|
103
|
+
retry
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def get_connection
|
108
|
+
begin
|
109
|
+
return Mysql2::Client.new({
|
110
|
+
:host => @host,
|
111
|
+
:port => @port,
|
112
|
+
:username => @username,
|
113
|
+
:password => @password,
|
114
|
+
:database => @database,
|
115
|
+
:encoding => @encoding,
|
116
|
+
:reconnect => true,
|
117
|
+
:stream => true,
|
118
|
+
:cache_rows => false
|
119
|
+
})
|
120
|
+
rescue Exception => e
|
121
|
+
$log.warn "mysql_appender: #{e}"
|
122
|
+
sleep @interval
|
123
|
+
retry
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
metadata
CHANGED
@@ -1,85 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-appender
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- tsuyoshi_terasaki
|
8
8
|
autorequire:
|
9
|
-
bindir:
|
9
|
+
bindir: exe
|
10
10
|
cert_chain: []
|
11
11
|
date: 2016-11-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
20
|
-
type: :
|
19
|
+
version: '1.13'
|
20
|
+
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '1.13'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
40
|
+
version: '10.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: minitest
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '5.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '5.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: fluentd
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
62
|
-
type: :
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: mysql2
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
76
|
-
type: :
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '0'
|
83
83
|
description: Simple incremental id's insert.
|
84
84
|
email:
|
85
85
|
- tsuyoshi_terasaki@realworld.jp
|
@@ -93,10 +93,11 @@ files:
|
|
93
93
|
- LICENSE.txt
|
94
94
|
- README.md
|
95
95
|
- Rakefile
|
96
|
+
- bin/console
|
97
|
+
- bin/setup
|
96
98
|
- fluent-plugin-mysql-appender.gemspec
|
97
|
-
- lib/fluent/plugin/
|
98
|
-
-
|
99
|
-
- test/test_helper.rb
|
99
|
+
- lib/fluent/plugin/mysql/appender.rb
|
100
|
+
- lib/fluent/plugin/mysql/appender/version.rb
|
100
101
|
homepage: https://github.com/rw-hub/fluent-plugin-mysql-appender
|
101
102
|
licenses:
|
102
103
|
- MIT
|
@@ -121,6 +122,4 @@ rubygems_version: 2.5.1
|
|
121
122
|
signing_key:
|
122
123
|
specification_version: 4
|
123
124
|
summary: Fluentd input plugin to track insert event from MySQL database server.
|
124
|
-
test_files:
|
125
|
-
- test/fluent/plugin/in_mysql_appender_test.rb
|
126
|
-
- test/test_helper.rb
|
125
|
+
test_files: []
|
@@ -1,122 +0,0 @@
|
|
1
|
-
module Fluent
|
2
|
-
class MysqlAppenderInput < Fluent::Input
|
3
|
-
Plugin.register_input('mysql_appender', self)
|
4
|
-
|
5
|
-
# Define `router` method to support v0.10.57 or earlier
|
6
|
-
unless method_defined?(:router)
|
7
|
-
define_method("router") { Engine }
|
8
|
-
end
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
require 'mysql2'
|
12
|
-
super
|
13
|
-
end
|
14
|
-
|
15
|
-
config_param :host, :string, :default => 'localhost'
|
16
|
-
config_param :port, :integer, :default => 3306
|
17
|
-
config_param :username, :string, :default => 'root'
|
18
|
-
config_param :password, :string, :default => nil, :secret => true
|
19
|
-
config_param :database, :string, :default => nil
|
20
|
-
config_param :encoding, :string, :default => 'utf8'
|
21
|
-
config_param :query, :string
|
22
|
-
config_param :primary_key, :string, :default => 'id'
|
23
|
-
config_param :interval, :string, :default => '1m'
|
24
|
-
config_param :tag, :string, :default => nil
|
25
|
-
|
26
|
-
def configure(conf)
|
27
|
-
super
|
28
|
-
@interval = Config.time_value(@interval)
|
29
|
-
|
30
|
-
if @tag.nil?
|
31
|
-
raise Fluent::ConfigError, "mysql_appender: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
|
32
|
-
end
|
33
|
-
|
34
|
-
$log.info "adding mysql_appender worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec"
|
35
|
-
end
|
36
|
-
|
37
|
-
def start
|
38
|
-
@thread = Thread.new(&method(:run))
|
39
|
-
end
|
40
|
-
|
41
|
-
def shutdown
|
42
|
-
Thread.kill(@thread)
|
43
|
-
end
|
44
|
-
|
45
|
-
def run
|
46
|
-
begin
|
47
|
-
poll
|
48
|
-
rescue StandardError => e
|
49
|
-
$log.error "mysql_appender: failed to execute query."
|
50
|
-
$log.error "error: #{e.message}"
|
51
|
-
$log.error e.backtrace.join("\n")
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def poll
|
56
|
-
con = get_connection()
|
57
|
-
max_id = -1
|
58
|
-
loop do
|
59
|
-
rows_count = 0
|
60
|
-
start_time = Time.now
|
61
|
-
if max_id == -1
|
62
|
-
select_query = @query + " order by #{primary_key} asc"
|
63
|
-
else
|
64
|
-
select_query = @query + " where #{primary_key} > #{max_id} order by #{primary_key} asc"
|
65
|
-
end
|
66
|
-
rows, con = query(select_query, con)
|
67
|
-
rows.each_with_index do |row, index|
|
68
|
-
tag = format_tag(@tag, {:event => :insert})
|
69
|
-
router.emit(tag, Engine.now, row)
|
70
|
-
rows_count += 1
|
71
|
-
if index == rows.size - 1
|
72
|
-
max_id = row[@primary_key]
|
73
|
-
end
|
74
|
-
end
|
75
|
-
con.close
|
76
|
-
elapsed_time = sprintf("%0.02f", Time.now - start_time)
|
77
|
-
$log.info "mysql_appender: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
|
78
|
-
sleep @interval
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
def format_tag(tag, param)
|
83
|
-
pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
|
84
|
-
tag.gsub(/(\${[a-z_]+})/) do
|
85
|
-
$log.warn "mysql_appender: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
|
86
|
-
pattern[$1]
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def query(query, con = nil)
|
91
|
-
begin
|
92
|
-
con = con.nil? ? get_connection : con
|
93
|
-
con = con.ping ? con : get_connection
|
94
|
-
return con.query(query), con
|
95
|
-
rescue Exception => e
|
96
|
-
$log.warn "mysql_appender: #{e}"
|
97
|
-
sleep @interval
|
98
|
-
retry
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def get_connection
|
103
|
-
begin
|
104
|
-
return Mysql2::Client.new({
|
105
|
-
:host => @host,
|
106
|
-
:port => @port,
|
107
|
-
:username => @username,
|
108
|
-
:password => @password,
|
109
|
-
:database => @database,
|
110
|
-
:encoding => @encoding,
|
111
|
-
:reconnect => true,
|
112
|
-
:stream => true,
|
113
|
-
:cache_rows => false
|
114
|
-
})
|
115
|
-
rescue Exception => e
|
116
|
-
$log.warn "mysql_appender: #{e}"
|
117
|
-
sleep @interval
|
118
|
-
retry
|
119
|
-
end
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
data/test/test_helper.rb
DELETED