fluent-plugin-mysql-replicator 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -0
- data/example/mysql_multi_table_to_elasticsearch.md +7 -0
- data/example/mysql_single_table_to_elasticsearch.md +7 -0
- data/fluent-plugin-mysql-replicator.gemspec +3 -1
- data/lib/fluent/plugin/in_mysql_replicator.rb +7 -2
- data/lib/fluent/plugin/in_mysql_replicator_multi.rb +7 -2
- data/lib/fluent/plugin/out_mysql_replicator_elasticsearch.rb +9 -0
- data/test/plugin/test_out_mysql_replicator_elasticsearch.rb +32 -1
- metadata +20 -3
data/.travis.yml
CHANGED
@@ -28,6 +28,13 @@ It is a guide to replicate multiple mysql table to elasticsearch.
|
|
28
28
|
host localhost
|
29
29
|
port 9200
|
30
30
|
|
31
|
+
# You can configure to use SSL for connecting to Elasticsearch.
|
32
|
+
# ssl true
|
33
|
+
|
34
|
+
# Basic authentication credentials can be configured
|
35
|
+
# username basic_auth_username
|
36
|
+
# password basic_auth_password
|
37
|
+
|
31
38
|
# Set Elasticsearch index, type, and unique id (primary_key) from tag.
|
32
39
|
tag_format (?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
|
33
40
|
|
@@ -37,6 +37,13 @@ It is a guide to replicate single mysql table to elasticsearch.
|
|
37
37
|
host localhost
|
38
38
|
port 9200
|
39
39
|
|
40
|
+
# You can configure to use SSL for connecting to Elasticsearch.
|
41
|
+
# ssl true
|
42
|
+
|
43
|
+
# Basic authentication credentials can be configured
|
44
|
+
# username basic_auth_username
|
45
|
+
# password basic_auth_password
|
46
|
+
|
40
47
|
# Set Elasticsearch index, type, and unique id (primary_key) from tag.
|
41
48
|
tag_format (?<core_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$
|
42
49
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |s|
|
3
3
|
s.name = "fluent-plugin-mysql-replicator"
|
4
|
-
s.version = "0.
|
4
|
+
s.version = "0.5.0"
|
5
5
|
s.authors = ["Kentaro Yoshida"]
|
6
6
|
s.email = ["y.ken.studio@gmail.com"]
|
7
7
|
s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
|
@@ -11,9 +11,11 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
12
12
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
13
13
|
s.require_paths = ["lib"]
|
14
|
+
s.license = "Apache-2.0"
|
14
15
|
|
15
16
|
s.add_development_dependency "rake"
|
16
17
|
s.add_development_dependency "webmock"
|
18
|
+
s.add_development_dependency "test-unit", ">= 3.1.0"
|
17
19
|
|
18
20
|
s.add_runtime_dependency "fluentd"
|
19
21
|
s.add_runtime_dependency "mysql2"
|
@@ -2,6 +2,11 @@ module Fluent
|
|
2
2
|
class MysqlReplicatorInput < Fluent::Input
|
3
3
|
Plugin.register_input('mysql_replicator', self)
|
4
4
|
|
5
|
+
# Define `router` method to support v0.10.57 or earlier
|
6
|
+
unless method_defined?(:router)
|
7
|
+
define_method("router") { Engine }
|
8
|
+
end
|
9
|
+
|
5
10
|
def initialize
|
6
11
|
require 'mysql2'
|
7
12
|
require 'digest/sha1'
|
@@ -11,7 +16,7 @@ module Fluent
|
|
11
16
|
config_param :host, :string, :default => 'localhost'
|
12
17
|
config_param :port, :integer, :default => 3306
|
13
18
|
config_param :username, :string, :default => 'root'
|
14
|
-
config_param :password, :string, :default => nil
|
19
|
+
config_param :password, :string, :default => nil, :secret => true
|
15
20
|
config_param :database, :string, :default => nil
|
16
21
|
config_param :encoding, :string, :default => 'utf8'
|
17
22
|
config_param :query, :string
|
@@ -123,7 +128,7 @@ module Fluent
|
|
123
128
|
end
|
124
129
|
|
125
130
|
def emit_record(tag, record)
|
126
|
-
|
131
|
+
router.emit(tag, Engine.now, record)
|
127
132
|
end
|
128
133
|
|
129
134
|
def query(query, con = nil)
|
@@ -2,6 +2,11 @@ module Fluent
|
|
2
2
|
class MysqlReplicatorMultiInput < Fluent::Input
|
3
3
|
Plugin.register_input('mysql_replicator_multi', self)
|
4
4
|
|
5
|
+
# Define `router` method to support v0.10.57 or earlier
|
6
|
+
unless method_defined?(:router)
|
7
|
+
define_method("router") { Engine }
|
8
|
+
end
|
9
|
+
|
5
10
|
def initialize
|
6
11
|
require 'mysql2'
|
7
12
|
require 'digest/sha1'
|
@@ -11,7 +16,7 @@ module Fluent
|
|
11
16
|
config_param :manager_host, :string, :default => 'localhost'
|
12
17
|
config_param :manager_port, :integer, :default => 3306
|
13
18
|
config_param :manager_username, :string, :default => nil
|
14
|
-
config_param :manager_password, :string, :default => ''
|
19
|
+
config_param :manager_password, :string, :default => '', :secret => true
|
15
20
|
config_param :manager_database, :string, :default => 'replicator_manager'
|
16
21
|
config_param :tag, :string, :default => nil
|
17
22
|
|
@@ -239,7 +244,7 @@ module Fluent
|
|
239
244
|
end
|
240
245
|
|
241
246
|
def emit_record(tag, record)
|
242
|
-
|
247
|
+
router.emit(tag, Engine.now, record)
|
243
248
|
end
|
244
249
|
|
245
250
|
def get_manager_connection
|
@@ -7,6 +7,9 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
|
|
7
7
|
config_param :host, :string, :default => 'localhost'
|
8
8
|
config_param :port, :integer, :default => 9200
|
9
9
|
config_param :tag_format, :string, :default => nil
|
10
|
+
config_param :ssl, :bool, :default => false
|
11
|
+
config_param :username, :string, :default => nil
|
12
|
+
config_param :password, :string, :default => nil, :secret => true
|
10
13
|
|
11
14
|
DEFAULT_TAG_FORMAT = /(?<index_name>[^\.]+)\.(?<type_name>[^\.]+)\.(?<event>[^\.]+)\.(?<primary_key>[^\.]+)$/
|
12
15
|
|
@@ -60,7 +63,13 @@ class Fluent::MysqlReplicatorElasticsearchOutput < Fluent::BufferedOutput
|
|
60
63
|
bulk_message << ""
|
61
64
|
|
62
65
|
http = Net::HTTP.new(@host, @port.to_i)
|
66
|
+
http.use_ssl = @ssl
|
67
|
+
|
63
68
|
request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
|
69
|
+
if @username && @password
|
70
|
+
request.basic_auth(@username, @password)
|
71
|
+
end
|
72
|
+
|
64
73
|
request.body = bulk_message.join("\n")
|
65
74
|
http.request(request).value
|
66
75
|
end
|
@@ -124,4 +124,35 @@ class MysqlReplicatorElasticsearchOutput < Test::Unit::TestCase
|
|
124
124
|
driver.run
|
125
125
|
}
|
126
126
|
end
|
127
|
-
|
127
|
+
|
128
|
+
def test_writes_to_https_host
|
129
|
+
driver.configure("ssl true\n")
|
130
|
+
elastic_request = stub_elastic("https://localhost:9200/_bulk")
|
131
|
+
driver.emit(sample_record)
|
132
|
+
driver.run
|
133
|
+
assert_requested(elastic_request)
|
134
|
+
end
|
135
|
+
|
136
|
+
def test_writes_to_http_basic_auth
|
137
|
+
driver.configure(%[
|
138
|
+
username foo\n
|
139
|
+
password bar\n
|
140
|
+
])
|
141
|
+
elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
|
142
|
+
driver.emit(sample_record)
|
143
|
+
driver.run
|
144
|
+
assert_requested(elastic_request)
|
145
|
+
end
|
146
|
+
|
147
|
+
def test_writes_to_http_basic_auth_failed
|
148
|
+
driver.configure(%[
|
149
|
+
username wront_user\n
|
150
|
+
password bar\n
|
151
|
+
])
|
152
|
+
elastic_request = stub_elastic("http://foo:bar@localhost:9200/_bulk")
|
153
|
+
driver.emit(sample_record)
|
154
|
+
assert_raise(WebMock::NetConnectNotAllowedError) {
|
155
|
+
driver.run
|
156
|
+
}
|
157
|
+
end
|
158
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mysql-replicator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2015-09-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -43,6 +43,22 @@ dependencies:
|
|
43
43
|
- - ! '>='
|
44
44
|
- !ruby/object:Gem::Version
|
45
45
|
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: test-unit
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.1.0
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 3.1.0
|
46
62
|
- !ruby/object:Gem::Dependency
|
47
63
|
name: fluentd
|
48
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +138,8 @@ files:
|
|
122
138
|
- test/plugin/test_out_mysql_replicator_elasticsearch.rb
|
123
139
|
- test/plugin/test_out_mysql_replicator_solr.rb
|
124
140
|
homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
|
125
|
-
licenses:
|
141
|
+
licenses:
|
142
|
+
- Apache-2.0
|
126
143
|
post_install_message:
|
127
144
|
rdoc_options: []
|
128
145
|
require_paths:
|