fluent-plugin-mysql-replicator 0.5.2 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: ffa1646346a6d10f15cc8837a0551ecdcddfd0c0
4
+ data.tar.gz: 5a3f350ffdeda01067e3b7ab90b35078333f9ee6
5
+ SHA512:
6
+ metadata.gz: d96f0e8226cb824bf44a14d961528cef03a3831df71a0a5a6810d68935733fbe1f940d44875ef7871f2b757845174e168bd69fd77239fad96f773ef967b99af2
7
+ data.tar.gz: 2c5886bdaa8050b70024cfeab747bdadaa6c0eb22bbb4b431aad818502969f58a3fbd23315565d41d74c4b9ee7cd9c9fca0f22681aa93c30dececa78647d5a22
@@ -1,15 +1,17 @@
1
1
  language: ruby
2
2
 
3
3
  rvm:
4
+ - 2.4.0
5
+ - 2.3.3
4
6
  - 2.2
5
7
  - 2.1
6
- - 2.0.0
7
- - 1.9.3
8
8
 
9
9
  services:
10
10
  - elasticsearch
11
11
 
12
+ before_install:
13
+ - gem update bundler
14
+
12
15
  before_script:
13
16
  - mysql < setup_mysql_replicator_multi.sql
14
17
  - curl https://raw.github.com/moliware/travis-solr/master/travis-solr.sh | SOLR_VERSION=4.6.1 bash
15
-
data/README.md CHANGED
@@ -6,22 +6,25 @@ Fluentd input plugin to track insert/update/delete event from MySQL database ser
6
6
  Not only that, it could multiple table replication into single or multi Elasticsearch/Solr.
7
7
  It's comming support replicate to another RDB/noSQL.
8
8
 
9
+ ## Requirements
10
+
11
+ | fluent-plugin-mysql-replicator | fluentd | ruby |
12
+ |--------------------|------------|--------|
13
+ | 0.6.1 | v0.14.x | >= 2.1 |
14
+ | 0.6.1 | v0.12.x | >= 1.9 |
15
+
9
16
  ## Installation
10
17
 
11
18
  install with gem or fluent-gem command as:
12
19
 
13
20
  `````
14
21
  # for system installed fluentd
15
- $ gem install fluent-plugin-mysql-replicator
22
+ $ gem install fluent-plugin-mysql-replicator -v 0.6.1
16
23
 
17
- # for td-agent
18
- $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-replicator
24
+ # for td-agent2
25
+ $ sudo td-agent-gem install fluent-plugin-mysql-replicator -v 0.6.1
19
26
  `````
20
27
 
21
- **Note:** [recommend] RPM package available which does not conflict system installed Ruby or td-agent.
22
- https://github.com/y-ken/yamabiko/releases
23
-
24
-
25
28
  ## Included plugins
26
29
 
27
30
  * Input Plugin: mysql_replicator
@@ -6,7 +6,7 @@ It is a guide to replicate multiple mysql table to elasticsearch.
6
6
 
7
7
  ```
8
8
  <source>
9
- type mysql_replicator_multi
9
+ @type mysql_replicator_multi
10
10
 
11
11
  # Database connection setting for manager table.
12
12
  manager_host localhost
@@ -22,7 +22,7 @@ It is a guide to replicate multiple mysql table to elasticsearch.
22
22
  </source>
23
23
 
24
24
  <match replicator.**>
25
- type mysql_replicator_elasticsearch
25
+ @type mysql_replicator_elasticsearch
26
26
 
27
27
  # Set Elasticsearch connection.
28
28
  host localhost
@@ -6,7 +6,7 @@ It is a guide to replicate multiple mysql table to solr.
6
6
 
7
7
  ```
8
8
  <source>
9
- type mysql_replicator_multi
9
+ @type mysql_replicator_multi
10
10
 
11
11
  # Database connection setting for manager table.
12
12
  manager_host localhost
@@ -22,7 +22,7 @@ It is a guide to replicate multiple mysql table to solr.
22
22
  </source>
23
23
 
24
24
  <match replicator.**>
25
- type mysql_replicator_solr
25
+ @type mysql_replicator_solr
26
26
 
27
27
  # Set Solr connection.
28
28
  host localhost
@@ -50,7 +50,7 @@ When you use default core (won't specify), change the value of `tag_format` like
50
50
 
51
51
  ```
52
52
  <match replicator.**>
53
- type mysql_replicator_solr
53
+ @type mysql_replicator_solr
54
54
 
55
55
  # Set Solr connection.
56
56
  host localhost
@@ -6,7 +6,7 @@ It is a guide to replicate single mysql table to elasticsearch.
6
6
 
7
7
  ```
8
8
  <source>
9
- type mysql_replicator
9
+ @type mysql_replicator
10
10
 
11
11
  # Set connection settings for replicate source.
12
12
  host localhost
@@ -31,7 +31,7 @@ It is a guide to replicate single mysql table to elasticsearch.
31
31
  </source>
32
32
 
33
33
  <match replicator.**>
34
- type mysql_replicator_elasticsearch
34
+ @type mysql_replicator_elasticsearch
35
35
 
36
36
  # Set Elasticsearch connection.
37
37
  host localhost
@@ -6,7 +6,7 @@ It is a guide to replicate single mysql table to solr.
6
6
 
7
7
  ```
8
8
  <source>
9
- type mysql_replicator
9
+ @type mysql_replicator
10
10
 
11
11
  # Set connection settings for replicate source.
12
12
  host localhost
@@ -31,7 +31,7 @@ It is a guide to replicate single mysql table to solr.
31
31
  </source>
32
32
 
33
33
  <match replicator.**>
34
- type mysql_replicator_solr
34
+ @type mysql_replicator_solr
35
35
 
36
36
  # Set Solr connection.
37
37
  host localhost
@@ -60,7 +60,7 @@ On this case, the solr url will be set `http://localhost:8983/solr`
60
60
 
61
61
  ```
62
62
  <match replicator.**>
63
- type mysql_replicator_solr
63
+ @type mysql_replicator_solr
64
64
 
65
65
  # Set Solr connection.
66
66
  host localhost
@@ -1,23 +1,25 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |s|
3
3
  s.name = "fluent-plugin-mysql-replicator"
4
- s.version = "0.5.2"
4
+ s.version = "0.6.1"
5
5
  s.authors = ["Kentaro Yoshida"]
6
6
  s.email = ["y.ken.studio@gmail.com"]
7
7
  s.homepage = "https://github.com/y-ken/fluent-plugin-mysql-replicator"
8
8
  s.summary = %q{Fluentd input plugin to track insert/update/delete event from MySQL database server. Not only that, it could multiple table replication and generate nested document for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.}
9
+ s.license = "Apache-2.0"
9
10
 
10
11
  s.files = `git ls-files`.split("\n")
11
12
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
12
13
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
13
14
  s.require_paths = ["lib"]
14
- s.license = "Apache-2.0"
15
+
16
+ s.required_ruby_version = "> 2.1"
15
17
 
16
18
  s.add_development_dependency "rake"
17
- s.add_development_dependency "webmock"
19
+ s.add_development_dependency "webmock", "~> 1.24.0"
18
20
  s.add_development_dependency "test-unit", ">= 3.1.0"
19
21
 
20
- s.add_runtime_dependency "fluentd"
22
+ s.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
21
23
  s.add_runtime_dependency "mysql2"
22
24
  s.add_runtime_dependency "rsolr"
23
25
  end
@@ -1,12 +1,9 @@
1
+ require 'fluent/input'
2
+
1
3
  module Fluent
2
4
  class MysqlReplicatorInput < Fluent::Input
3
5
  Plugin.register_input('mysql_replicator', self)
4
6
 
5
- # Define `router` method to support v0.10.57 or earlier
6
- unless method_defined?(:router)
7
- define_method("router") { Engine }
8
- end
9
-
10
7
  def initialize
11
8
  require 'mysql2'
12
9
  require 'digest/sha1'
@@ -34,7 +31,7 @@ module Fluent
34
31
  raise Fluent::ConfigError, "mysql_replicator: missing 'tag' parameter. Please add following line into config like 'tag replicator.mydatabase.mytable.${event}.${primary_key}'"
35
32
  end
36
33
 
37
- $log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
34
+ log.info "adding mysql_replicator worker. :tag=>#{tag} :query=>#{@query} :prepared_query=>#{@prepared_query} :interval=>#{@interval}sec :enable_delete=>#{enable_delete}"
38
35
  end
39
36
 
40
37
  def start
@@ -49,9 +46,9 @@ module Fluent
49
46
  begin
50
47
  poll
51
48
  rescue StandardError => e
52
- $log.error "mysql_replicator: failed to execute query."
53
- $log.error "error: #{e.message}"
54
- $log.error e.backtrace.join("\n")
49
+ log.error "mysql_replicator: failed to execute query."
50
+ log.error "error: #{e.message}"
51
+ log.error e.backtrace.join("\n")
55
52
  end
56
53
  end
57
54
 
@@ -85,7 +82,7 @@ module Fluent
85
82
  prepared_con.close
86
83
  end
87
84
  if row[@primary_key].nil?
88
- $log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
85
+ log.error "mysql_replicator: missing primary_key. :tag=>#{tag} :primary_key=>#{primary_key}"
89
86
  break
90
87
  end
91
88
  if !table_hash.include?(row[@primary_key])
@@ -101,7 +98,9 @@ module Fluent
101
98
  con.close
102
99
  ids = current_ids
103
100
  if @enable_delete
104
- if previous_ids.empty?
101
+ if current_ids.empty?
102
+ deleted_ids = Array.new
103
+ elsif previous_ids.empty?
105
104
  deleted_ids = [*1...current_ids.max] - current_ids
106
105
  else
107
106
  deleted_ids = previous_ids - current_ids
@@ -115,7 +114,7 @@ module Fluent
115
114
  end
116
115
  end
117
116
  elapsed_time = sprintf("%0.02f", Time.now - start_time)
118
- $log.info "mysql_replicator: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
117
+ log.info "mysql_replicator: finished execution :tag=>#{tag} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
119
118
  sleep @interval
120
119
  end
121
120
  end
@@ -127,13 +126,13 @@ module Fluent
127
126
  def format_tag(tag, param)
128
127
  pattern = {'${event}' => param[:event].to_s, '${primary_key}' => @primary_key}
129
128
  tag.gsub(/(\${[a-z_]+})/) do
130
- $log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
129
+ log.warn "mysql_replicator: missing placeholder. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
131
130
  pattern[$1]
132
131
  end
133
132
  end
134
133
 
135
134
  def emit_record(tag, record)
136
- router.emit(tag, Engine.now, record)
135
+ router.emit(tag, Fluent::Engine.now, record)
137
136
  end
138
137
 
139
138
  def query(query, con = nil)
@@ -142,7 +141,7 @@ module Fluent
142
141
  con = con.ping ? con : get_connection
143
142
  return con.query(query), con
144
143
  rescue Exception => e
145
- $log.warn "mysql_replicator: #{e}"
144
+ log.warn "mysql_replicator: #{e}"
146
145
  sleep @interval
147
146
  retry
148
147
  end
@@ -162,7 +161,7 @@ module Fluent
162
161
  :cache_rows => false
163
162
  })
164
163
  rescue Exception => e
165
- $log.warn "mysql_replicator: #{e}"
164
+ log.warn "mysql_replicator: #{e}"
166
165
  sleep @interval
167
166
  retry
168
167
  end
@@ -1,12 +1,9 @@
1
+ require 'fluent/input'
2
+
1
3
  module Fluent
2
4
  class MysqlReplicatorMultiInput < Fluent::Input
3
5
  Plugin.register_input('mysql_replicator_multi', self)
4
6
 
5
- # Define `router` method to support v0.10.57 or earlier
6
- unless method_defined?(:router)
7
- define_method("router") { Engine }
8
- end
9
-
10
7
  def initialize
11
8
  require 'mysql2'
12
9
  require 'digest/sha1'
@@ -46,10 +43,10 @@ module Fluent
46
43
  poll(config)
47
44
  }
48
45
  end
49
- $log.error "mysql_replicator_multi: stop working due to empty configuration" if @threads.empty?
46
+ log.error "mysql_replicator_multi: stop working due to empty configuration" if @threads.empty?
50
47
  rescue StandardError => e
51
- $log.error "error: #{e.message}"
52
- $log.error e.backtrace.join("\n")
48
+ log.error "error: #{e.message}"
49
+ log.error e.backtrace.join("\n")
53
50
  end
54
51
  end
55
52
 
@@ -73,7 +70,7 @@ module Fluent
73
70
  begin
74
71
  masked_config = Hash[config.map {|k,v| (k == 'password') ? [k, v.to_s.gsub(/./, '*')] : [k,v]}]
75
72
  @mutex.synchronize {
76
- $log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
73
+ log.info "mysql_replicator_multi: polling start. :config=>#{masked_config}"
77
74
  }
78
75
  primary_key = config['primary_key']
79
76
  previous_id = current_id = nil
@@ -99,7 +96,7 @@ module Fluent
99
96
  current_id = row[primary_key]
100
97
  @mutex.synchronize {
101
98
  if row[primary_key].nil?
102
- $log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
99
+ log.error "mysql_replicator_multi: missing primary_key. :setting_name=>#{config['name']} :primary_key=>#{primary_key}"
103
100
  break
104
101
  end
105
102
  detect_insert_update(config, row)
@@ -111,15 +108,15 @@ module Fluent
111
108
  db.close
112
109
  elapsed_time = sprintf("%0.02f", Time.now - start_time)
113
110
  @mutex.synchronize {
114
- $log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
111
+ log.info "mysql_replicator_multi: execution finished. :setting_name=>#{config['name']} :rows_count=>#{rows_count} :elapsed_time=>#{elapsed_time} sec"
115
112
  }
116
113
  sleep config['interval']
117
114
  end
118
115
  rescue StandardError => e
119
116
  @mutex.synchronize {
120
- $log.error "mysql_replicator_multi: failed to execute query. :config=>#{masked_config}"
121
- $log.error "error: #{e.message}"
122
- $log.error e.backtrace.join("\n")
117
+ log.error "mysql_replicator_multi: failed to execute query. :config=>#{masked_config}"
118
+ log.error "error: #{e.message}"
119
+ log.error e.backtrace.join("\n")
123
120
  }
124
121
  end
125
122
  end
@@ -203,7 +200,7 @@ module Fluent
203
200
  def format_tag(tag, param)
204
201
  pattern = {'${name}' => param[:name], '${event}' => param[:event].to_s, '${primary_key}' => param[:primary_key]}
205
202
  tag.gsub(/(\${[a-z_]+})/) do
206
- $log.warn "mysql_replicator_multi: unknown placeholder found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
203
+ log.warn "mysql_replicator_multi: unknown placeholder found. :tag=>#{tag} :placeholder=>#{$1}" unless pattern.include?($1)
207
204
  pattern[$1]
208
205
  end
209
206
  end
@@ -226,9 +223,9 @@ module Fluent
226
223
  end
227
224
  rescue StandardError => e
228
225
  @mutex.synchronize {
229
- $log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
230
- $log.error "error: #{e.message}"
231
- $log.error e.backtrace.join("\n")
226
+ log.error "mysql_replicator_multi: failed to flush buffered query. :config=>#{masked_config}"
227
+ log.error "error: #{e.message}"
228
+ log.error e.backtrace.join("\n")
232
229
  }
233
230
  end
234
231
  end
@@ -244,7 +241,7 @@ module Fluent
244
241
  end
245
242
 
246
243
  def emit_record(tag, record)
247
- router.emit(tag, Engine.now, record)
244
+ router.emit(tag, Fluent::Engine.now, record)
248
245
  end
249
246
 
250
247
  def get_manager_connection
metadata CHANGED
@@ -1,110 +1,103 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mysql-replicator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
5
- prerelease:
4
+ version: 0.6.1
6
5
  platform: ruby
7
6
  authors:
8
7
  - Kentaro Yoshida
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2015-11-13 00:00:00.000000000 Z
11
+ date: 2017-04-15 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rake
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - ">="
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :development
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - ">="
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: webmock
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - "~>"
36
32
  - !ruby/object:Gem::Version
37
- version: '0'
33
+ version: 1.24.0
38
34
  type: :development
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - "~>"
44
39
  - !ruby/object:Gem::Version
45
- version: '0'
40
+ version: 1.24.0
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: test-unit
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: 3.1.0
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: 3.1.0
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: fluentd
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - ">="
68
60
  - !ruby/object:Gem::Version
69
- version: '0'
61
+ version: 0.10.58
62
+ - - "<"
63
+ - !ruby/object:Gem::Version
64
+ version: '2'
70
65
  type: :runtime
71
66
  prerelease: false
72
67
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
68
  requirements:
75
- - - ! '>='
69
+ - - ">="
76
70
  - !ruby/object:Gem::Version
77
- version: '0'
71
+ version: 0.10.58
72
+ - - "<"
73
+ - !ruby/object:Gem::Version
74
+ version: '2'
78
75
  - !ruby/object:Gem::Dependency
79
76
  name: mysql2
80
77
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
78
  requirements:
83
- - - ! '>='
79
+ - - ">="
84
80
  - !ruby/object:Gem::Version
85
81
  version: '0'
86
82
  type: :runtime
87
83
  prerelease: false
88
84
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
85
  requirements:
91
- - - ! '>='
86
+ - - ">="
92
87
  - !ruby/object:Gem::Version
93
88
  version: '0'
94
89
  - !ruby/object:Gem::Dependency
95
90
  name: rsolr
96
91
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
92
  requirements:
99
- - - ! '>='
93
+ - - ">="
100
94
  - !ruby/object:Gem::Version
101
95
  version: '0'
102
96
  type: :runtime
103
97
  prerelease: false
104
98
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
99
  requirements:
107
- - - ! '>='
100
+ - - ">="
108
101
  - !ruby/object:Gem::Version
109
102
  version: '0'
110
103
  description:
@@ -114,8 +107,8 @@ executables: []
114
107
  extensions: []
115
108
  extra_rdoc_files: []
116
109
  files:
117
- - .gitignore
118
- - .travis.yml
110
+ - ".gitignore"
111
+ - ".travis.yml"
119
112
  - Gemfile
120
113
  - LICENSE
121
114
  - README.md
@@ -140,27 +133,26 @@ files:
140
133
  homepage: https://github.com/y-ken/fluent-plugin-mysql-replicator
141
134
  licenses:
142
135
  - Apache-2.0
136
+ metadata: {}
143
137
  post_install_message:
144
138
  rdoc_options: []
145
139
  require_paths:
146
140
  - lib
147
141
  required_ruby_version: !ruby/object:Gem::Requirement
148
- none: false
149
142
  requirements:
150
- - - ! '>='
143
+ - - ">"
151
144
  - !ruby/object:Gem::Version
152
- version: '0'
145
+ version: '2.1'
153
146
  required_rubygems_version: !ruby/object:Gem::Requirement
154
- none: false
155
147
  requirements:
156
- - - ! '>='
148
+ - - ">="
157
149
  - !ruby/object:Gem::Version
158
150
  version: '0'
159
151
  requirements: []
160
152
  rubyforge_project:
161
- rubygems_version: 1.8.23
153
+ rubygems_version: 2.4.5
162
154
  signing_key:
163
- specification_version: 3
155
+ specification_version: 4
164
156
  summary: Fluentd input plugin to track insert/update/delete event from MySQL database
165
157
  server. Not only that, it could multiple table replication and generate nested document
166
158
  for Elasticsearch/Solr. It's comming support replicate to another RDB/noSQL.