flydata 0.3.11 → 0.3.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +9 -4
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +1 -1
- data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb +2 -0
- data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +12 -6
- data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +25 -0
- data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -27
- data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb +128 -68
- data/spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb +56 -0
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +47 -0
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +43 -0
- metadata +7 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: df760e4cdf0cdaa8feea1af41d0441dfd0d1b25f
|
4
|
+
data.tar.gz: 48ccb962e9470ac32df552f0d7c330de57c69019
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5b9558a312db75ae958d258a59b953a646b5a577d14647b032f7d2797ee464474a8e0c6058abc1c16563b2aaaa486635e94b807c2052a861ed9b7ad798cd3ca1
|
7
|
+
data.tar.gz: 745fc312f4ab0f4aaaca698e21d9ae128281fc2681e28baddbffdbba0e486716b098047cad438f4159a029142b18ad2c520e7890cfefa991f6ecb7172ccc9e66
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.12
|
data/flydata.gemspec
CHANGED
@@ -2,14 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
+
# stub: flydata 0.3.12 ruby lib
|
5
6
|
|
6
7
|
Gem::Specification.new do |s|
|
7
8
|
s.name = "flydata"
|
8
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.12"
|
9
10
|
|
10
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
|
+
s.require_paths = ["lib"]
|
11
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2015-04-
|
14
|
+
s.date = "2015-04-08"
|
13
15
|
s.description = "FlyData Agent"
|
14
16
|
s.email = "sysadmin@flydata.com"
|
15
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -108,6 +110,7 @@ Gem::Specification.new do |s|
|
|
108
110
|
"lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb",
|
109
111
|
"lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
|
110
112
|
"lib/flydata/fluent-plugins/mysql/table_meta.rb",
|
113
|
+
"lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb",
|
111
114
|
"lib/flydata/fluent-plugins/out_forward_ssl.rb",
|
112
115
|
"lib/flydata/fluent-plugins/preference.rb",
|
113
116
|
"lib/flydata/flydata_crontab.sh",
|
@@ -152,7 +155,10 @@ Gem::Specification.new do |s|
|
|
152
155
|
"spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
|
153
156
|
"spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
|
154
157
|
"spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb",
|
158
|
+
"spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb",
|
159
|
+
"spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb",
|
155
160
|
"spec/flydata/fluent-plugins/mysql/table_meta_spec.rb",
|
161
|
+
"spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb",
|
156
162
|
"spec/flydata/heroku_spec.rb",
|
157
163
|
"spec/flydata/output/forwarder_spec.rb",
|
158
164
|
"spec/flydata/parser/mysql/alter_table_parser_spec.rb",
|
@@ -166,8 +172,7 @@ Gem::Specification.new do |s|
|
|
166
172
|
]
|
167
173
|
s.homepage = "http://flydata.com/"
|
168
174
|
s.licenses = ["All right reserved."]
|
169
|
-
s.
|
170
|
-
s.rubygems_version = "2.0.14"
|
175
|
+
s.rubygems_version = "2.2.2"
|
171
176
|
s.summary = "FlyData Agent"
|
172
177
|
|
173
178
|
if s.respond_to? :specification_version then
|
@@ -74,7 +74,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
74
74
|
@omit_events = Hash.new
|
75
75
|
@tables_append_only.split(/,\s*/).each do |table|
|
76
76
|
@tables << table unless @tables.include?(table)
|
77
|
-
@omit_events[table] = [:delete]
|
77
|
+
@omit_events[table] = [:delete, :truncate_table]
|
78
78
|
end
|
79
79
|
|
80
80
|
#Remove tables that do not have pos files
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
|
2
|
+
require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
|
2
3
|
|
3
4
|
module Mysql
|
4
5
|
class BinlogQueryDispatcher
|
@@ -55,6 +56,7 @@ module Mysql
|
|
55
56
|
def initialize(context)
|
56
57
|
@handlers = [
|
57
58
|
AlterTableQueryHandler.new(context),
|
59
|
+
TruncateTableQueryHandler.new(context),
|
58
60
|
]
|
59
61
|
end
|
60
62
|
end
|
@@ -3,19 +3,25 @@ require 'flydata/fluent-plugins/mysql/binlog_query_handler'
|
|
3
3
|
module Mysql
|
4
4
|
|
5
5
|
class DdlQueryHandler < BinlogQueryHandler
|
6
|
-
DDL_TABLE_QUERY = /^(?:(?:ALTER|CREATE|DROP|RENAME) +(?:\w+ +)*TABLE +([^ ]+)|TRUNCATE +(?:TABLE +)?([^ ]+))/i
|
6
|
+
DDL_TABLE_QUERY = /^(?:(?:ALTER|CREATE|DROP|RENAME) +(?:\w+ +)*TABLE +([^ ]+)|TRUNCATE +(?:TABLE +)?([^ ;]+))/i
|
7
7
|
|
8
8
|
def acceptable_db?(record)
|
9
|
-
|
9
|
+
supported_database == table_info(record)[:db_name]
|
10
|
+
end
|
11
|
+
|
12
|
+
def table_info(record)
|
13
|
+
table_info = { db_name: record["db_name"], table_name: nil }
|
10
14
|
if DDL_TABLE_QUERY =~ record["normalized_query"]
|
11
|
-
table_name_in_query = $1 ? $1 : $2
|
15
|
+
table_name_in_query = ($1 ? $1 : $2).tr("`", "")
|
12
16
|
|
13
17
|
if (idx = table_name_in_query.index("."))
|
14
|
-
db_name = table_name_in_query[0...idx]
|
18
|
+
table_info[:db_name] = table_name_in_query[0...idx]
|
19
|
+
table_info[:table_name] = table_name_in_query[idx+1..-1]
|
20
|
+
else
|
21
|
+
table_info[:table_name] = table_name_in_query
|
15
22
|
end
|
16
23
|
end
|
17
|
-
|
18
|
-
supported_database == db_name
|
24
|
+
table_info
|
19
25
|
end
|
20
26
|
end
|
21
27
|
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'flydata/parser_provider'
|
2
|
+
require 'flydata/fluent-plugins/mysql/ddl_query_handler'
|
3
|
+
|
4
|
+
module Mysql
|
5
|
+
class TruncateTableQueryHandler < DdlQueryHandler
|
6
|
+
PATTERN = /^TRUNCATE TABLE/i
|
7
|
+
|
8
|
+
def initialize(context)
|
9
|
+
super
|
10
|
+
end
|
11
|
+
|
12
|
+
def pattern
|
13
|
+
PATTERN
|
14
|
+
end
|
15
|
+
|
16
|
+
def process(record)
|
17
|
+
emit_record(:truncate_table, record) do |opt|
|
18
|
+
{
|
19
|
+
table_name: table_info(record)[:table_name],
|
20
|
+
query: record["query"]
|
21
|
+
}
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -1,36 +1,12 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
2
|
require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
|
3
|
+
require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
|
3
4
|
|
4
5
|
module Mysql
|
5
6
|
describe AlterTableQueryHandler do
|
6
|
-
|
7
|
-
|
8
|
-
allow(r).to receive(:get_table_binlog_pos).and_return("mysql-bin.000065\t120")
|
9
|
-
r
|
10
|
-
end
|
11
|
-
let(:database) { "testdb" }
|
12
|
-
let(:table) { "foo" }
|
13
|
-
let(:table_meta) { double'table_meta' }
|
14
|
-
let(:context) do
|
15
|
-
r = double('context')
|
16
|
-
allow(r).to receive(:sync_fm).and_return(sync_fm)
|
17
|
-
allow(r).to receive(:database).and_return([database])
|
18
|
-
allow(r).to receive(:tables).and_return([table])
|
19
|
-
allow(r).to receive(:table_meta).and_return(table_meta)
|
20
|
-
r
|
21
|
-
end
|
22
|
-
subject { described_class.new(context) }
|
7
|
+
include_context "query handler context"
|
8
|
+
|
23
9
|
describe '#process' do
|
24
|
-
let(:query) { "a_query" }
|
25
|
-
let(:normalized_query) { double('normalized_query') }
|
26
|
-
let(:record) do
|
27
|
-
r = double('record')
|
28
|
-
allow(r).to receive(:[]).with("db_name").and_return(database)
|
29
|
-
allow(r).to receive(:[]).with("query").and_return(query)
|
30
|
-
allow(r).to receive(:[]).with("table_name").and_return(table)
|
31
|
-
allow(r).to receive(:[]).with("normalized_query").and_return(normalized_query)
|
32
|
-
r
|
33
|
-
end
|
34
10
|
let(:parse_result) do
|
35
11
|
r = double('parse_result')
|
36
12
|
r
|
@@ -49,10 +49,14 @@ EOT
|
|
49
49
|
expect(r).to receive(:merge).with({"normalized_query" => normalized_query}).and_return(r)
|
50
50
|
r
|
51
51
|
end
|
52
|
-
let(:
|
53
|
-
r = double('
|
52
|
+
let(:alter_query_handler) do
|
53
|
+
r = double('alter_query_handler')
|
54
54
|
allow(r).to receive(:pattern).and_return(/^ALTER TABLE/i)
|
55
|
-
|
55
|
+
r
|
56
|
+
end
|
57
|
+
let(:truncate_query_handler) do
|
58
|
+
r = double('truncate_query_handler')
|
59
|
+
allow(r).to receive(:pattern).and_return(/^TRUNCATE TABLE/i)
|
56
60
|
r
|
57
61
|
end
|
58
62
|
let(:context) { double('context') }
|
@@ -60,117 +64,173 @@ EOT
|
|
60
64
|
|
61
65
|
shared_examples "a dispatcher that calls query handler with correct query" do
|
62
66
|
it do
|
63
|
-
expect(
|
67
|
+
expect(correct_query_handler).to receive(:process).with(record)
|
64
68
|
expect(subject.dispatch(record))
|
65
69
|
end
|
66
70
|
end
|
67
71
|
|
72
|
+
before do
|
73
|
+
expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
|
74
|
+
expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
|
75
|
+
end
|
76
|
+
|
68
77
|
describe '#dispatch' do
|
69
|
-
context "
|
70
|
-
let(:
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
78
|
+
context "alter table queries" do
|
79
|
+
let(:correct_query_handler) { alter_query_handler }
|
80
|
+
context "query with comments" do
|
81
|
+
let(:query) { QUERY_WITH_COMMENTS }
|
82
|
+
let(:normalized_query) { NORMALIZED_QUERY_WITH_COMMENTS }
|
83
|
+
|
84
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
85
|
+
end
|
86
|
+
context "customer query" do
|
87
|
+
let(:query) { CUSTOMER_QUERY }
|
88
|
+
let(:normalized_query) { NORMALIZED_CUSTOMER_QUERY }
|
89
|
+
|
90
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
91
|
+
end
|
92
|
+
context "query with leading newline" do
|
93
|
+
let(:query) { QUERY_WITH_LEADING_NEWLINE }
|
94
|
+
let(:normalized_query) { NORMALIZED_QUERY_WITH_COMMENTS }
|
95
|
+
|
96
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
97
|
+
end
|
98
|
+
context "query with special characters in DEFAULT" do
|
99
|
+
let(:query) { QUERY_WITH_SPECIAL_CHARS_IN_DEFAULT }
|
100
|
+
let(:normalized_query) { NORMALIZED_QUERY_WITH_SPECIAL_CHARS }
|
101
|
+
|
102
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
103
|
+
end
|
104
|
+
context "query with preceding commands" do
|
105
|
+
let(:query) { <<EOS }
|
91
106
|
use `mydatabase`/*!*/;
|
92
107
|
ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*application:MyApp,line:/usr/local/lib/ruby/2.0.0/benchmark.rb:281:in `measure'*/
|
93
108
|
/*!*/;
|
94
109
|
SET TIMESTAMP=1415925954/*!*/;
|
95
110
|
EOS
|
96
|
-
|
111
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
97
112
|
ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
|
98
113
|
EOS
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
114
|
+
|
115
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
116
|
+
end
|
117
|
+
context "query with a comment with no space" do
|
118
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
103
119
|
alter table test add/* adding a column */test varchar(4);
|
104
120
|
EOS
|
105
|
-
|
121
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
106
122
|
alter table test add test varchar(4);
|
107
123
|
EOS
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
124
|
+
|
125
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
126
|
+
end
|
127
|
+
context "query with strings including comment quotes" do
|
128
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
112
129
|
alter table test add test varchar(256) DEFAULT ' /* ' COMMENT ' */ -- a part of COMMENT string';
|
113
130
|
EOS
|
114
|
-
|
131
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
115
132
|
alter table test add test varchar(256) DEFAULT ' /* ' COMMENT ' */ -- a part of COMMENT string';
|
116
133
|
EOS
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
134
|
+
|
135
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
136
|
+
end
|
137
|
+
context "query with comments including string quotes and ending without a line feed" do
|
138
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
121
139
|
alter table /* /* ' */ test add test varchar(256) -- this is a comment';
|
122
140
|
EOS
|
123
|
-
|
141
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
124
142
|
alter table test add test varchar(256) ;
|
125
143
|
EOS
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
144
|
+
|
145
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
146
|
+
end
|
147
|
+
context "query with DEFAULT using double-quote" do
|
148
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
130
149
|
alter table test add test varchar(256) DEFAULT "This # is default -- with \\\" in it";
|
131
150
|
EOS
|
132
|
-
|
151
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
133
152
|
alter table test add test varchar(256) DEFAULT "This # is default -- with \\\" in it";
|
134
153
|
EOS
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
154
|
+
|
155
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
156
|
+
end
|
157
|
+
context "query with # style comment" do
|
158
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
139
159
|
alter table test add test varchar(256) # You never know this works in MySQL '"
|
140
160
|
DEFAULT "This is default with \\\" in it"
|
141
161
|
EOS
|
142
|
-
|
162
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
143
163
|
alter table test add test varchar(256) DEFAULT "This is default with \\\" in it";
|
144
164
|
EOS
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
165
|
+
|
166
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
167
|
+
end
|
168
|
+
context "query with single quote string with escape" do
|
169
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
149
170
|
alter table test add test varchar(256) COMMENT 'Stave\\'s # -- diner'
|
150
171
|
EOS
|
151
|
-
|
172
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
152
173
|
alter table test add test varchar(256) COMMENT 'Stave\\'s # -- diner';
|
153
174
|
EOS
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
175
|
+
|
176
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
177
|
+
end
|
178
|
+
context "query with quoted string with UTF8 3 byte chars" do
|
179
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
158
180
|
alter table test add test varchar(256) DEFAULT "無国籍" COMMENT 'SJIS is トラブルメーカー'
|
159
181
|
EOS
|
160
|
-
|
182
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
161
183
|
alter table test add test varchar(256) DEFAULT "無国籍" COMMENT 'SJIS is トラブルメーカー';
|
162
184
|
EOS
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
185
|
+
|
186
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
187
|
+
end
|
188
|
+
context "query with comments with UTF8 3 byte chars" do
|
189
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
167
190
|
alter table test add test varchar(256) /* 無国籍 */ -- 'SJIS is トラブルメーカー'
|
168
191
|
DEFAULT NULL;
|
169
192
|
EOS
|
170
|
-
|
193
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
171
194
|
alter table test add test varchar(256) DEFAULT NULL;
|
172
195
|
EOS
|
173
|
-
|
196
|
+
|
197
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
198
|
+
end
|
199
|
+
end
|
200
|
+
context "truncate table queries" do
|
201
|
+
let(:correct_query_handler) { truncate_query_handler }
|
202
|
+
|
203
|
+
context "simple truncate query" do
|
204
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
205
|
+
TRUNCATE table users;
|
206
|
+
EOS
|
207
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
208
|
+
TRUNCATE table users;
|
209
|
+
EOS
|
210
|
+
|
211
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
212
|
+
end
|
213
|
+
context "truncate query with dbname and table name and a comment in the beginning" do
|
214
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
215
|
+
# I am going to truncate this table
|
216
|
+
truncate table flydata_sync.users;
|
217
|
+
EOS
|
218
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
219
|
+
truncate table flydata_sync.users;
|
220
|
+
EOS
|
221
|
+
|
222
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
223
|
+
end
|
224
|
+
context "truncate query with quoted dbname and table name with comments on the same line" do
|
225
|
+
let(:query) { <<EOS.gsub(/\n$/, '') }
|
226
|
+
truncate table `flydata_sync`.`users`; /* 無国籍 */ -- 'SJIS is トラブルメーカー'
|
227
|
+
EOS
|
228
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
229
|
+
truncate table `flydata_sync`.`users`;
|
230
|
+
EOS
|
231
|
+
|
232
|
+
it_behaves_like "a dispatcher that calls query handler with correct query"
|
233
|
+
end
|
174
234
|
end
|
175
235
|
end
|
176
236
|
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/fluent-plugins/mysql/ddl_query_handler'
|
3
|
+
require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Mysql
|
6
|
+
describe DdlQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
describe "#table_info" do
|
10
|
+
context "alter table query with no db name" do
|
11
|
+
let(:normalized_query) { "ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0;" }
|
12
|
+
it do
|
13
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb", table_name: "apps"})
|
14
|
+
end
|
15
|
+
end
|
16
|
+
context "alter table query with db name" do
|
17
|
+
let(:normalized_query) { "alter table testdb1.testtable1 add testcol varchar(256) DEFAULT ' /* ' COMMENT ' */ -- a part of COMMENT string' ;" }
|
18
|
+
it do
|
19
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb1", table_name: "testtable1"})
|
20
|
+
end
|
21
|
+
end
|
22
|
+
context "alter table query with quoted db name" do
|
23
|
+
let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
|
24
|
+
alter table `testdb2`.`testtable2` add test varchar(256) DEFAULT "This # is default -- with \\\" in it";
|
25
|
+
EOS
|
26
|
+
it do
|
27
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb2", table_name: "testtable2"})
|
28
|
+
end
|
29
|
+
end
|
30
|
+
context "truncate table query with no db name" do
|
31
|
+
let(:normalized_query) { "TRUNCATE TABLE apps ;" }
|
32
|
+
it do
|
33
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb", table_name: "apps"})
|
34
|
+
end
|
35
|
+
end
|
36
|
+
context "truncate table query with db name" do
|
37
|
+
let(:normalized_query) { "truncate table `testdb1`.`apps1`;" }
|
38
|
+
it do
|
39
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb1", table_name: "apps1"})
|
40
|
+
end
|
41
|
+
end
|
42
|
+
context "truncate table query with quoted db name" do
|
43
|
+
let(:normalized_query) { "truncate table testdb2.apps2;" }
|
44
|
+
it do
|
45
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb2", table_name: "apps2"})
|
46
|
+
end
|
47
|
+
end
|
48
|
+
context "query does not match DDL_TABLE_QUERY" do #This should not happen, ideally
|
49
|
+
let(:normalized_query) { "CREATE TRIGGER mytrigger BEFORE INSERT ON TABLE_1 FOR EACH ROW SET NEW.MY_DATETIME_COLUMN = NOW();" }
|
50
|
+
it do
|
51
|
+
expect(subject.table_info(record)).to eq({db_name: "testdb", table_name: nil})
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Mysql
|
2
|
+
shared_context "query handler context" do
|
3
|
+
let(:database) { "testdb" }
|
4
|
+
let(:table) { "foo" }
|
5
|
+
let(:seq) { 200 }
|
6
|
+
let(:sync_fm) do
|
7
|
+
r = double('sync_fm')
|
8
|
+
allow(r).to receive(:get_table_binlog_pos).and_return("mysql-bin.000065\t120")
|
9
|
+
allow(r).to receive(:increment_and_save_table_position).with(table).and_yield(seq).and_return(nil)
|
10
|
+
allow(r).to receive(:delete_table_binlog_pos).with(table)
|
11
|
+
r
|
12
|
+
end
|
13
|
+
let(:table_meta) { double'table_meta' }
|
14
|
+
let(:current_binlog_file) { "mysql-bin.000066" }
|
15
|
+
let(:tag) { "some_tag" }
|
16
|
+
let(:table_rev) { 1 }
|
17
|
+
let(:context) do
|
18
|
+
r = double('context')
|
19
|
+
allow(r).to receive(:sync_fm).and_return(sync_fm)
|
20
|
+
allow(r).to receive(:database).and_return(database)
|
21
|
+
allow(r).to receive(:tables).and_return([table])
|
22
|
+
allow(r).to receive(:table_meta).and_return(table_meta)
|
23
|
+
allow(r).to receive(:current_binlog_file).and_return(current_binlog_file)
|
24
|
+
allow(r).to receive(:omit_events).and_return({})
|
25
|
+
allow(r).to receive(:table_revs).and_return({ table => table_rev})
|
26
|
+
allow(r).to receive(:tag).and_return(tag)
|
27
|
+
r
|
28
|
+
end
|
29
|
+
let(:query) { "a_query" }
|
30
|
+
let(:normalized_query) { double('normalized_query') }
|
31
|
+
let(:event_length) { 20 }
|
32
|
+
let(:next_position) { 200 }
|
33
|
+
let(:timestamp) { 1427973738 }
|
34
|
+
let(:record) do
|
35
|
+
r = double('record')
|
36
|
+
allow(r).to receive(:[]).with("db_name").and_return(database)
|
37
|
+
allow(r).to receive(:[]).with("query").and_return(query)
|
38
|
+
allow(r).to receive(:[]).with("table_name").and_return(table)
|
39
|
+
allow(r).to receive(:[]).with("normalized_query").and_return(normalized_query)
|
40
|
+
allow(r).to receive(:[]).with("next_position").and_return(next_position)
|
41
|
+
allow(r).to receive(:[]).with("event_length").and_return(event_length)
|
42
|
+
allow(r).to receive(:[]).with("timestamp").and_return("#{timestamp}")
|
43
|
+
r
|
44
|
+
end
|
45
|
+
subject { described_class.new(context) }
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
|
3
|
+
require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Mysql
|
6
|
+
describe TruncateTableQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
describe '#process' do
|
10
|
+
let(:truncate_query) { "TRUNCATE TABLE foo" }
|
11
|
+
let(:expected_record) do
|
12
|
+
{
|
13
|
+
table_name: table,
|
14
|
+
query: truncate_query,
|
15
|
+
type: :truncate_table,
|
16
|
+
respect_order: true,
|
17
|
+
src_pos: "#{current_binlog_file}\t#{next_position - event_length}",
|
18
|
+
table_rev: table_rev,
|
19
|
+
seq: seq
|
20
|
+
}
|
21
|
+
end
|
22
|
+
before do
|
23
|
+
allow(record).to receive(:[]).with("query").and_return(truncate_query)
|
24
|
+
allow(record).to receive(:[]).with("normalized_query").and_return(truncate_query)
|
25
|
+
end
|
26
|
+
context "for a non append only table" do
|
27
|
+
it "should call Fluent's emit with appropriate params" do
|
28
|
+
expect(Fluent::Engine).to receive(:emit).with(tag, timestamp, expected_record)
|
29
|
+
expect(subject.process(record))
|
30
|
+
end
|
31
|
+
end
|
32
|
+
context "for an append only table" do
|
33
|
+
before do
|
34
|
+
allow(context).to receive(:omit_events).and_return({ table => [:delete, :truncate_table] })
|
35
|
+
end
|
36
|
+
it "should not call Fluent's emit" do
|
37
|
+
expect(Fluent::Engine).to receive(:emit).never
|
38
|
+
expect(subject.process(record))
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-04-
|
15
|
+
date: 2015-04-08 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -553,6 +553,7 @@ files:
|
|
553
553
|
- lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb
|
554
554
|
- lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
|
555
555
|
- lib/flydata/fluent-plugins/mysql/table_meta.rb
|
556
|
+
- lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb
|
556
557
|
- lib/flydata/fluent-plugins/out_forward_ssl.rb
|
557
558
|
- lib/flydata/fluent-plugins/preference.rb
|
558
559
|
- lib/flydata/flydata_crontab.sh
|
@@ -597,7 +598,10 @@ files:
|
|
597
598
|
- spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
|
598
599
|
- spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
|
599
600
|
- spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
|
601
|
+
- spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb
|
602
|
+
- spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
|
600
603
|
- spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
|
604
|
+
- spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
|
601
605
|
- spec/flydata/heroku_spec.rb
|
602
606
|
- spec/flydata/output/forwarder_spec.rb
|
603
607
|
- spec/flydata/parser/mysql/alter_table_parser_spec.rb
|
@@ -628,7 +632,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
628
632
|
version: '0'
|
629
633
|
requirements: []
|
630
634
|
rubyforge_project:
|
631
|
-
rubygems_version: 2.
|
635
|
+
rubygems_version: 2.2.2
|
632
636
|
signing_key:
|
633
637
|
specification_version: 4
|
634
638
|
summary: FlyData Agent
|