flydata 0.7.4 → 0.7.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +0 -0
- data/lib/flydata/api/data_entry.rb +163 -4
- data/lib/flydata/command/sync.rb +4 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +3 -2
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb +2 -1
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb +1 -1
- data/lib/flydata/source/sync.rb +2 -2
- data/lib/flydata/source_mysql/parser/dump_parser.rb +4 -3
- data/lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb +1 -1
- data/lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb +2 -0
- data/lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb +34 -0
- data/spec/flydata/api/data_entry_spec.rb +337 -9
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -0
- data/spec/flydata/source_mysql/parser/dump_parser_spec.rb +51 -1
- data/spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb +23 -2
- data/spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb +42 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
|
4
|
+
data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
|
7
|
+
data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.5
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -10,14 +10,152 @@ module Flydata
|
|
10
10
|
super
|
11
11
|
end
|
12
12
|
|
13
|
+
# Response example (/buffer_stat.json for a Sync data entry
|
14
|
+
# {
|
15
|
+
# "complete": true,
|
16
|
+
# "state": "complete",
|
17
|
+
# "stat": {
|
18
|
+
# "buffer": {
|
19
|
+
# "proxy": {
|
20
|
+
# "buffer_size": 0,
|
21
|
+
# "queue_length": 0
|
22
|
+
# },
|
23
|
+
# "redshift": {
|
24
|
+
# "buffer_size": 0,
|
25
|
+
# "queue_length": 0
|
26
|
+
# },
|
27
|
+
# "sync_redshift": {
|
28
|
+
# "buffer_size": 0,
|
29
|
+
# "queue_length": 0
|
30
|
+
# }
|
31
|
+
# },
|
32
|
+
# "queue": {
|
33
|
+
# "copy_queue": {
|
34
|
+
# "num_items": 0
|
35
|
+
# },
|
36
|
+
# "query_queue": {
|
37
|
+
# "num_items": 0
|
38
|
+
# }
|
39
|
+
# }
|
40
|
+
# },
|
41
|
+
# "message": "",
|
42
|
+
# "success": true
|
43
|
+
# }
|
44
|
+
|
45
|
+
NUM_TABLES_PER_REQUEST = 150
|
46
|
+
|
13
47
|
def buffer_stat(data_entry_id, options = {})
|
14
|
-
|
15
|
-
|
48
|
+
table_array = options[:tables] || ['']
|
49
|
+
table_array = [''] if table_array.empty?
|
50
|
+
results = []
|
51
|
+
error = false
|
52
|
+
table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
|
53
|
+
tables = ta.join(',')
|
54
|
+
result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
|
55
|
+
unless result["state"]
|
56
|
+
error = result["message"]
|
57
|
+
break
|
58
|
+
end
|
59
|
+
results << result
|
60
|
+
end
|
61
|
+
if error
|
62
|
+
# mimicking the error value of /buffer_stat API
|
63
|
+
{"complete"=> false,
|
64
|
+
"message" => error,
|
65
|
+
"success" => true,
|
66
|
+
}
|
67
|
+
else
|
68
|
+
stat = results.inject({"buffer"=>{},"queue"=>{}}) do |h, result|
|
69
|
+
%w[redshift sync_redshift].each do |fluent_process|
|
70
|
+
%w[buffer_size queue_length].each do |stat|
|
71
|
+
if result["stat"]["buffer"][fluent_process] && result["stat"]["buffer"][fluent_process][stat]
|
72
|
+
h["buffer"][fluent_process] ||= {}
|
73
|
+
h["buffer"][fluent_process][stat] ||= 0
|
74
|
+
h["buffer"][fluent_process][stat] += result["stat"]["buffer"][fluent_process][stat]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
%w[copy_queue query_queue].each do |queue|
|
79
|
+
if result["stat"]["queue"][queue] && result["stat"]["queue"][queue]["num_items"]
|
80
|
+
h["queue"][queue] ||= {}
|
81
|
+
h["queue"][queue]["num_items"] ||= 0
|
82
|
+
h["queue"][queue]["num_items"] += result["stat"]["queue"][queue]["num_items"]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
h
|
86
|
+
end
|
87
|
+
if proxy = results.first["stat"]["buffer"]["proxy"]
|
88
|
+
# proxy values are not per-table
|
89
|
+
stat["buffer"]["proxy"] ||= {}
|
90
|
+
stat["buffer"]["proxy"]["buffer_size"] = proxy["buffer_size"]
|
91
|
+
stat["buffer"]["proxy"]["queue_length"] = proxy["queue_length"]
|
92
|
+
end
|
93
|
+
# TODO Refactor. The logic below is copied from web data_entries_controller
|
94
|
+
buffer_empty = stat["buffer"].all?{|k,v| v["buffer_size"] == 0}
|
95
|
+
queue_empty = stat["queue"].all?{|k,v| v["num_items"] == 0}
|
96
|
+
|
97
|
+
complete = (buffer_empty && queue_empty)
|
98
|
+
state = if complete
|
99
|
+
'complete'
|
100
|
+
elsif !buffer_empty
|
101
|
+
'processing'
|
102
|
+
else
|
103
|
+
'uploading'
|
104
|
+
end
|
105
|
+
message = results.count == 1 ? results.first["message"]
|
106
|
+
: build_message(stat, complete)
|
107
|
+
|
108
|
+
{
|
109
|
+
"complete"=>complete,
|
110
|
+
"state" => state,
|
111
|
+
"stat" => stat,
|
112
|
+
"message" => message,
|
113
|
+
"success" => true,
|
114
|
+
}
|
115
|
+
end
|
16
116
|
end
|
17
117
|
|
118
|
+
# Response example (/table_status.json)
|
119
|
+
# {
|
120
|
+
# "table_status": [
|
121
|
+
# {
|
122
|
+
# "updated_at": 1462586673,
|
123
|
+
# "status": "ready",
|
124
|
+
# "table_name": "items",
|
125
|
+
# "seq": 0,
|
126
|
+
# "created_at": 1462586673,
|
127
|
+
# "src_pos": "-",
|
128
|
+
# "records_copied": "0",
|
129
|
+
# "num_items": 0,
|
130
|
+
# "next_item": false
|
131
|
+
# },
|
132
|
+
# {
|
133
|
+
# "updated_at": 1462586706,
|
134
|
+
# "status": "ready",
|
135
|
+
# "table_name": "users",
|
136
|
+
# "seq": 0,
|
137
|
+
# "created_at": 1462586674,
|
138
|
+
# "src_pos": "-",
|
139
|
+
# "records_copied": "0",
|
140
|
+
# "records_sent": "8",
|
141
|
+
# "num_items": 0,
|
142
|
+
# "next_item": false
|
143
|
+
# }
|
144
|
+
# ]
|
145
|
+
# }
|
146
|
+
|
18
147
|
def table_status(data_entry_id, options = {})
|
19
|
-
|
20
|
-
|
148
|
+
table_array = options[:tables] || ['']
|
149
|
+
results = []
|
150
|
+
table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
|
151
|
+
tables = ta.join(',')
|
152
|
+
result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
|
153
|
+
results << result
|
154
|
+
end
|
155
|
+
results.inject({"table_status"=>[], "success"=>true}) do |h, result|
|
156
|
+
h["table_status"] += result["table_status"]
|
157
|
+
h
|
158
|
+
end
|
21
159
|
end
|
22
160
|
|
23
161
|
def cleanup_sync(data_entry_id, tables, options = {})
|
@@ -39,6 +177,27 @@ module Flydata
|
|
39
177
|
def complete_init_sync(data_entry_id, stats_hash)
|
40
178
|
@client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
|
41
179
|
end
|
180
|
+
|
181
|
+
private
|
182
|
+
|
183
|
+
# TODO Refactor. This code is copied from web data_entries_controller
|
184
|
+
def build_message(stat, complete)
|
185
|
+
return "" if complete
|
186
|
+
|
187
|
+
message = nil
|
188
|
+
buffer_size = stat["buffer"].inject(0) {|m, (k, v)| m + v["buffer_size"]}
|
189
|
+
chunk_num = stat["queue"].inject(0) {|m, (k, v)| m + v["num_items"]}
|
190
|
+
|
191
|
+
if buffer_size > 0
|
192
|
+
message = " -> %<buffer_size>s bytes remaining..." % {buffer_size: buffer_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
|
193
|
+
elsif chunk_num > 0
|
194
|
+
message = " -> %<chunk_num>s data chunk(s) remaining..." % {chunk_num: chunk_num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
|
195
|
+
else
|
196
|
+
message = " -> Calculating remaining bytes..."
|
197
|
+
end
|
198
|
+
|
199
|
+
message
|
200
|
+
end
|
42
201
|
end
|
43
202
|
end
|
44
203
|
end
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -266,6 +266,7 @@ EOS
|
|
266
266
|
def self.slop_repair
|
267
267
|
Slop.new do
|
268
268
|
on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
|
269
|
+
on 'skip-start', 'Skip auto start after repair is completed.'
|
269
270
|
on 'force-run', 'Run forcefully, ignoring exclusive run info'
|
270
271
|
end
|
271
272
|
end
|
@@ -274,6 +275,9 @@ EOS
|
|
274
275
|
# - Entry method
|
275
276
|
def repair
|
276
277
|
_repair
|
278
|
+
unless opts.skip_start?
|
279
|
+
Flydata::Command::Sender.new.start
|
280
|
+
end
|
277
281
|
end
|
278
282
|
run_exclusive :repair
|
279
283
|
|
@@ -42,9 +42,10 @@ module Fluent
|
|
42
42
|
positions_path = @sync_fm.table_positions_dir_path
|
43
43
|
Dir.mkdir positions_path unless File.exists? positions_path
|
44
44
|
|
45
|
-
@tables = @tables.split(/(?:\s*,\s*|\s+)/)
|
45
|
+
@tables = @tables.to_s.split(/(?:\s*,\s*|\s+)/)
|
46
46
|
@omit_events = Hash.new
|
47
|
-
@tables_append_only.split(/(?:\s*,\s*|\s+)/)
|
47
|
+
@tables_append_only = @tables_append_only.to_s.split(/(?:\s*,\s*|\s+)/)
|
48
|
+
@tables_append_only.each do |table|
|
48
49
|
@tables << table unless @tables.include?(table)
|
49
50
|
@omit_events[table] = [:delete, :truncate_table]
|
50
51
|
end
|
@@ -30,7 +30,8 @@ module Fluent
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def apply_custom_conf(conf, key, type, option)
|
33
|
-
if conf[type.to_s] and
|
33
|
+
if conf[type.to_s] and conf[type.to_s].has_key?(key.to_s)
|
34
|
+
value = conf[type.to_s][key.to_s]
|
34
35
|
var_name = option[:var_name] || key
|
35
36
|
instance_variable_set(:"@#{var_name}", value)
|
36
37
|
end
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
|
28
28
|
def start
|
29
29
|
if File.exists?(@lock_file)
|
30
|
-
$log.
|
30
|
+
$log.warn "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
|
31
31
|
@abort = true
|
32
32
|
Process.kill(:TERM, Process.ppid)
|
33
33
|
return
|
data/lib/flydata/source/sync.rb
CHANGED
@@ -57,10 +57,10 @@ class Sync < Component
|
|
57
57
|
if prefs['tables_append_only']
|
58
58
|
prefs['tables_append_only'] =
|
59
59
|
prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
|
60
|
-
prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
|
60
|
+
prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
|
61
61
|
prefs['tables_append_only']).uniq
|
62
62
|
else
|
63
|
-
prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
|
63
|
+
prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
|
64
64
|
end
|
65
65
|
prefs['invalid_tables'] =
|
66
66
|
prefs['invalid_tables'].kind_of?(String) ?
|
@@ -211,7 +211,7 @@ EOS
|
|
211
211
|
def flush_tables_with_read_lock_query(client, database, tbls)
|
212
212
|
tbls ||= []
|
213
213
|
tables = ""
|
214
|
-
if mysql_server_version(client) >=
|
214
|
+
if mysql_server_version(client).to_f >= 5.5
|
215
215
|
# FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
|
216
216
|
tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
|
217
217
|
end
|
@@ -629,13 +629,14 @@ SELECT
|
|
629
629
|
FROM
|
630
630
|
information_schema.tables
|
631
631
|
WHERE
|
632
|
-
table_schema
|
632
|
+
table_schema = '%s' AND table_name in (%s);
|
633
633
|
EOT
|
634
634
|
|
635
635
|
def initialize(de_conf)
|
636
636
|
@de_conf = de_conf
|
637
|
+
@database = de_conf['database']
|
637
638
|
@tables = de_conf['tables']
|
638
|
-
@query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
|
639
|
+
@query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
|
639
640
|
end
|
640
641
|
|
641
642
|
def get_db_bytesize
|
@@ -36,7 +36,7 @@ Received unsupported alter table query. normalized query:'#{record['normalized_q
|
|
36
36
|
EOS
|
37
37
|
$log.error(msg)
|
38
38
|
end
|
39
|
-
$log.info(ret) # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
|
39
|
+
$log.info(ret.to_s + " at \"#{binlog_pos(record)}\"") # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
|
40
40
|
ret
|
41
41
|
end
|
42
42
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
|
2
2
|
require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
|
3
3
|
require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
|
4
|
+
require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module SourceMysql
|
@@ -63,6 +64,7 @@ module PluginSupport
|
|
63
64
|
AlterTableQueryHandler.new(context),
|
64
65
|
TruncateTableQueryHandler.new(context),
|
65
66
|
DropDatabaseQueryHandler.new(context),
|
67
|
+
UnsupportedQueryHandler.new(context), # This must be a last element
|
66
68
|
]
|
67
69
|
end
|
68
70
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'flydata/parser/parser_provider'
|
2
|
+
require 'flydata/source_mysql/plugin_support/ddl_query_handler'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module SourceMysql
|
6
|
+
|
7
|
+
module PluginSupport
|
8
|
+
class UnsupportedQueryHandler < TableDdlQueryHandler
|
9
|
+
PATTERN = /.*/ # Catch all ddl
|
10
|
+
|
11
|
+
def initialize(context)
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def pattern
|
16
|
+
PATTERN
|
17
|
+
end
|
18
|
+
|
19
|
+
def process(record)
|
20
|
+
query = record['normalized_query']
|
21
|
+
if acceptable_db?(record) && should_log?(query)
|
22
|
+
$log.info({ type: :unsupported_query, query: query.to_s.strip}.to_s + " at \"#{binlog_pos(record)}\"")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def should_log?(query)
|
27
|
+
return false if query.to_s.empty?
|
28
|
+
@context.tables.any?{|tbl| query.include?(tbl)}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -2,17 +2,345 @@ require 'spec_helper'
|
|
2
2
|
require 'flydata/api/data_entry'
|
3
3
|
|
4
4
|
module Flydata
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
5
|
+
module Api
|
6
|
+
|
7
|
+
describe DataEntry do
|
8
|
+
let(:subject_object) { described_class.new(api_client) }
|
9
|
+
|
10
|
+
let(:api_client) { double('api_client') }
|
11
|
+
|
12
|
+
let(:data_entry_id) { 1234 }
|
13
|
+
let(:options) { { tables: tables } }
|
14
|
+
|
15
|
+
describe '#create' do
|
16
|
+
subject { subject_object.create(params) }
|
17
|
+
|
18
|
+
let(:params) { {data_port_id: 1, log_path: 'log-path'} }
|
19
|
+
it 'creates data entry' do
|
20
|
+
expect(api_client).to receive(:post)
|
21
|
+
|
22
|
+
subject
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_result(complete, state,
|
27
|
+
proxy_size, proxy_length, redshift_size, redshift_length,
|
28
|
+
sync_redshift_size, sync_redshift_length,
|
29
|
+
copy_num, query_num, message)
|
30
|
+
buffer = nil
|
31
|
+
queue = nil
|
32
|
+
if sync_redshift_size
|
33
|
+
buffer = { "proxy" => {
|
34
|
+
"buffer_size" => proxy_size,
|
35
|
+
"queue_length" => proxy_length,
|
36
|
+
},
|
37
|
+
"redshift" => {
|
38
|
+
"buffer_size" => redshift_size,
|
39
|
+
"queue_length" => redshift_length,
|
40
|
+
},
|
41
|
+
"sync_redshift" => {
|
42
|
+
"buffer_size" => sync_redshift_size,
|
43
|
+
"queue_length" => sync_redshift_length,
|
44
|
+
},
|
45
|
+
}
|
46
|
+
queue = { "copy_queue" => { "num_items" => copy_num },
|
47
|
+
"query_queue" => { "num_items" => query_num },
|
48
|
+
}
|
49
|
+
else
|
50
|
+
buffer = { "redshift" => {
|
51
|
+
"buffer_size" => redshift_size,
|
52
|
+
"queue_length" => redshift_length,
|
53
|
+
},
|
54
|
+
}
|
55
|
+
queue = {
|
56
|
+
"copy_queue" => { "num_items" => copy_num },
|
57
|
+
}
|
58
|
+
end
|
59
|
+
|
60
|
+
{
|
61
|
+
"complete" => complete,
|
62
|
+
"state" => state,
|
63
|
+
"stat" => { "buffer" => buffer, "queue" => queue },
|
64
|
+
"message" => message,
|
65
|
+
"success" => true
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
describe '#buffer_stat' do
|
70
|
+
subject { subject_object.buffer_stat(data_entry_id, options) }
|
71
|
+
|
72
|
+
let(:result1) {
|
73
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
74
|
+
redshift_size1, redshift_length1,
|
75
|
+
sync_redshift_size1, sync_redshift_length1,
|
76
|
+
copy_num1, query_num1, message1)
|
77
|
+
}
|
78
|
+
let(:complete1) { false }
|
79
|
+
let(:state1) { "processing" }
|
80
|
+
let(:proxy_size1) { 1 }
|
81
|
+
let(:proxy_length1) { 3 }
|
82
|
+
let(:redshift_size1) { 5 }
|
83
|
+
let(:redshift_length1) { 7 }
|
84
|
+
let(:sync_redshift_size1) { 11 }
|
85
|
+
let(:sync_redshift_length1) { 13 }
|
86
|
+
let(:copy_num1) { 17 }
|
87
|
+
let(:query_num1) { 19 }
|
88
|
+
let(:message1) { double('message1') }
|
89
|
+
let(:message2) { double('message2') }
|
90
|
+
|
91
|
+
shared_examples "calling api once and return expected result" do
|
92
|
+
it do
|
93
|
+
expect(api_client).to receive(:post).
|
94
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
95
|
+
nil, tables: expected_tables).
|
96
|
+
and_return(api_result)
|
97
|
+
|
98
|
+
expect(subject).to eq expected_result
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context "with no tables" do
|
103
|
+
let(:tables) { nil }
|
104
|
+
let(:expected_tables) { '' }
|
105
|
+
let(:api_result) { result1 }
|
106
|
+
let(:expected_result) { result1 }
|
107
|
+
let(:message) { message1 }
|
108
|
+
|
109
|
+
it_behaves_like "calling api once and return expected result"
|
110
|
+
end
|
111
|
+
context "with empty table list" do
|
112
|
+
let(:tables) { [] }
|
113
|
+
let(:expected_tables) { '' }
|
114
|
+
let(:api_result) { result1 }
|
115
|
+
let(:expected_result) { result1 }
|
116
|
+
let(:message) { message1 }
|
117
|
+
|
118
|
+
it_behaves_like "calling api once and return expected result"
|
119
|
+
end
|
120
|
+
context "with less than 100 tables" do
|
121
|
+
let(:tables) { %w[a b] }
|
122
|
+
let(:expected_tables) { "a,b" }
|
123
|
+
let(:api_result) { result1 }
|
124
|
+
let(:expected_result) { result1 }
|
125
|
+
|
126
|
+
it_behaves_like "calling api once and return expected result"
|
127
|
+
|
128
|
+
context 'when buffer is empty' do
|
129
|
+
let(:proxy_size1) { 0 }
|
130
|
+
let(:proxy_length1) { 0 }
|
131
|
+
let(:redshift_size1) { 0 }
|
132
|
+
let(:redshift_length1) { 0 }
|
133
|
+
let(:sync_redshift_size1) { 0 }
|
134
|
+
let(:sync_redshift_length1) { 0 }
|
135
|
+
|
136
|
+
let(:api_result) { result1 }
|
137
|
+
|
138
|
+
context 'when queue is empty' do
|
139
|
+
let(:copy_num1) { 0 }
|
140
|
+
let(:query_num1) { 0 }
|
141
|
+
|
142
|
+
let(:expected_result) {
|
143
|
+
build_result(true, "complete", proxy_size1, proxy_length1,
|
144
|
+
redshift_size1, redshift_length1,
|
145
|
+
sync_redshift_size1, sync_redshift_length1,
|
146
|
+
copy_num1, query_num1, message1)
|
147
|
+
}
|
148
|
+
|
149
|
+
it_behaves_like "calling api once and return expected result"
|
150
|
+
end
|
151
|
+
context 'when queue is not empty' do
|
152
|
+
let(:expected_result) {
|
153
|
+
build_result(false, "uploading", proxy_size1, proxy_length1,
|
154
|
+
redshift_size1, redshift_length1,
|
155
|
+
sync_redshift_size1, sync_redshift_length1,
|
156
|
+
copy_num1, query_num1, message1)
|
157
|
+
}
|
158
|
+
|
159
|
+
it_behaves_like "calling api once and return expected result"
|
160
|
+
end
|
161
|
+
end
|
162
|
+
context 'when buffer is not empty' do
|
163
|
+
context 'when queue is empty' do
|
164
|
+
let(:copy_num1) { 0 }
|
165
|
+
let(:query_num1) { 0 }
|
166
|
+
|
167
|
+
let(:expected_result) {
|
168
|
+
build_result(false, "processing", proxy_size1, proxy_length1,
|
169
|
+
redshift_size1, redshift_length1,
|
170
|
+
sync_redshift_size1, sync_redshift_length1,
|
171
|
+
copy_num1, query_num1, message1)
|
172
|
+
}
|
173
|
+
|
174
|
+
it_behaves_like "calling api once and return expected result"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
context "with more than 100 tables" do
|
179
|
+
let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
|
180
|
+
let(:expected_tables1) { tables[0...150].join(",") }
|
181
|
+
let(:expected_tables2) { tables[150..-1].join(",") }
|
182
|
+
|
183
|
+
let(:expected_message) { double('expected_message') }
|
184
|
+
|
185
|
+
context "when no error" do
|
186
|
+
shared_examples "calling api twice and return combined result" do
|
187
|
+
it do
|
188
|
+
expect(api_client).to receive(:post).
|
189
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
190
|
+
nil, tables: expected_tables1).
|
191
|
+
and_return(api_result1)
|
192
|
+
expect(api_client).to receive(:post).
|
193
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
194
|
+
nil, tables: expected_tables2).
|
195
|
+
and_return(api_result2)
|
196
|
+
expect(subject_object).to receive(:build_message).
|
197
|
+
and_return(expected_message)
|
198
|
+
|
199
|
+
expect(subject).to eq expected_result
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
context "with Sync entry" do
|
204
|
+
let(:api_result1) {
|
205
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
206
|
+
redshift_size1, redshift_length1,
|
207
|
+
sync_redshift_size1, sync_redshift_length1,
|
208
|
+
copy_num1, query_num1, message1)
|
209
|
+
}
|
210
|
+
let(:api_result2) {
|
211
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
212
|
+
redshift_size1, redshift_length1,
|
213
|
+
sync_redshift_size1, sync_redshift_length1,
|
214
|
+
copy_num1, query_num1, message2)
|
215
|
+
}
|
216
|
+
let(:expected_result) {
|
217
|
+
build_result(complete1, state1,
|
218
|
+
proxy_size1,
|
219
|
+
proxy_length1,
|
220
|
+
redshift_size1 + redshift_size1,
|
221
|
+
redshift_length1 + redshift_length1,
|
222
|
+
sync_redshift_size1 + sync_redshift_size1,
|
223
|
+
sync_redshift_length1 + sync_redshift_length1,
|
224
|
+
copy_num1 + copy_num1,
|
225
|
+
query_num1 + query_num1, expected_message)
|
226
|
+
}
|
227
|
+
it_behaves_like "calling api twice and return combined result"
|
228
|
+
end
|
229
|
+
context "with AutoLoad entry" do
|
230
|
+
let(:api_result1) {
|
231
|
+
build_result(complete1, state1, nil, nil,
|
232
|
+
redshift_size1, redshift_length1,
|
233
|
+
nil, nil,
|
234
|
+
copy_num1, query_num1, message1)
|
235
|
+
}
|
236
|
+
let(:api_result2) {
|
237
|
+
build_result(complete1, state1, nil, nil,
|
238
|
+
redshift_size1, redshift_length1,
|
239
|
+
nil, nil,
|
240
|
+
copy_num1, query_num1, message2)
|
241
|
+
}
|
242
|
+
let(:expected_result) {
|
243
|
+
build_result(complete1, state1,
|
244
|
+
nil,
|
245
|
+
nil,
|
246
|
+
redshift_size1 + redshift_size1,
|
247
|
+
redshift_length1 + redshift_length1,
|
248
|
+
nil,
|
249
|
+
nil,
|
250
|
+
copy_num1 + copy_num1,
|
251
|
+
query_num1 + query_num1, expected_message)
|
252
|
+
}
|
253
|
+
it_behaves_like "calling api twice and return combined result"
|
254
|
+
end
|
255
|
+
end
|
256
|
+
context "with an error with the first api call" do
|
257
|
+
let(:api_result1) { {
|
258
|
+
"complete" => false,
|
259
|
+
"message" => expected_message,
|
260
|
+
"success" => true,
|
261
|
+
} }
|
262
|
+
let(:api_result2) { result1 }
|
263
|
+
let(:expected_result) { api_result1 }
|
264
|
+
it do
|
265
|
+
expect(api_client).to receive(:post).
|
266
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
267
|
+
nil, tables: expected_tables1).
|
268
|
+
and_return(api_result1)
|
269
|
+
expect(subject_object).not_to receive(:build_message)
|
270
|
+
|
271
|
+
expect(subject).to eq expected_result
|
14
272
|
end
|
15
273
|
end
|
16
274
|
end
|
17
275
|
end
|
276
|
+
|
277
|
+
describe '#table_status' do
|
278
|
+
subject { subject_object.table_status(data_entry_id, options) }
|
279
|
+
|
280
|
+
let(:result1) {
|
281
|
+
{ "table_status" => [ status11, status12 ], "success" => true }
|
282
|
+
}
|
283
|
+
let(:status11) { double('status11') }
|
284
|
+
let(:status12) { double('status12') }
|
285
|
+
|
286
|
+
shared_examples "calling api with tables and return expected result" do
|
287
|
+
it do
|
288
|
+
expect(api_client).to receive(:post).
|
289
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
290
|
+
nil, tables: expected_tables).
|
291
|
+
and_return(api_result)
|
292
|
+
|
293
|
+
expect(subject).to eq expected_result
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
context "with no tables" do
|
298
|
+
let(:tables) { nil }
|
299
|
+
let(:expected_tables) { '' }
|
300
|
+
let(:api_result) { result1 }
|
301
|
+
let(:expected_result) { result1 }
|
302
|
+
|
303
|
+
it_behaves_like "calling api with tables and return expected result"
|
304
|
+
end
|
305
|
+
|
306
|
+
context "with less than 100 tables" do
|
307
|
+
let(:tables) { %w[a b] }
|
308
|
+
let(:expected_tables) { 'a,b' }
|
309
|
+
let(:api_result) { result1 }
|
310
|
+
let(:expected_result) { result1 }
|
311
|
+
|
312
|
+
it_behaves_like "calling api with tables and return expected result"
|
313
|
+
end
|
314
|
+
|
315
|
+
let(:result2) {
|
316
|
+
{ "table_status" => [ status21, status22 ], "success" => true }
|
317
|
+
}
|
318
|
+
let(:status21) { double('status21') }
|
319
|
+
let(:status22) { double('status22') }
|
320
|
+
|
321
|
+
context "with more than 100 tables" do
|
322
|
+
let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
|
323
|
+
let(:expected_tables1) { tables[0...150].join(",") }
|
324
|
+
let(:expected_tables2) { tables[150..-1].join(",") }
|
325
|
+
|
326
|
+
it do
|
327
|
+
expect(api_client).to receive(:post).
|
328
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
329
|
+
nil, tables: expected_tables1).
|
330
|
+
and_return(result1)
|
331
|
+
|
332
|
+
expect(api_client).to receive(:post).
|
333
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
334
|
+
nil, tables: expected_tables2).
|
335
|
+
and_return(result2)
|
336
|
+
|
337
|
+
expect(subject).to eq(
|
338
|
+
"table_status" => [ status11, status12, status21, status22 ],
|
339
|
+
"success" => true )
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
end
|
18
346
|
end
|
@@ -264,6 +264,10 @@ EOT
|
|
264
264
|
expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
|
265
265
|
end
|
266
266
|
|
267
|
+
it 'sets data tables_append_only as array' do
|
268
|
+
expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
|
269
|
+
end
|
270
|
+
|
267
271
|
it 'sets sync_fm' do
|
268
272
|
expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
|
269
273
|
end
|
@@ -31,10 +31,12 @@ module Flydata
|
|
31
31
|
end
|
32
32
|
|
33
33
|
describe MysqlDumpGeneratorNoMasterData do
|
34
|
+
let(:subject_object) { described_class.new(default_conf) }
|
35
|
+
|
34
36
|
let(:stderr) { double(:stderr) }
|
35
37
|
let(:wait_thr) { double(:wait_thr) }
|
36
38
|
|
37
|
-
let(:default_dump_generator) {
|
39
|
+
let(:default_dump_generator) { subject_object }
|
38
40
|
let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
|
39
41
|
let(:mysql_client) do
|
40
42
|
m = double(:mysql_client)
|
@@ -80,6 +82,54 @@ module Flydata
|
|
80
82
|
File.delete(file_path) if File.exists?(file_path)
|
81
83
|
end
|
82
84
|
end
|
85
|
+
|
86
|
+
describe '#flush_tables_with_read_lock_query' do
|
87
|
+
subject { subject_object.send(:flush_tables_with_read_lock_query,
|
88
|
+
client, database, tbls) }
|
89
|
+
|
90
|
+
let(:client) { double('client') }
|
91
|
+
let(:database) { "db1" }
|
92
|
+
let(:tbls) { ['table1'] }
|
93
|
+
|
94
|
+
before do
|
95
|
+
allow(subject_object).to receive(:mysql_server_version).
|
96
|
+
and_return(version)
|
97
|
+
end
|
98
|
+
|
99
|
+
let(:flush_tables_with_tables) { <<EOS.strip }
|
100
|
+
FLUSH TABLES `db1`.`table1` WITH READ LOCK;
|
101
|
+
EOS
|
102
|
+
let(:flush_tables_without_table) { <<EOS.strip }
|
103
|
+
FLUSH TABLES WITH READ LOCK;
|
104
|
+
EOS
|
105
|
+
|
106
|
+
context 'with MySQL 5.1' do
|
107
|
+
let(:version) { "5.1.67-log" }
|
108
|
+
it 'returns flush command without table because the syntax is only supported by MySQL 5.5 or later' do
|
109
|
+
is_expected.to eq(flush_tables_without_table)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
context 'with Debian MySQL 5.5' do
|
113
|
+
let(:version) { "5.5.24-2+deb.sury.org~precise+2-log" }
|
114
|
+
it { is_expected.to eq(flush_tables_with_tables) }
|
115
|
+
end
|
116
|
+
context 'with Amazon RDS MySQL 5.6' do
|
117
|
+
let(:version) { "5.6.19-log" }
|
118
|
+
it { is_expected.to eq(flush_tables_with_tables) }
|
119
|
+
end
|
120
|
+
context 'with MariaDB 10.0' do
|
121
|
+
let(:version) { "10.0.24-MariaDB" }
|
122
|
+
it 'returns flush command with tables because MariaDB 10.0 is compatible with MySQL 5.6' do
|
123
|
+
is_expected.to eq(flush_tables_with_tables)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
context 'with Amazon Aurora' do
|
127
|
+
let(:version) { "5.6.10" }
|
128
|
+
it do
|
129
|
+
is_expected.to eq(flush_tables_with_tables)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
83
133
|
end
|
84
134
|
end
|
85
135
|
|
@@ -64,6 +64,11 @@ EOT
|
|
64
64
|
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
|
65
65
|
r
|
66
66
|
end
|
67
|
+
let(:unsupported_query_handler) do
|
68
|
+
r = double('unsupported_query_handler')
|
69
|
+
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::UnsupportedQueryHandler::PATTERN)
|
70
|
+
r
|
71
|
+
end
|
67
72
|
let(:context) { double('context') }
|
68
73
|
let(:subject_object) { described_class.new(context) }
|
69
74
|
subject { subject_object }
|
@@ -79,6 +84,7 @@ EOT
|
|
79
84
|
expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
|
80
85
|
expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
|
81
86
|
expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
|
87
|
+
allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
|
82
88
|
end
|
83
89
|
|
84
90
|
describe '#dispatch' do
|
@@ -115,11 +121,26 @@ ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*applic
|
|
115
121
|
/*!*/;
|
116
122
|
SET TIMESTAMP=1415925954/*!*/;
|
117
123
|
EOS
|
118
|
-
let(:
|
124
|
+
let(:normalized_query_1) { 'use `mydatabase` ;' }
|
125
|
+
let(:normalized_query_2) { <<EOS.gsub(/\n$/, '') }
|
119
126
|
ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
|
120
127
|
EOS
|
128
|
+
let(:normalized_query_3) { 'SET TIMESTAMP=1415925954 ;' }
|
129
|
+
|
130
|
+
let(:record) do
|
131
|
+
r = double('record')
|
132
|
+
allow(r).to receive(:[]).with("query").and_return(query)
|
133
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_1}).and_return(r)
|
134
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_2}).and_return(r)
|
135
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_3}).and_return(r)
|
136
|
+
r
|
137
|
+
end
|
121
138
|
|
122
|
-
|
139
|
+
it do
|
140
|
+
expect(unsupported_query_handler).to receive(:process).with(record).twice
|
141
|
+
expect(alter_query_handler).to receive(:process).with(record).once
|
142
|
+
expect(subject.dispatch(record))
|
143
|
+
end
|
123
144
|
end
|
124
145
|
context "query with a comment with no space" do
|
125
146
|
let(:query) { <<EOS.gsub(/\n$/, '') }
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
|
+
describe UnsupportedQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
describe '#process' do
|
10
|
+
before do
|
11
|
+
record['normalized_query'] = normalized_query
|
12
|
+
end
|
13
|
+
|
14
|
+
context 'when the query includes a sync table' do
|
15
|
+
let(:normalized_query) { 'drop table foo;' }
|
16
|
+
it do
|
17
|
+
expect($log).to receive(:info).with("{:type=>:unsupported_query, :query=>\"drop table foo;\"} at \"mysql-bin.000066\t180\"")
|
18
|
+
subject.process(record)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'when the query does not include a sync table' do
|
23
|
+
let(:normalized_query) { 'drop table bar;' }
|
24
|
+
it do
|
25
|
+
expect($log).to receive(:info).never
|
26
|
+
subject.process(record)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
context 'when the query is issued in another database' do
|
31
|
+
let(:normalized_query) { 'drop table foo;' }
|
32
|
+
before do
|
33
|
+
record['db_name'] = 'another_db'
|
34
|
+
end
|
35
|
+
it do
|
36
|
+
expect($log).to receive(:info).never
|
37
|
+
subject.process(record)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-
|
15
|
+
date: 2016-06-02 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -731,6 +731,7 @@ files:
|
|
731
731
|
- lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
|
732
732
|
- lib/flydata/source_mysql/plugin_support/source_position_file.rb
|
733
733
|
- lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
|
734
|
+
- lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb
|
734
735
|
- lib/flydata/source_mysql/setup.rb
|
735
736
|
- lib/flydata/source_mysql/source_pos.rb
|
736
737
|
- lib/flydata/source_mysql/sync.rb
|
@@ -828,6 +829,7 @@ files:
|
|
828
829
|
- spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
|
829
830
|
- spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
|
830
831
|
- spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
|
832
|
+
- spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb
|
831
833
|
- spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
|
832
834
|
- spec/flydata/source_mysql/table_ddl_spec.rb
|
833
835
|
- spec/flydata/source_mysql/table_meta_spec.rb
|
@@ -867,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
867
869
|
version: '0'
|
868
870
|
requirements: []
|
869
871
|
rubyforge_project:
|
870
|
-
rubygems_version: 2.
|
872
|
+
rubygems_version: 2.0.14.1
|
871
873
|
signing_key:
|
872
874
|
specification_version: 4
|
873
875
|
summary: FlyData Agent
|