flydata 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +0 -0
- data/lib/flydata/api/data_entry.rb +163 -4
- data/lib/flydata/command/sync.rb +4 -0
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +3 -2
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb +2 -1
- data/lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb +1 -1
- data/lib/flydata/source/sync.rb +2 -2
- data/lib/flydata/source_mysql/parser/dump_parser.rb +4 -3
- data/lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb +1 -1
- data/lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb +2 -0
- data/lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb +34 -0
- data/spec/flydata/api/data_entry_spec.rb +337 -9
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -0
- data/spec/flydata/source_mysql/parser/dump_parser_spec.rb +51 -1
- data/spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb +23 -2
- data/spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb +42 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
|
4
|
+
data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
|
7
|
+
data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.5
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -10,14 +10,152 @@ module Flydata
|
|
10
10
|
super
|
11
11
|
end
|
12
12
|
|
13
|
+
# Response example (/buffer_stat.json for a Sync data entry
|
14
|
+
# {
|
15
|
+
# "complete": true,
|
16
|
+
# "state": "complete",
|
17
|
+
# "stat": {
|
18
|
+
# "buffer": {
|
19
|
+
# "proxy": {
|
20
|
+
# "buffer_size": 0,
|
21
|
+
# "queue_length": 0
|
22
|
+
# },
|
23
|
+
# "redshift": {
|
24
|
+
# "buffer_size": 0,
|
25
|
+
# "queue_length": 0
|
26
|
+
# },
|
27
|
+
# "sync_redshift": {
|
28
|
+
# "buffer_size": 0,
|
29
|
+
# "queue_length": 0
|
30
|
+
# }
|
31
|
+
# },
|
32
|
+
# "queue": {
|
33
|
+
# "copy_queue": {
|
34
|
+
# "num_items": 0
|
35
|
+
# },
|
36
|
+
# "query_queue": {
|
37
|
+
# "num_items": 0
|
38
|
+
# }
|
39
|
+
# }
|
40
|
+
# },
|
41
|
+
# "message": "",
|
42
|
+
# "success": true
|
43
|
+
# }
|
44
|
+
|
45
|
+
NUM_TABLES_PER_REQUEST = 150
|
46
|
+
|
13
47
|
def buffer_stat(data_entry_id, options = {})
|
14
|
-
|
15
|
-
|
48
|
+
table_array = options[:tables] || ['']
|
49
|
+
table_array = [''] if table_array.empty?
|
50
|
+
results = []
|
51
|
+
error = false
|
52
|
+
table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
|
53
|
+
tables = ta.join(',')
|
54
|
+
result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
|
55
|
+
unless result["state"]
|
56
|
+
error = result["message"]
|
57
|
+
break
|
58
|
+
end
|
59
|
+
results << result
|
60
|
+
end
|
61
|
+
if error
|
62
|
+
# mimicking the error value of /buffer_stat API
|
63
|
+
{"complete"=> false,
|
64
|
+
"message" => error,
|
65
|
+
"success" => true,
|
66
|
+
}
|
67
|
+
else
|
68
|
+
stat = results.inject({"buffer"=>{},"queue"=>{}}) do |h, result|
|
69
|
+
%w[redshift sync_redshift].each do |fluent_process|
|
70
|
+
%w[buffer_size queue_length].each do |stat|
|
71
|
+
if result["stat"]["buffer"][fluent_process] && result["stat"]["buffer"][fluent_process][stat]
|
72
|
+
h["buffer"][fluent_process] ||= {}
|
73
|
+
h["buffer"][fluent_process][stat] ||= 0
|
74
|
+
h["buffer"][fluent_process][stat] += result["stat"]["buffer"][fluent_process][stat]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
%w[copy_queue query_queue].each do |queue|
|
79
|
+
if result["stat"]["queue"][queue] && result["stat"]["queue"][queue]["num_items"]
|
80
|
+
h["queue"][queue] ||= {}
|
81
|
+
h["queue"][queue]["num_items"] ||= 0
|
82
|
+
h["queue"][queue]["num_items"] += result["stat"]["queue"][queue]["num_items"]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
h
|
86
|
+
end
|
87
|
+
if proxy = results.first["stat"]["buffer"]["proxy"]
|
88
|
+
# proxy values are not per-table
|
89
|
+
stat["buffer"]["proxy"] ||= {}
|
90
|
+
stat["buffer"]["proxy"]["buffer_size"] = proxy["buffer_size"]
|
91
|
+
stat["buffer"]["proxy"]["queue_length"] = proxy["queue_length"]
|
92
|
+
end
|
93
|
+
# TODO Refactor. The logic below is copied from web data_entries_controller
|
94
|
+
buffer_empty = stat["buffer"].all?{|k,v| v["buffer_size"] == 0}
|
95
|
+
queue_empty = stat["queue"].all?{|k,v| v["num_items"] == 0}
|
96
|
+
|
97
|
+
complete = (buffer_empty && queue_empty)
|
98
|
+
state = if complete
|
99
|
+
'complete'
|
100
|
+
elsif !buffer_empty
|
101
|
+
'processing'
|
102
|
+
else
|
103
|
+
'uploading'
|
104
|
+
end
|
105
|
+
message = results.count == 1 ? results.first["message"]
|
106
|
+
: build_message(stat, complete)
|
107
|
+
|
108
|
+
{
|
109
|
+
"complete"=>complete,
|
110
|
+
"state" => state,
|
111
|
+
"stat" => stat,
|
112
|
+
"message" => message,
|
113
|
+
"success" => true,
|
114
|
+
}
|
115
|
+
end
|
16
116
|
end
|
17
117
|
|
118
|
+
# Response example (/table_status.json)
|
119
|
+
# {
|
120
|
+
# "table_status": [
|
121
|
+
# {
|
122
|
+
# "updated_at": 1462586673,
|
123
|
+
# "status": "ready",
|
124
|
+
# "table_name": "items",
|
125
|
+
# "seq": 0,
|
126
|
+
# "created_at": 1462586673,
|
127
|
+
# "src_pos": "-",
|
128
|
+
# "records_copied": "0",
|
129
|
+
# "num_items": 0,
|
130
|
+
# "next_item": false
|
131
|
+
# },
|
132
|
+
# {
|
133
|
+
# "updated_at": 1462586706,
|
134
|
+
# "status": "ready",
|
135
|
+
# "table_name": "users",
|
136
|
+
# "seq": 0,
|
137
|
+
# "created_at": 1462586674,
|
138
|
+
# "src_pos": "-",
|
139
|
+
# "records_copied": "0",
|
140
|
+
# "records_sent": "8",
|
141
|
+
# "num_items": 0,
|
142
|
+
# "next_item": false
|
143
|
+
# }
|
144
|
+
# ]
|
145
|
+
# }
|
146
|
+
|
18
147
|
def table_status(data_entry_id, options = {})
|
19
|
-
|
20
|
-
|
148
|
+
table_array = options[:tables] || ['']
|
149
|
+
results = []
|
150
|
+
table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
|
151
|
+
tables = ta.join(',')
|
152
|
+
result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
|
153
|
+
results << result
|
154
|
+
end
|
155
|
+
results.inject({"table_status"=>[], "success"=>true}) do |h, result|
|
156
|
+
h["table_status"] += result["table_status"]
|
157
|
+
h
|
158
|
+
end
|
21
159
|
end
|
22
160
|
|
23
161
|
def cleanup_sync(data_entry_id, tables, options = {})
|
@@ -39,6 +177,27 @@ module Flydata
|
|
39
177
|
def complete_init_sync(data_entry_id, stats_hash)
|
40
178
|
@client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
|
41
179
|
end
|
180
|
+
|
181
|
+
private
|
182
|
+
|
183
|
+
# TODO Refactor. This code is copied from web data_entries_controller
|
184
|
+
def build_message(stat, complete)
|
185
|
+
return "" if complete
|
186
|
+
|
187
|
+
message = nil
|
188
|
+
buffer_size = stat["buffer"].inject(0) {|m, (k, v)| m + v["buffer_size"]}
|
189
|
+
chunk_num = stat["queue"].inject(0) {|m, (k, v)| m + v["num_items"]}
|
190
|
+
|
191
|
+
if buffer_size > 0
|
192
|
+
message = " -> %<buffer_size>s bytes remaining..." % {buffer_size: buffer_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
|
193
|
+
elsif chunk_num > 0
|
194
|
+
message = " -> %<chunk_num>s data chunk(s) remaining..." % {chunk_num: chunk_num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
|
195
|
+
else
|
196
|
+
message = " -> Calculating remaining bytes..."
|
197
|
+
end
|
198
|
+
|
199
|
+
message
|
200
|
+
end
|
42
201
|
end
|
43
202
|
end
|
44
203
|
end
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -266,6 +266,7 @@ EOS
|
|
266
266
|
def self.slop_repair
|
267
267
|
Slop.new do
|
268
268
|
on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
|
269
|
+
on 'skip-start', 'Skip auto start after repair is completed.'
|
269
270
|
on 'force-run', 'Run forcefully, ignoring exclusive run info'
|
270
271
|
end
|
271
272
|
end
|
@@ -274,6 +275,9 @@ EOS
|
|
274
275
|
# - Entry method
|
275
276
|
def repair
|
276
277
|
_repair
|
278
|
+
unless opts.skip_start?
|
279
|
+
Flydata::Command::Sender.new.start
|
280
|
+
end
|
277
281
|
end
|
278
282
|
run_exclusive :repair
|
279
283
|
|
@@ -42,9 +42,10 @@ module Fluent
|
|
42
42
|
positions_path = @sync_fm.table_positions_dir_path
|
43
43
|
Dir.mkdir positions_path unless File.exists? positions_path
|
44
44
|
|
45
|
-
@tables = @tables.split(/(?:\s*,\s*|\s+)/)
|
45
|
+
@tables = @tables.to_s.split(/(?:\s*,\s*|\s+)/)
|
46
46
|
@omit_events = Hash.new
|
47
|
-
@tables_append_only.split(/(?:\s*,\s*|\s+)/)
|
47
|
+
@tables_append_only = @tables_append_only.to_s.split(/(?:\s*,\s*|\s+)/)
|
48
|
+
@tables_append_only.each do |table|
|
48
49
|
@tables << table unless @tables.include?(table)
|
49
50
|
@omit_events[table] = [:delete, :truncate_table]
|
50
51
|
end
|
@@ -30,7 +30,8 @@ module Fluent
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def apply_custom_conf(conf, key, type, option)
|
33
|
-
if conf[type.to_s] and
|
33
|
+
if conf[type.to_s] and conf[type.to_s].has_key?(key.to_s)
|
34
|
+
value = conf[type.to_s][key.to_s]
|
34
35
|
var_name = option[:var_name] || key
|
35
36
|
instance_variable_set(:"@#{var_name}", value)
|
36
37
|
end
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
|
28
28
|
def start
|
29
29
|
if File.exists?(@lock_file)
|
30
|
-
$log.
|
30
|
+
$log.warn "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
|
31
31
|
@abort = true
|
32
32
|
Process.kill(:TERM, Process.ppid)
|
33
33
|
return
|
data/lib/flydata/source/sync.rb
CHANGED
@@ -57,10 +57,10 @@ class Sync < Component
|
|
57
57
|
if prefs['tables_append_only']
|
58
58
|
prefs['tables_append_only'] =
|
59
59
|
prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
|
60
|
-
prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
|
60
|
+
prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
|
61
61
|
prefs['tables_append_only']).uniq
|
62
62
|
else
|
63
|
-
prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
|
63
|
+
prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
|
64
64
|
end
|
65
65
|
prefs['invalid_tables'] =
|
66
66
|
prefs['invalid_tables'].kind_of?(String) ?
|
@@ -211,7 +211,7 @@ EOS
|
|
211
211
|
def flush_tables_with_read_lock_query(client, database, tbls)
|
212
212
|
tbls ||= []
|
213
213
|
tables = ""
|
214
|
-
if mysql_server_version(client) >=
|
214
|
+
if mysql_server_version(client).to_f >= 5.5
|
215
215
|
# FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
|
216
216
|
tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
|
217
217
|
end
|
@@ -629,13 +629,14 @@ SELECT
|
|
629
629
|
FROM
|
630
630
|
information_schema.tables
|
631
631
|
WHERE
|
632
|
-
table_schema
|
632
|
+
table_schema = '%s' AND table_name in (%s);
|
633
633
|
EOT
|
634
634
|
|
635
635
|
def initialize(de_conf)
|
636
636
|
@de_conf = de_conf
|
637
|
+
@database = de_conf['database']
|
637
638
|
@tables = de_conf['tables']
|
638
|
-
@query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
|
639
|
+
@query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
|
639
640
|
end
|
640
641
|
|
641
642
|
def get_db_bytesize
|
@@ -36,7 +36,7 @@ Received unsupported alter table query. normalized query:'#{record['normalized_q
|
|
36
36
|
EOS
|
37
37
|
$log.error(msg)
|
38
38
|
end
|
39
|
-
$log.info(ret) # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
|
39
|
+
$log.info(ret.to_s + " at \"#{binlog_pos(record)}\"") # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
|
40
40
|
ret
|
41
41
|
end
|
42
42
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
|
2
2
|
require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
|
3
3
|
require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
|
4
|
+
require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module SourceMysql
|
@@ -63,6 +64,7 @@ module PluginSupport
|
|
63
64
|
AlterTableQueryHandler.new(context),
|
64
65
|
TruncateTableQueryHandler.new(context),
|
65
66
|
DropDatabaseQueryHandler.new(context),
|
67
|
+
UnsupportedQueryHandler.new(context), # This must be a last element
|
66
68
|
]
|
67
69
|
end
|
68
70
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'flydata/parser/parser_provider'
|
2
|
+
require 'flydata/source_mysql/plugin_support/ddl_query_handler'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module SourceMysql
|
6
|
+
|
7
|
+
module PluginSupport
|
8
|
+
class UnsupportedQueryHandler < TableDdlQueryHandler
|
9
|
+
PATTERN = /.*/ # Catch all ddl
|
10
|
+
|
11
|
+
def initialize(context)
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def pattern
|
16
|
+
PATTERN
|
17
|
+
end
|
18
|
+
|
19
|
+
def process(record)
|
20
|
+
query = record['normalized_query']
|
21
|
+
if acceptable_db?(record) && should_log?(query)
|
22
|
+
$log.info({ type: :unsupported_query, query: query.to_s.strip}.to_s + " at \"#{binlog_pos(record)}\"")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def should_log?(query)
|
27
|
+
return false if query.to_s.empty?
|
28
|
+
@context.tables.any?{|tbl| query.include?(tbl)}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -2,17 +2,345 @@ require 'spec_helper'
|
|
2
2
|
require 'flydata/api/data_entry'
|
3
3
|
|
4
4
|
module Flydata
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
5
|
+
module Api
|
6
|
+
|
7
|
+
describe DataEntry do
|
8
|
+
let(:subject_object) { described_class.new(api_client) }
|
9
|
+
|
10
|
+
let(:api_client) { double('api_client') }
|
11
|
+
|
12
|
+
let(:data_entry_id) { 1234 }
|
13
|
+
let(:options) { { tables: tables } }
|
14
|
+
|
15
|
+
describe '#create' do
|
16
|
+
subject { subject_object.create(params) }
|
17
|
+
|
18
|
+
let(:params) { {data_port_id: 1, log_path: 'log-path'} }
|
19
|
+
it 'creates data entry' do
|
20
|
+
expect(api_client).to receive(:post)
|
21
|
+
|
22
|
+
subject
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_result(complete, state,
|
27
|
+
proxy_size, proxy_length, redshift_size, redshift_length,
|
28
|
+
sync_redshift_size, sync_redshift_length,
|
29
|
+
copy_num, query_num, message)
|
30
|
+
buffer = nil
|
31
|
+
queue = nil
|
32
|
+
if sync_redshift_size
|
33
|
+
buffer = { "proxy" => {
|
34
|
+
"buffer_size" => proxy_size,
|
35
|
+
"queue_length" => proxy_length,
|
36
|
+
},
|
37
|
+
"redshift" => {
|
38
|
+
"buffer_size" => redshift_size,
|
39
|
+
"queue_length" => redshift_length,
|
40
|
+
},
|
41
|
+
"sync_redshift" => {
|
42
|
+
"buffer_size" => sync_redshift_size,
|
43
|
+
"queue_length" => sync_redshift_length,
|
44
|
+
},
|
45
|
+
}
|
46
|
+
queue = { "copy_queue" => { "num_items" => copy_num },
|
47
|
+
"query_queue" => { "num_items" => query_num },
|
48
|
+
}
|
49
|
+
else
|
50
|
+
buffer = { "redshift" => {
|
51
|
+
"buffer_size" => redshift_size,
|
52
|
+
"queue_length" => redshift_length,
|
53
|
+
},
|
54
|
+
}
|
55
|
+
queue = {
|
56
|
+
"copy_queue" => { "num_items" => copy_num },
|
57
|
+
}
|
58
|
+
end
|
59
|
+
|
60
|
+
{
|
61
|
+
"complete" => complete,
|
62
|
+
"state" => state,
|
63
|
+
"stat" => { "buffer" => buffer, "queue" => queue },
|
64
|
+
"message" => message,
|
65
|
+
"success" => true
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
describe '#buffer_stat' do
|
70
|
+
subject { subject_object.buffer_stat(data_entry_id, options) }
|
71
|
+
|
72
|
+
let(:result1) {
|
73
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
74
|
+
redshift_size1, redshift_length1,
|
75
|
+
sync_redshift_size1, sync_redshift_length1,
|
76
|
+
copy_num1, query_num1, message1)
|
77
|
+
}
|
78
|
+
let(:complete1) { false }
|
79
|
+
let(:state1) { "processing" }
|
80
|
+
let(:proxy_size1) { 1 }
|
81
|
+
let(:proxy_length1) { 3 }
|
82
|
+
let(:redshift_size1) { 5 }
|
83
|
+
let(:redshift_length1) { 7 }
|
84
|
+
let(:sync_redshift_size1) { 11 }
|
85
|
+
let(:sync_redshift_length1) { 13 }
|
86
|
+
let(:copy_num1) { 17 }
|
87
|
+
let(:query_num1) { 19 }
|
88
|
+
let(:message1) { double('message1') }
|
89
|
+
let(:message2) { double('message2') }
|
90
|
+
|
91
|
+
shared_examples "calling api once and return expected result" do
|
92
|
+
it do
|
93
|
+
expect(api_client).to receive(:post).
|
94
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
95
|
+
nil, tables: expected_tables).
|
96
|
+
and_return(api_result)
|
97
|
+
|
98
|
+
expect(subject).to eq expected_result
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context "with no tables" do
|
103
|
+
let(:tables) { nil }
|
104
|
+
let(:expected_tables) { '' }
|
105
|
+
let(:api_result) { result1 }
|
106
|
+
let(:expected_result) { result1 }
|
107
|
+
let(:message) { message1 }
|
108
|
+
|
109
|
+
it_behaves_like "calling api once and return expected result"
|
110
|
+
end
|
111
|
+
context "with empty table list" do
|
112
|
+
let(:tables) { [] }
|
113
|
+
let(:expected_tables) { '' }
|
114
|
+
let(:api_result) { result1 }
|
115
|
+
let(:expected_result) { result1 }
|
116
|
+
let(:message) { message1 }
|
117
|
+
|
118
|
+
it_behaves_like "calling api once and return expected result"
|
119
|
+
end
|
120
|
+
context "with less than 100 tables" do
|
121
|
+
let(:tables) { %w[a b] }
|
122
|
+
let(:expected_tables) { "a,b" }
|
123
|
+
let(:api_result) { result1 }
|
124
|
+
let(:expected_result) { result1 }
|
125
|
+
|
126
|
+
it_behaves_like "calling api once and return expected result"
|
127
|
+
|
128
|
+
context 'when buffer is empty' do
|
129
|
+
let(:proxy_size1) { 0 }
|
130
|
+
let(:proxy_length1) { 0 }
|
131
|
+
let(:redshift_size1) { 0 }
|
132
|
+
let(:redshift_length1) { 0 }
|
133
|
+
let(:sync_redshift_size1) { 0 }
|
134
|
+
let(:sync_redshift_length1) { 0 }
|
135
|
+
|
136
|
+
let(:api_result) { result1 }
|
137
|
+
|
138
|
+
context 'when queue is empty' do
|
139
|
+
let(:copy_num1) { 0 }
|
140
|
+
let(:query_num1) { 0 }
|
141
|
+
|
142
|
+
let(:expected_result) {
|
143
|
+
build_result(true, "complete", proxy_size1, proxy_length1,
|
144
|
+
redshift_size1, redshift_length1,
|
145
|
+
sync_redshift_size1, sync_redshift_length1,
|
146
|
+
copy_num1, query_num1, message1)
|
147
|
+
}
|
148
|
+
|
149
|
+
it_behaves_like "calling api once and return expected result"
|
150
|
+
end
|
151
|
+
context 'when queue is not empty' do
|
152
|
+
let(:expected_result) {
|
153
|
+
build_result(false, "uploading", proxy_size1, proxy_length1,
|
154
|
+
redshift_size1, redshift_length1,
|
155
|
+
sync_redshift_size1, sync_redshift_length1,
|
156
|
+
copy_num1, query_num1, message1)
|
157
|
+
}
|
158
|
+
|
159
|
+
it_behaves_like "calling api once and return expected result"
|
160
|
+
end
|
161
|
+
end
|
162
|
+
context 'when buffer is not empty' do
|
163
|
+
context 'when queue is empty' do
|
164
|
+
let(:copy_num1) { 0 }
|
165
|
+
let(:query_num1) { 0 }
|
166
|
+
|
167
|
+
let(:expected_result) {
|
168
|
+
build_result(false, "processing", proxy_size1, proxy_length1,
|
169
|
+
redshift_size1, redshift_length1,
|
170
|
+
sync_redshift_size1, sync_redshift_length1,
|
171
|
+
copy_num1, query_num1, message1)
|
172
|
+
}
|
173
|
+
|
174
|
+
it_behaves_like "calling api once and return expected result"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
context "with more than 100 tables" do
|
179
|
+
let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
|
180
|
+
let(:expected_tables1) { tables[0...150].join(",") }
|
181
|
+
let(:expected_tables2) { tables[150..-1].join(",") }
|
182
|
+
|
183
|
+
let(:expected_message) { double('expected_message') }
|
184
|
+
|
185
|
+
context "when no error" do
|
186
|
+
shared_examples "calling api twice and return combined result" do
|
187
|
+
it do
|
188
|
+
expect(api_client).to receive(:post).
|
189
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
190
|
+
nil, tables: expected_tables1).
|
191
|
+
and_return(api_result1)
|
192
|
+
expect(api_client).to receive(:post).
|
193
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
194
|
+
nil, tables: expected_tables2).
|
195
|
+
and_return(api_result2)
|
196
|
+
expect(subject_object).to receive(:build_message).
|
197
|
+
and_return(expected_message)
|
198
|
+
|
199
|
+
expect(subject).to eq expected_result
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
context "with Sync entry" do
|
204
|
+
let(:api_result1) {
|
205
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
206
|
+
redshift_size1, redshift_length1,
|
207
|
+
sync_redshift_size1, sync_redshift_length1,
|
208
|
+
copy_num1, query_num1, message1)
|
209
|
+
}
|
210
|
+
let(:api_result2) {
|
211
|
+
build_result(complete1, state1, proxy_size1, proxy_length1,
|
212
|
+
redshift_size1, redshift_length1,
|
213
|
+
sync_redshift_size1, sync_redshift_length1,
|
214
|
+
copy_num1, query_num1, message2)
|
215
|
+
}
|
216
|
+
let(:expected_result) {
|
217
|
+
build_result(complete1, state1,
|
218
|
+
proxy_size1,
|
219
|
+
proxy_length1,
|
220
|
+
redshift_size1 + redshift_size1,
|
221
|
+
redshift_length1 + redshift_length1,
|
222
|
+
sync_redshift_size1 + sync_redshift_size1,
|
223
|
+
sync_redshift_length1 + sync_redshift_length1,
|
224
|
+
copy_num1 + copy_num1,
|
225
|
+
query_num1 + query_num1, expected_message)
|
226
|
+
}
|
227
|
+
it_behaves_like "calling api twice and return combined result"
|
228
|
+
end
|
229
|
+
context "with AutoLoad entry" do
|
230
|
+
let(:api_result1) {
|
231
|
+
build_result(complete1, state1, nil, nil,
|
232
|
+
redshift_size1, redshift_length1,
|
233
|
+
nil, nil,
|
234
|
+
copy_num1, query_num1, message1)
|
235
|
+
}
|
236
|
+
let(:api_result2) {
|
237
|
+
build_result(complete1, state1, nil, nil,
|
238
|
+
redshift_size1, redshift_length1,
|
239
|
+
nil, nil,
|
240
|
+
copy_num1, query_num1, message2)
|
241
|
+
}
|
242
|
+
let(:expected_result) {
|
243
|
+
build_result(complete1, state1,
|
244
|
+
nil,
|
245
|
+
nil,
|
246
|
+
redshift_size1 + redshift_size1,
|
247
|
+
redshift_length1 + redshift_length1,
|
248
|
+
nil,
|
249
|
+
nil,
|
250
|
+
copy_num1 + copy_num1,
|
251
|
+
query_num1 + query_num1, expected_message)
|
252
|
+
}
|
253
|
+
it_behaves_like "calling api twice and return combined result"
|
254
|
+
end
|
255
|
+
end
|
256
|
+
context "with an error with the first api call" do
|
257
|
+
let(:api_result1) { {
|
258
|
+
"complete" => false,
|
259
|
+
"message" => expected_message,
|
260
|
+
"success" => true,
|
261
|
+
} }
|
262
|
+
let(:api_result2) { result1 }
|
263
|
+
let(:expected_result) { api_result1 }
|
264
|
+
it do
|
265
|
+
expect(api_client).to receive(:post).
|
266
|
+
with("/data_entries/#{data_entry_id}/buffer_stat/",
|
267
|
+
nil, tables: expected_tables1).
|
268
|
+
and_return(api_result1)
|
269
|
+
expect(subject_object).not_to receive(:build_message)
|
270
|
+
|
271
|
+
expect(subject).to eq expected_result
|
14
272
|
end
|
15
273
|
end
|
16
274
|
end
|
17
275
|
end
|
276
|
+
|
277
|
+
describe '#table_status' do
|
278
|
+
subject { subject_object.table_status(data_entry_id, options) }
|
279
|
+
|
280
|
+
let(:result1) {
|
281
|
+
{ "table_status" => [ status11, status12 ], "success" => true }
|
282
|
+
}
|
283
|
+
let(:status11) { double('status11') }
|
284
|
+
let(:status12) { double('status12') }
|
285
|
+
|
286
|
+
shared_examples "calling api with tables and return expected result" do
|
287
|
+
it do
|
288
|
+
expect(api_client).to receive(:post).
|
289
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
290
|
+
nil, tables: expected_tables).
|
291
|
+
and_return(api_result)
|
292
|
+
|
293
|
+
expect(subject).to eq expected_result
|
294
|
+
end
|
295
|
+
end
|
296
|
+
|
297
|
+
context "with no tables" do
|
298
|
+
let(:tables) { nil }
|
299
|
+
let(:expected_tables) { '' }
|
300
|
+
let(:api_result) { result1 }
|
301
|
+
let(:expected_result) { result1 }
|
302
|
+
|
303
|
+
it_behaves_like "calling api with tables and return expected result"
|
304
|
+
end
|
305
|
+
|
306
|
+
context "with less than 100 tables" do
|
307
|
+
let(:tables) { %w[a b] }
|
308
|
+
let(:expected_tables) { 'a,b' }
|
309
|
+
let(:api_result) { result1 }
|
310
|
+
let(:expected_result) { result1 }
|
311
|
+
|
312
|
+
it_behaves_like "calling api with tables and return expected result"
|
313
|
+
end
|
314
|
+
|
315
|
+
let(:result2) {
|
316
|
+
{ "table_status" => [ status21, status22 ], "success" => true }
|
317
|
+
}
|
318
|
+
let(:status21) { double('status21') }
|
319
|
+
let(:status22) { double('status22') }
|
320
|
+
|
321
|
+
context "with more than 100 tables" do
|
322
|
+
let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
|
323
|
+
let(:expected_tables1) { tables[0...150].join(",") }
|
324
|
+
let(:expected_tables2) { tables[150..-1].join(",") }
|
325
|
+
|
326
|
+
it do
|
327
|
+
expect(api_client).to receive(:post).
|
328
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
329
|
+
nil, tables: expected_tables1).
|
330
|
+
and_return(result1)
|
331
|
+
|
332
|
+
expect(api_client).to receive(:post).
|
333
|
+
with("/data_entries/#{data_entry_id}/table_status",
|
334
|
+
nil, tables: expected_tables2).
|
335
|
+
and_return(result2)
|
336
|
+
|
337
|
+
expect(subject).to eq(
|
338
|
+
"table_status" => [ status11, status12, status21, status22 ],
|
339
|
+
"success" => true )
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
end
|
18
346
|
end
|
@@ -264,6 +264,10 @@ EOT
|
|
264
264
|
expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
|
265
265
|
end
|
266
266
|
|
267
|
+
it 'sets data tables_append_only as array' do
|
268
|
+
expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
|
269
|
+
end
|
270
|
+
|
267
271
|
it 'sets sync_fm' do
|
268
272
|
expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
|
269
273
|
end
|
@@ -31,10 +31,12 @@ module Flydata
|
|
31
31
|
end
|
32
32
|
|
33
33
|
describe MysqlDumpGeneratorNoMasterData do
|
34
|
+
let(:subject_object) { described_class.new(default_conf) }
|
35
|
+
|
34
36
|
let(:stderr) { double(:stderr) }
|
35
37
|
let(:wait_thr) { double(:wait_thr) }
|
36
38
|
|
37
|
-
let(:default_dump_generator) {
|
39
|
+
let(:default_dump_generator) { subject_object }
|
38
40
|
let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
|
39
41
|
let(:mysql_client) do
|
40
42
|
m = double(:mysql_client)
|
@@ -80,6 +82,54 @@ module Flydata
|
|
80
82
|
File.delete(file_path) if File.exists?(file_path)
|
81
83
|
end
|
82
84
|
end
|
85
|
+
|
86
|
+
describe '#flush_tables_with_read_lock_query' do
|
87
|
+
subject { subject_object.send(:flush_tables_with_read_lock_query,
|
88
|
+
client, database, tbls) }
|
89
|
+
|
90
|
+
let(:client) { double('client') }
|
91
|
+
let(:database) { "db1" }
|
92
|
+
let(:tbls) { ['table1'] }
|
93
|
+
|
94
|
+
before do
|
95
|
+
allow(subject_object).to receive(:mysql_server_version).
|
96
|
+
and_return(version)
|
97
|
+
end
|
98
|
+
|
99
|
+
let(:flush_tables_with_tables) { <<EOS.strip }
|
100
|
+
FLUSH TABLES `db1`.`table1` WITH READ LOCK;
|
101
|
+
EOS
|
102
|
+
let(:flush_tables_without_table) { <<EOS.strip }
|
103
|
+
FLUSH TABLES WITH READ LOCK;
|
104
|
+
EOS
|
105
|
+
|
106
|
+
context 'with MySQL 5.1' do
|
107
|
+
let(:version) { "5.1.67-log" }
|
108
|
+
it 'returns flush command without table because the syntax is only supported by MySQL 5.5 or later' do
|
109
|
+
is_expected.to eq(flush_tables_without_table)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
context 'with Debian MySQL 5.5' do
|
113
|
+
let(:version) { "5.5.24-2+deb.sury.org~precise+2-log" }
|
114
|
+
it { is_expected.to eq(flush_tables_with_tables) }
|
115
|
+
end
|
116
|
+
context 'with Amazon RDS MySQL 5.6' do
|
117
|
+
let(:version) { "5.6.19-log" }
|
118
|
+
it { is_expected.to eq(flush_tables_with_tables) }
|
119
|
+
end
|
120
|
+
context 'with MariaDB 10.0' do
|
121
|
+
let(:version) { "10.0.24-MariaDB" }
|
122
|
+
it 'returns flush command with tables because MariaDB 10.0 is compatible with MySQL 5.6' do
|
123
|
+
is_expected.to eq(flush_tables_with_tables)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
context 'with Amazon Aurora' do
|
127
|
+
let(:version) { "5.6.10" }
|
128
|
+
it do
|
129
|
+
is_expected.to eq(flush_tables_with_tables)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
83
133
|
end
|
84
134
|
end
|
85
135
|
|
@@ -64,6 +64,11 @@ EOT
|
|
64
64
|
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
|
65
65
|
r
|
66
66
|
end
|
67
|
+
let(:unsupported_query_handler) do
|
68
|
+
r = double('unsupported_query_handler')
|
69
|
+
allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::UnsupportedQueryHandler::PATTERN)
|
70
|
+
r
|
71
|
+
end
|
67
72
|
let(:context) { double('context') }
|
68
73
|
let(:subject_object) { described_class.new(context) }
|
69
74
|
subject { subject_object }
|
@@ -79,6 +84,7 @@ EOT
|
|
79
84
|
expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
|
80
85
|
expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
|
81
86
|
expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
|
87
|
+
allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
|
82
88
|
end
|
83
89
|
|
84
90
|
describe '#dispatch' do
|
@@ -115,11 +121,26 @@ ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*applic
|
|
115
121
|
/*!*/;
|
116
122
|
SET TIMESTAMP=1415925954/*!*/;
|
117
123
|
EOS
|
118
|
-
let(:
|
124
|
+
let(:normalized_query_1) { 'use `mydatabase` ;' }
|
125
|
+
let(:normalized_query_2) { <<EOS.gsub(/\n$/, '') }
|
119
126
|
ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
|
120
127
|
EOS
|
128
|
+
let(:normalized_query_3) { 'SET TIMESTAMP=1415925954 ;' }
|
129
|
+
|
130
|
+
let(:record) do
|
131
|
+
r = double('record')
|
132
|
+
allow(r).to receive(:[]).with("query").and_return(query)
|
133
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_1}).and_return(r)
|
134
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_2}).and_return(r)
|
135
|
+
expect(r).to receive(:merge).with({"normalized_query" => normalized_query_3}).and_return(r)
|
136
|
+
r
|
137
|
+
end
|
121
138
|
|
122
|
-
|
139
|
+
it do
|
140
|
+
expect(unsupported_query_handler).to receive(:process).with(record).twice
|
141
|
+
expect(alter_query_handler).to receive(:process).with(record).once
|
142
|
+
expect(subject.dispatch(record))
|
143
|
+
end
|
123
144
|
end
|
124
145
|
context "query with a comment with no space" do
|
125
146
|
let(:query) { <<EOS.gsub(/\n$/, '') }
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
|
+
describe UnsupportedQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
describe '#process' do
|
10
|
+
before do
|
11
|
+
record['normalized_query'] = normalized_query
|
12
|
+
end
|
13
|
+
|
14
|
+
context 'when the query includes a sync table' do
|
15
|
+
let(:normalized_query) { 'drop table foo;' }
|
16
|
+
it do
|
17
|
+
expect($log).to receive(:info).with("{:type=>:unsupported_query, :query=>\"drop table foo;\"} at \"mysql-bin.000066\t180\"")
|
18
|
+
subject.process(record)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'when the query does not include a sync table' do
|
23
|
+
let(:normalized_query) { 'drop table bar;' }
|
24
|
+
it do
|
25
|
+
expect($log).to receive(:info).never
|
26
|
+
subject.process(record)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
context 'when the query is issued in another database' do
|
31
|
+
let(:normalized_query) { 'drop table foo;' }
|
32
|
+
before do
|
33
|
+
record['db_name'] = 'another_db'
|
34
|
+
end
|
35
|
+
it do
|
36
|
+
expect($log).to receive(:info).never
|
37
|
+
subject.process(record)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-
|
15
|
+
date: 2016-06-02 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -731,6 +731,7 @@ files:
|
|
731
731
|
- lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
|
732
732
|
- lib/flydata/source_mysql/plugin_support/source_position_file.rb
|
733
733
|
- lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
|
734
|
+
- lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb
|
734
735
|
- lib/flydata/source_mysql/setup.rb
|
735
736
|
- lib/flydata/source_mysql/source_pos.rb
|
736
737
|
- lib/flydata/source_mysql/sync.rb
|
@@ -828,6 +829,7 @@ files:
|
|
828
829
|
- spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
|
829
830
|
- spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
|
830
831
|
- spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
|
832
|
+
- spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb
|
831
833
|
- spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
|
832
834
|
- spec/flydata/source_mysql/table_ddl_spec.rb
|
833
835
|
- spec/flydata/source_mysql/table_meta_spec.rb
|
@@ -867,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
867
869
|
version: '0'
|
868
870
|
requirements: []
|
869
871
|
rubyforge_project:
|
870
|
-
rubygems_version: 2.
|
872
|
+
rubygems_version: 2.0.14.1
|
871
873
|
signing_key:
|
872
874
|
specification_version: 4
|
873
875
|
summary: FlyData Agent
|