flydata 0.7.4 → 0.7.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35042672a68ea10f4ec090aa77f218c885aed555
4
- data.tar.gz: 4223e306573350bb767eda1259e71ccc9d7d182a
3
+ metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
4
+ data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
5
5
  SHA512:
6
- metadata.gz: 143b46f4b1fb7c7e7472932b668caeb66c2201f91bf2bc334553583cf03a6f7800b4b1d895e50174e6043c2328abcf8836b2dc7c2e1848457bf1e0993d3149d9
7
- data.tar.gz: 22ec19871da8e8d1545cf019ec09f4113fddf5ed59da2fe5f285cbce2e616db8f5ecb9320169efefd4444d79594b1907b55a507d8674a9999e2255f7c2ee69ca
6
+ metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
7
+ data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.4
1
+ 0.7.5
data/flydata.gemspec CHANGED
Binary file
@@ -10,14 +10,152 @@ module Flydata
10
10
  super
11
11
  end
12
12
 
13
+ # Response example (/buffer_stat.json for a Sync data entry
14
+ # {
15
+ # "complete": true,
16
+ # "state": "complete",
17
+ # "stat": {
18
+ # "buffer": {
19
+ # "proxy": {
20
+ # "buffer_size": 0,
21
+ # "queue_length": 0
22
+ # },
23
+ # "redshift": {
24
+ # "buffer_size": 0,
25
+ # "queue_length": 0
26
+ # },
27
+ # "sync_redshift": {
28
+ # "buffer_size": 0,
29
+ # "queue_length": 0
30
+ # }
31
+ # },
32
+ # "queue": {
33
+ # "copy_queue": {
34
+ # "num_items": 0
35
+ # },
36
+ # "query_queue": {
37
+ # "num_items": 0
38
+ # }
39
+ # }
40
+ # },
41
+ # "message": "",
42
+ # "success": true
43
+ # }
44
+
45
+ NUM_TABLES_PER_REQUEST = 150
46
+
13
47
  def buffer_stat(data_entry_id, options = {})
14
- tables = options[:tables] ? options[:tables].join(',') : ''
15
- @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
48
+ table_array = options[:tables] || ['']
49
+ table_array = [''] if table_array.empty?
50
+ results = []
51
+ error = false
52
+ table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
53
+ tables = ta.join(',')
54
+ result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
55
+ unless result["state"]
56
+ error = result["message"]
57
+ break
58
+ end
59
+ results << result
60
+ end
61
+ if error
62
+ # mimicking the error value of /buffer_stat API
63
+ {"complete"=> false,
64
+ "message" => error,
65
+ "success" => true,
66
+ }
67
+ else
68
+ stat = results.inject({"buffer"=>{},"queue"=>{}}) do |h, result|
69
+ %w[redshift sync_redshift].each do |fluent_process|
70
+ %w[buffer_size queue_length].each do |stat|
71
+ if result["stat"]["buffer"][fluent_process] && result["stat"]["buffer"][fluent_process][stat]
72
+ h["buffer"][fluent_process] ||= {}
73
+ h["buffer"][fluent_process][stat] ||= 0
74
+ h["buffer"][fluent_process][stat] += result["stat"]["buffer"][fluent_process][stat]
75
+ end
76
+ end
77
+ end
78
+ %w[copy_queue query_queue].each do |queue|
79
+ if result["stat"]["queue"][queue] && result["stat"]["queue"][queue]["num_items"]
80
+ h["queue"][queue] ||= {}
81
+ h["queue"][queue]["num_items"] ||= 0
82
+ h["queue"][queue]["num_items"] += result["stat"]["queue"][queue]["num_items"]
83
+ end
84
+ end
85
+ h
86
+ end
87
+ if proxy = results.first["stat"]["buffer"]["proxy"]
88
+ # proxy values are not per-table
89
+ stat["buffer"]["proxy"] ||= {}
90
+ stat["buffer"]["proxy"]["buffer_size"] = proxy["buffer_size"]
91
+ stat["buffer"]["proxy"]["queue_length"] = proxy["queue_length"]
92
+ end
93
+ # TODO Refactor. The logic below is copied from web data_entries_controller
94
+ buffer_empty = stat["buffer"].all?{|k,v| v["buffer_size"] == 0}
95
+ queue_empty = stat["queue"].all?{|k,v| v["num_items"] == 0}
96
+
97
+ complete = (buffer_empty && queue_empty)
98
+ state = if complete
99
+ 'complete'
100
+ elsif !buffer_empty
101
+ 'processing'
102
+ else
103
+ 'uploading'
104
+ end
105
+ message = results.count == 1 ? results.first["message"]
106
+ : build_message(stat, complete)
107
+
108
+ {
109
+ "complete"=>complete,
110
+ "state" => state,
111
+ "stat" => stat,
112
+ "message" => message,
113
+ "success" => true,
114
+ }
115
+ end
16
116
  end
17
117
 
118
+ # Response example (/table_status.json)
119
+ # {
120
+ # "table_status": [
121
+ # {
122
+ # "updated_at": 1462586673,
123
+ # "status": "ready",
124
+ # "table_name": "items",
125
+ # "seq": 0,
126
+ # "created_at": 1462586673,
127
+ # "src_pos": "-",
128
+ # "records_copied": "0",
129
+ # "num_items": 0,
130
+ # "next_item": false
131
+ # },
132
+ # {
133
+ # "updated_at": 1462586706,
134
+ # "status": "ready",
135
+ # "table_name": "users",
136
+ # "seq": 0,
137
+ # "created_at": 1462586674,
138
+ # "src_pos": "-",
139
+ # "records_copied": "0",
140
+ # "records_sent": "8",
141
+ # "num_items": 0,
142
+ # "next_item": false
143
+ # }
144
+ # ]
145
+ # }
146
+
18
147
  def table_status(data_entry_id, options = {})
19
- tables = options[:tables] ? options[:tables].join(',') : ''
20
- @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
148
+ table_array = options[:tables] || ['']
149
+ results = []
150
+ table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
151
+ tables = ta.join(',')
152
+ result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
153
+ results << result
154
+ end
155
+ results.inject({"table_status"=>[], "success"=>true}) do |h, result|
156
+ h["table_status"] += result["table_status"]
157
+ h
158
+ end
21
159
  end
22
160
 
23
161
  def cleanup_sync(data_entry_id, tables, options = {})
@@ -39,6 +177,27 @@ module Flydata
39
177
  def complete_init_sync(data_entry_id, stats_hash)
40
178
  @client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
41
179
  end
180
+
181
+ private
182
+
183
+ # TODO Refactor. This code is copied from web data_entries_controller
184
+ def build_message(stat, complete)
185
+ return "" if complete
186
+
187
+ message = nil
188
+ buffer_size = stat["buffer"].inject(0) {|m, (k, v)| m + v["buffer_size"]}
189
+ chunk_num = stat["queue"].inject(0) {|m, (k, v)| m + v["num_items"]}
190
+
191
+ if buffer_size > 0
192
+ message = " -> %<buffer_size>s bytes remaining..." % {buffer_size: buffer_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
193
+ elsif chunk_num > 0
194
+ message = " -> %<chunk_num>s data chunk(s) remaining..." % {chunk_num: chunk_num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
195
+ else
196
+ message = " -> Calculating remaining bytes..."
197
+ end
198
+
199
+ message
200
+ end
42
201
  end
43
202
  end
44
203
  end
@@ -266,6 +266,7 @@ EOS
266
266
  def self.slop_repair
267
267
  Slop.new do
268
268
  on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
269
+ on 'skip-start', 'Skip auto start after repair is completed.'
269
270
  on 'force-run', 'Run forcefully, ignoring exclusive run info'
270
271
  end
271
272
  end
@@ -274,6 +275,9 @@ EOS
274
275
  # - Entry method
275
276
  def repair
276
277
  _repair
278
+ unless opts.skip_start?
279
+ Flydata::Command::Sender.new.start
280
+ end
277
281
  end
278
282
  run_exclusive :repair
279
283
 
@@ -42,9 +42,10 @@ module Fluent
42
42
  positions_path = @sync_fm.table_positions_dir_path
43
43
  Dir.mkdir positions_path unless File.exists? positions_path
44
44
 
45
- @tables = @tables.split(/(?:\s*,\s*|\s+)/)
45
+ @tables = @tables.to_s.split(/(?:\s*,\s*|\s+)/)
46
46
  @omit_events = Hash.new
47
- @tables_append_only.split(/(?:\s*,\s*|\s+)/).each do |table|
47
+ @tables_append_only = @tables_append_only.to_s.split(/(?:\s*,\s*|\s+)/)
48
+ @tables_append_only.each do |table|
48
49
  @tables << table unless @tables.include?(table)
49
50
  @omit_events[table] = [:delete, :truncate_table]
50
51
  end
@@ -30,7 +30,8 @@ module Fluent
30
30
  end
31
31
 
32
32
  def apply_custom_conf(conf, key, type, option)
33
- if conf[type.to_s] and value = conf[type.to_s][key.to_s]
33
+ if conf[type.to_s] and conf[type.to_s].has_key?(key.to_s)
34
+ value = conf[type.to_s][key.to_s]
34
35
  var_name = option[:var_name] || key
35
36
  instance_variable_set(:"@#{var_name}", value)
36
37
  end
@@ -27,7 +27,7 @@ module Fluent
27
27
 
28
28
  def start
29
29
  if File.exists?(@lock_file)
30
- $log.error "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
30
+ $log.warn "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
31
31
  @abort = true
32
32
  Process.kill(:TERM, Process.ppid)
33
33
  return
@@ -57,10 +57,10 @@ class Sync < Component
57
57
  if prefs['tables_append_only']
58
58
  prefs['tables_append_only'] =
59
59
  prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
60
- prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
60
+ prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
61
61
  prefs['tables_append_only']).uniq
62
62
  else
63
- prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
63
+ prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
64
64
  end
65
65
  prefs['invalid_tables'] =
66
66
  prefs['invalid_tables'].kind_of?(String) ?
@@ -211,7 +211,7 @@ EOS
211
211
  def flush_tables_with_read_lock_query(client, database, tbls)
212
212
  tbls ||= []
213
213
  tables = ""
214
- if mysql_server_version(client) >= "5.5"
214
+ if mysql_server_version(client).to_f >= 5.5
215
215
  # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
216
216
  tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
217
217
  end
@@ -629,13 +629,14 @@ SELECT
629
629
  FROM
630
630
  information_schema.tables
631
631
  WHERE
632
- table_schema NOT IN ('information_schema','performance_schema','mysql') AND table_name in (%s);
632
+ table_schema = '%s' AND table_name in (%s);
633
633
  EOT
634
634
 
635
635
  def initialize(de_conf)
636
636
  @de_conf = de_conf
637
+ @database = de_conf['database']
637
638
  @tables = de_conf['tables']
638
- @query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
639
+ @query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
639
640
  end
640
641
 
641
642
  def get_db_bytesize
@@ -36,7 +36,7 @@ Received unsupported alter table query. normalized query:'#{record['normalized_q
36
36
  EOS
37
37
  $log.error(msg)
38
38
  end
39
- $log.info(ret) # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
39
+ $log.info(ret.to_s + " at \"#{binlog_pos(record)}\"") # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
40
40
  ret
41
41
  end
42
42
  end
@@ -1,6 +1,7 @@
1
1
  require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
2
2
  require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
3
3
  require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
4
+ require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
4
5
 
5
6
  module Flydata
6
7
  module SourceMysql
@@ -63,6 +64,7 @@ module PluginSupport
63
64
  AlterTableQueryHandler.new(context),
64
65
  TruncateTableQueryHandler.new(context),
65
66
  DropDatabaseQueryHandler.new(context),
67
+ UnsupportedQueryHandler.new(context), # This must be a last element
66
68
  ]
67
69
  end
68
70
  end
@@ -0,0 +1,34 @@
1
+ require 'flydata/parser/parser_provider'
2
+ require 'flydata/source_mysql/plugin_support/ddl_query_handler'
3
+
4
+ module Flydata
5
+ module SourceMysql
6
+
7
+ module PluginSupport
8
+ class UnsupportedQueryHandler < TableDdlQueryHandler
9
+ PATTERN = /.*/ # Catch all ddl
10
+
11
+ def initialize(context)
12
+ super
13
+ end
14
+
15
+ def pattern
16
+ PATTERN
17
+ end
18
+
19
+ def process(record)
20
+ query = record['normalized_query']
21
+ if acceptable_db?(record) && should_log?(query)
22
+ $log.info({ type: :unsupported_query, query: query.to_s.strip}.to_s + " at \"#{binlog_pos(record)}\"")
23
+ end
24
+ end
25
+
26
+ def should_log?(query)
27
+ return false if query.to_s.empty?
28
+ @context.tables.any?{|tbl| query.include?(tbl)}
29
+ end
30
+ end
31
+ end
32
+
33
+ end
34
+ end
@@ -2,17 +2,345 @@ require 'spec_helper'
2
2
  require 'flydata/api/data_entry'
3
3
 
4
4
  module Flydata
5
- module Api
6
- describe DataEntry do
7
- describe '#create' do
8
- it 'creates data entry' do
9
- api_client = double('api_client')
10
- allow(api_client).to receive(:post)
11
-
12
- api = DataEntry.new(api_client)
13
- api.create(data_port_id: 1, log_path: 'log-path')
5
+ module Api
6
+
7
+ describe DataEntry do
8
+ let(:subject_object) { described_class.new(api_client) }
9
+
10
+ let(:api_client) { double('api_client') }
11
+
12
+ let(:data_entry_id) { 1234 }
13
+ let(:options) { { tables: tables } }
14
+
15
+ describe '#create' do
16
+ subject { subject_object.create(params) }
17
+
18
+ let(:params) { {data_port_id: 1, log_path: 'log-path'} }
19
+ it 'creates data entry' do
20
+ expect(api_client).to receive(:post)
21
+
22
+ subject
23
+ end
24
+ end
25
+
26
+ def build_result(complete, state,
27
+ proxy_size, proxy_length, redshift_size, redshift_length,
28
+ sync_redshift_size, sync_redshift_length,
29
+ copy_num, query_num, message)
30
+ buffer = nil
31
+ queue = nil
32
+ if sync_redshift_size
33
+ buffer = { "proxy" => {
34
+ "buffer_size" => proxy_size,
35
+ "queue_length" => proxy_length,
36
+ },
37
+ "redshift" => {
38
+ "buffer_size" => redshift_size,
39
+ "queue_length" => redshift_length,
40
+ },
41
+ "sync_redshift" => {
42
+ "buffer_size" => sync_redshift_size,
43
+ "queue_length" => sync_redshift_length,
44
+ },
45
+ }
46
+ queue = { "copy_queue" => { "num_items" => copy_num },
47
+ "query_queue" => { "num_items" => query_num },
48
+ }
49
+ else
50
+ buffer = { "redshift" => {
51
+ "buffer_size" => redshift_size,
52
+ "queue_length" => redshift_length,
53
+ },
54
+ }
55
+ queue = {
56
+ "copy_queue" => { "num_items" => copy_num },
57
+ }
58
+ end
59
+
60
+ {
61
+ "complete" => complete,
62
+ "state" => state,
63
+ "stat" => { "buffer" => buffer, "queue" => queue },
64
+ "message" => message,
65
+ "success" => true
66
+ }
67
+ end
68
+
69
+ describe '#buffer_stat' do
70
+ subject { subject_object.buffer_stat(data_entry_id, options) }
71
+
72
+ let(:result1) {
73
+ build_result(complete1, state1, proxy_size1, proxy_length1,
74
+ redshift_size1, redshift_length1,
75
+ sync_redshift_size1, sync_redshift_length1,
76
+ copy_num1, query_num1, message1)
77
+ }
78
+ let(:complete1) { false }
79
+ let(:state1) { "processing" }
80
+ let(:proxy_size1) { 1 }
81
+ let(:proxy_length1) { 3 }
82
+ let(:redshift_size1) { 5 }
83
+ let(:redshift_length1) { 7 }
84
+ let(:sync_redshift_size1) { 11 }
85
+ let(:sync_redshift_length1) { 13 }
86
+ let(:copy_num1) { 17 }
87
+ let(:query_num1) { 19 }
88
+ let(:message1) { double('message1') }
89
+ let(:message2) { double('message2') }
90
+
91
+ shared_examples "calling api once and return expected result" do
92
+ it do
93
+ expect(api_client).to receive(:post).
94
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
95
+ nil, tables: expected_tables).
96
+ and_return(api_result)
97
+
98
+ expect(subject).to eq expected_result
99
+ end
100
+ end
101
+
102
+ context "with no tables" do
103
+ let(:tables) { nil }
104
+ let(:expected_tables) { '' }
105
+ let(:api_result) { result1 }
106
+ let(:expected_result) { result1 }
107
+ let(:message) { message1 }
108
+
109
+ it_behaves_like "calling api once and return expected result"
110
+ end
111
+ context "with empty table list" do
112
+ let(:tables) { [] }
113
+ let(:expected_tables) { '' }
114
+ let(:api_result) { result1 }
115
+ let(:expected_result) { result1 }
116
+ let(:message) { message1 }
117
+
118
+ it_behaves_like "calling api once and return expected result"
119
+ end
120
+ context "with less than 100 tables" do
121
+ let(:tables) { %w[a b] }
122
+ let(:expected_tables) { "a,b" }
123
+ let(:api_result) { result1 }
124
+ let(:expected_result) { result1 }
125
+
126
+ it_behaves_like "calling api once and return expected result"
127
+
128
+ context 'when buffer is empty' do
129
+ let(:proxy_size1) { 0 }
130
+ let(:proxy_length1) { 0 }
131
+ let(:redshift_size1) { 0 }
132
+ let(:redshift_length1) { 0 }
133
+ let(:sync_redshift_size1) { 0 }
134
+ let(:sync_redshift_length1) { 0 }
135
+
136
+ let(:api_result) { result1 }
137
+
138
+ context 'when queue is empty' do
139
+ let(:copy_num1) { 0 }
140
+ let(:query_num1) { 0 }
141
+
142
+ let(:expected_result) {
143
+ build_result(true, "complete", proxy_size1, proxy_length1,
144
+ redshift_size1, redshift_length1,
145
+ sync_redshift_size1, sync_redshift_length1,
146
+ copy_num1, query_num1, message1)
147
+ }
148
+
149
+ it_behaves_like "calling api once and return expected result"
150
+ end
151
+ context 'when queue is not empty' do
152
+ let(:expected_result) {
153
+ build_result(false, "uploading", proxy_size1, proxy_length1,
154
+ redshift_size1, redshift_length1,
155
+ sync_redshift_size1, sync_redshift_length1,
156
+ copy_num1, query_num1, message1)
157
+ }
158
+
159
+ it_behaves_like "calling api once and return expected result"
160
+ end
161
+ end
162
+ context 'when buffer is not empty' do
163
+ context 'when queue is empty' do
164
+ let(:copy_num1) { 0 }
165
+ let(:query_num1) { 0 }
166
+
167
+ let(:expected_result) {
168
+ build_result(false, "processing", proxy_size1, proxy_length1,
169
+ redshift_size1, redshift_length1,
170
+ sync_redshift_size1, sync_redshift_length1,
171
+ copy_num1, query_num1, message1)
172
+ }
173
+
174
+ it_behaves_like "calling api once and return expected result"
175
+ end
176
+ end
177
+ end
178
+ context "with more than 100 tables" do
179
+ let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
180
+ let(:expected_tables1) { tables[0...150].join(",") }
181
+ let(:expected_tables2) { tables[150..-1].join(",") }
182
+
183
+ let(:expected_message) { double('expected_message') }
184
+
185
+ context "when no error" do
186
+ shared_examples "calling api twice and return combined result" do
187
+ it do
188
+ expect(api_client).to receive(:post).
189
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
190
+ nil, tables: expected_tables1).
191
+ and_return(api_result1)
192
+ expect(api_client).to receive(:post).
193
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
194
+ nil, tables: expected_tables2).
195
+ and_return(api_result2)
196
+ expect(subject_object).to receive(:build_message).
197
+ and_return(expected_message)
198
+
199
+ expect(subject).to eq expected_result
200
+ end
201
+ end
202
+
203
+ context "with Sync entry" do
204
+ let(:api_result1) {
205
+ build_result(complete1, state1, proxy_size1, proxy_length1,
206
+ redshift_size1, redshift_length1,
207
+ sync_redshift_size1, sync_redshift_length1,
208
+ copy_num1, query_num1, message1)
209
+ }
210
+ let(:api_result2) {
211
+ build_result(complete1, state1, proxy_size1, proxy_length1,
212
+ redshift_size1, redshift_length1,
213
+ sync_redshift_size1, sync_redshift_length1,
214
+ copy_num1, query_num1, message2)
215
+ }
216
+ let(:expected_result) {
217
+ build_result(complete1, state1,
218
+ proxy_size1,
219
+ proxy_length1,
220
+ redshift_size1 + redshift_size1,
221
+ redshift_length1 + redshift_length1,
222
+ sync_redshift_size1 + sync_redshift_size1,
223
+ sync_redshift_length1 + sync_redshift_length1,
224
+ copy_num1 + copy_num1,
225
+ query_num1 + query_num1, expected_message)
226
+ }
227
+ it_behaves_like "calling api twice and return combined result"
228
+ end
229
+ context "with AutoLoad entry" do
230
+ let(:api_result1) {
231
+ build_result(complete1, state1, nil, nil,
232
+ redshift_size1, redshift_length1,
233
+ nil, nil,
234
+ copy_num1, query_num1, message1)
235
+ }
236
+ let(:api_result2) {
237
+ build_result(complete1, state1, nil, nil,
238
+ redshift_size1, redshift_length1,
239
+ nil, nil,
240
+ copy_num1, query_num1, message2)
241
+ }
242
+ let(:expected_result) {
243
+ build_result(complete1, state1,
244
+ nil,
245
+ nil,
246
+ redshift_size1 + redshift_size1,
247
+ redshift_length1 + redshift_length1,
248
+ nil,
249
+ nil,
250
+ copy_num1 + copy_num1,
251
+ query_num1 + query_num1, expected_message)
252
+ }
253
+ it_behaves_like "calling api twice and return combined result"
254
+ end
255
+ end
256
+ context "with an error with the first api call" do
257
+ let(:api_result1) { {
258
+ "complete" => false,
259
+ "message" => expected_message,
260
+ "success" => true,
261
+ } }
262
+ let(:api_result2) { result1 }
263
+ let(:expected_result) { api_result1 }
264
+ it do
265
+ expect(api_client).to receive(:post).
266
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
267
+ nil, tables: expected_tables1).
268
+ and_return(api_result1)
269
+ expect(subject_object).not_to receive(:build_message)
270
+
271
+ expect(subject).to eq expected_result
14
272
  end
15
273
  end
16
274
  end
17
275
  end
276
+
277
+ describe '#table_status' do
278
+ subject { subject_object.table_status(data_entry_id, options) }
279
+
280
+ let(:result1) {
281
+ { "table_status" => [ status11, status12 ], "success" => true }
282
+ }
283
+ let(:status11) { double('status11') }
284
+ let(:status12) { double('status12') }
285
+
286
+ shared_examples "calling api with tables and return expected result" do
287
+ it do
288
+ expect(api_client).to receive(:post).
289
+ with("/data_entries/#{data_entry_id}/table_status",
290
+ nil, tables: expected_tables).
291
+ and_return(api_result)
292
+
293
+ expect(subject).to eq expected_result
294
+ end
295
+ end
296
+
297
+ context "with no tables" do
298
+ let(:tables) { nil }
299
+ let(:expected_tables) { '' }
300
+ let(:api_result) { result1 }
301
+ let(:expected_result) { result1 }
302
+
303
+ it_behaves_like "calling api with tables and return expected result"
304
+ end
305
+
306
+ context "with less than 100 tables" do
307
+ let(:tables) { %w[a b] }
308
+ let(:expected_tables) { 'a,b' }
309
+ let(:api_result) { result1 }
310
+ let(:expected_result) { result1 }
311
+
312
+ it_behaves_like "calling api with tables and return expected result"
313
+ end
314
+
315
+ let(:result2) {
316
+ { "table_status" => [ status21, status22 ], "success" => true }
317
+ }
318
+ let(:status21) { double('status21') }
319
+ let(:status22) { double('status22') }
320
+
321
+ context "with more than 100 tables" do
322
+ let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
323
+ let(:expected_tables1) { tables[0...150].join(",") }
324
+ let(:expected_tables2) { tables[150..-1].join(",") }
325
+
326
+ it do
327
+ expect(api_client).to receive(:post).
328
+ with("/data_entries/#{data_entry_id}/table_status",
329
+ nil, tables: expected_tables1).
330
+ and_return(result1)
331
+
332
+ expect(api_client).to receive(:post).
333
+ with("/data_entries/#{data_entry_id}/table_status",
334
+ nil, tables: expected_tables2).
335
+ and_return(result2)
336
+
337
+ expect(subject).to eq(
338
+ "table_status" => [ status11, status12, status21, status22 ],
339
+ "success" => true )
340
+ end
341
+ end
342
+ end
343
+ end
344
+
345
+ end
18
346
  end
@@ -264,6 +264,10 @@ EOT
264
264
  expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
265
265
  end
266
266
 
267
+ it 'sets data tables_append_only as array' do
268
+ expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
269
+ end
270
+
267
271
  it 'sets sync_fm' do
268
272
  expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
269
273
  end
@@ -31,10 +31,12 @@ module Flydata
31
31
  end
32
32
 
33
33
  describe MysqlDumpGeneratorNoMasterData do
34
+ let(:subject_object) { described_class.new(default_conf) }
35
+
34
36
  let(:stderr) { double(:stderr) }
35
37
  let(:wait_thr) { double(:wait_thr) }
36
38
 
37
- let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
39
+ let(:default_dump_generator) { subject_object }
38
40
  let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
39
41
  let(:mysql_client) do
40
42
  m = double(:mysql_client)
@@ -80,6 +82,54 @@ module Flydata
80
82
  File.delete(file_path) if File.exists?(file_path)
81
83
  end
82
84
  end
85
+
86
+ describe '#flush_tables_with_read_lock_query' do
87
+ subject { subject_object.send(:flush_tables_with_read_lock_query,
88
+ client, database, tbls) }
89
+
90
+ let(:client) { double('client') }
91
+ let(:database) { "db1" }
92
+ let(:tbls) { ['table1'] }
93
+
94
+ before do
95
+ allow(subject_object).to receive(:mysql_server_version).
96
+ and_return(version)
97
+ end
98
+
99
+ let(:flush_tables_with_tables) { <<EOS.strip }
100
+ FLUSH TABLES `db1`.`table1` WITH READ LOCK;
101
+ EOS
102
+ let(:flush_tables_without_table) { <<EOS.strip }
103
+ FLUSH TABLES WITH READ LOCK;
104
+ EOS
105
+
106
+ context 'with MySQL 5.1' do
107
+ let(:version) { "5.1.67-log" }
108
+ it 'returns flush command without table because the syntax is only supported by MySQL 5.5 or later' do
109
+ is_expected.to eq(flush_tables_without_table)
110
+ end
111
+ end
112
+ context 'with Debian MySQL 5.5' do
113
+ let(:version) { "5.5.24-2+deb.sury.org~precise+2-log" }
114
+ it { is_expected.to eq(flush_tables_with_tables) }
115
+ end
116
+ context 'with Amazon RDS MySQL 5.6' do
117
+ let(:version) { "5.6.19-log" }
118
+ it { is_expected.to eq(flush_tables_with_tables) }
119
+ end
120
+ context 'with MariaDB 10.0' do
121
+ let(:version) { "10.0.24-MariaDB" }
122
+ it 'returns flush command with tables because MariaDB 10.0 is compatible with MySQL 5.6' do
123
+ is_expected.to eq(flush_tables_with_tables)
124
+ end
125
+ end
126
+ context 'with Amazon Aurora' do
127
+ let(:version) { "5.6.10" }
128
+ it do
129
+ is_expected.to eq(flush_tables_with_tables)
130
+ end
131
+ end
132
+ end
83
133
  end
84
134
  end
85
135
 
@@ -64,6 +64,11 @@ EOT
64
64
  allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
65
65
  r
66
66
  end
67
+ let(:unsupported_query_handler) do
68
+ r = double('unsupported_query_handler')
69
+ allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::UnsupportedQueryHandler::PATTERN)
70
+ r
71
+ end
67
72
  let(:context) { double('context') }
68
73
  let(:subject_object) { described_class.new(context) }
69
74
  subject { subject_object }
@@ -79,6 +84,7 @@ EOT
79
84
  expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
80
85
  expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
81
86
  expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
87
+ allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
82
88
  end
83
89
 
84
90
  describe '#dispatch' do
@@ -115,11 +121,26 @@ ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*applic
115
121
  /*!*/;
116
122
  SET TIMESTAMP=1415925954/*!*/;
117
123
  EOS
118
- let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
124
+ let(:normalized_query_1) { 'use `mydatabase` ;' }
125
+ let(:normalized_query_2) { <<EOS.gsub(/\n$/, '') }
119
126
  ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
120
127
  EOS
128
+ let(:normalized_query_3) { 'SET TIMESTAMP=1415925954 ;' }
129
+
130
+ let(:record) do
131
+ r = double('record')
132
+ allow(r).to receive(:[]).with("query").and_return(query)
133
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_1}).and_return(r)
134
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_2}).and_return(r)
135
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_3}).and_return(r)
136
+ r
137
+ end
121
138
 
122
- it_behaves_like "a dispatcher that calls query handler with correct query"
139
+ it do
140
+ expect(unsupported_query_handler).to receive(:process).with(record).twice
141
+ expect(alter_query_handler).to receive(:process).with(record).once
142
+ expect(subject.dispatch(record))
143
+ end
123
144
  end
124
145
  context "query with a comment with no space" do
125
146
  let(:query) { <<EOS.gsub(/\n$/, '') }
@@ -0,0 +1,42 @@
1
+ require 'fluent_plugins_spec_helper'
2
+ require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
+
5
+ module Flydata::SourceMysql::PluginSupport
6
+ describe UnsupportedQueryHandler do
7
+ include_context "query handler context"
8
+
9
+ describe '#process' do
10
+ before do
11
+ record['normalized_query'] = normalized_query
12
+ end
13
+
14
+ context 'when the query includes a sync table' do
15
+ let(:normalized_query) { 'drop table foo;' }
16
+ it do
17
+ expect($log).to receive(:info).with("{:type=>:unsupported_query, :query=>\"drop table foo;\"} at \"mysql-bin.000066\t180\"")
18
+ subject.process(record)
19
+ end
20
+ end
21
+
22
+ context 'when the query does not include a sync table' do
23
+ let(:normalized_query) { 'drop table bar;' }
24
+ it do
25
+ expect($log).to receive(:info).never
26
+ subject.process(record)
27
+ end
28
+ end
29
+
30
+ context 'when the query is issued in another database' do
31
+ let(:normalized_query) { 'drop table foo;' }
32
+ before do
33
+ record['db_name'] = 'another_db'
34
+ end
35
+ it do
36
+ expect($log).to receive(:info).never
37
+ subject.process(record)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.4
4
+ version: 0.7.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-04-28 00:00:00.000000000 Z
15
+ date: 2016-06-02 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -731,6 +731,7 @@ files:
731
731
  - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
732
732
  - lib/flydata/source_mysql/plugin_support/source_position_file.rb
733
733
  - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
734
+ - lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb
734
735
  - lib/flydata/source_mysql/setup.rb
735
736
  - lib/flydata/source_mysql/source_pos.rb
736
737
  - lib/flydata/source_mysql/sync.rb
@@ -828,6 +829,7 @@ files:
828
829
  - spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
829
830
  - spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
830
831
  - spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
832
+ - spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb
831
833
  - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
832
834
  - spec/flydata/source_mysql/table_ddl_spec.rb
833
835
  - spec/flydata/source_mysql/table_meta_spec.rb
@@ -867,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
867
869
  version: '0'
868
870
  requirements: []
869
871
  rubyforge_project:
870
- rubygems_version: 2.4.3
872
+ rubygems_version: 2.0.14.1
871
873
  signing_key:
872
874
  specification_version: 4
873
875
  summary: FlyData Agent