flydata 0.7.4 → 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35042672a68ea10f4ec090aa77f218c885aed555
4
- data.tar.gz: 4223e306573350bb767eda1259e71ccc9d7d182a
3
+ metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
4
+ data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
5
5
  SHA512:
6
- metadata.gz: 143b46f4b1fb7c7e7472932b668caeb66c2201f91bf2bc334553583cf03a6f7800b4b1d895e50174e6043c2328abcf8836b2dc7c2e1848457bf1e0993d3149d9
7
- data.tar.gz: 22ec19871da8e8d1545cf019ec09f4113fddf5ed59da2fe5f285cbce2e616db8f5ecb9320169efefd4444d79594b1907b55a507d8674a9999e2255f7c2ee69ca
6
+ metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
7
+ data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.4
1
+ 0.7.5
data/flydata.gemspec CHANGED
Binary file
@@ -10,14 +10,152 @@ module Flydata
10
10
  super
11
11
  end
12
12
 
13
+ # Response example (/buffer_stat.json for a Sync data entry
14
+ # {
15
+ # "complete": true,
16
+ # "state": "complete",
17
+ # "stat": {
18
+ # "buffer": {
19
+ # "proxy": {
20
+ # "buffer_size": 0,
21
+ # "queue_length": 0
22
+ # },
23
+ # "redshift": {
24
+ # "buffer_size": 0,
25
+ # "queue_length": 0
26
+ # },
27
+ # "sync_redshift": {
28
+ # "buffer_size": 0,
29
+ # "queue_length": 0
30
+ # }
31
+ # },
32
+ # "queue": {
33
+ # "copy_queue": {
34
+ # "num_items": 0
35
+ # },
36
+ # "query_queue": {
37
+ # "num_items": 0
38
+ # }
39
+ # }
40
+ # },
41
+ # "message": "",
42
+ # "success": true
43
+ # }
44
+
45
+ NUM_TABLES_PER_REQUEST = 150
46
+
13
47
  def buffer_stat(data_entry_id, options = {})
14
- tables = options[:tables] ? options[:tables].join(',') : ''
15
- @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
48
+ table_array = options[:tables] || ['']
49
+ table_array = [''] if table_array.empty?
50
+ results = []
51
+ error = false
52
+ table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
53
+ tables = ta.join(',')
54
+ result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
55
+ unless result["state"]
56
+ error = result["message"]
57
+ break
58
+ end
59
+ results << result
60
+ end
61
+ if error
62
+ # mimicking the error value of /buffer_stat API
63
+ {"complete"=> false,
64
+ "message" => error,
65
+ "success" => true,
66
+ }
67
+ else
68
+ stat = results.inject({"buffer"=>{},"queue"=>{}}) do |h, result|
69
+ %w[redshift sync_redshift].each do |fluent_process|
70
+ %w[buffer_size queue_length].each do |stat|
71
+ if result["stat"]["buffer"][fluent_process] && result["stat"]["buffer"][fluent_process][stat]
72
+ h["buffer"][fluent_process] ||= {}
73
+ h["buffer"][fluent_process][stat] ||= 0
74
+ h["buffer"][fluent_process][stat] += result["stat"]["buffer"][fluent_process][stat]
75
+ end
76
+ end
77
+ end
78
+ %w[copy_queue query_queue].each do |queue|
79
+ if result["stat"]["queue"][queue] && result["stat"]["queue"][queue]["num_items"]
80
+ h["queue"][queue] ||= {}
81
+ h["queue"][queue]["num_items"] ||= 0
82
+ h["queue"][queue]["num_items"] += result["stat"]["queue"][queue]["num_items"]
83
+ end
84
+ end
85
+ h
86
+ end
87
+ if proxy = results.first["stat"]["buffer"]["proxy"]
88
+ # proxy values are not per-table
89
+ stat["buffer"]["proxy"] ||= {}
90
+ stat["buffer"]["proxy"]["buffer_size"] = proxy["buffer_size"]
91
+ stat["buffer"]["proxy"]["queue_length"] = proxy["queue_length"]
92
+ end
93
+ # TODO Refactor. The logic below is copied from web data_entries_controller
94
+ buffer_empty = stat["buffer"].all?{|k,v| v["buffer_size"] == 0}
95
+ queue_empty = stat["queue"].all?{|k,v| v["num_items"] == 0}
96
+
97
+ complete = (buffer_empty && queue_empty)
98
+ state = if complete
99
+ 'complete'
100
+ elsif !buffer_empty
101
+ 'processing'
102
+ else
103
+ 'uploading'
104
+ end
105
+ message = results.count == 1 ? results.first["message"]
106
+ : build_message(stat, complete)
107
+
108
+ {
109
+ "complete"=>complete,
110
+ "state" => state,
111
+ "stat" => stat,
112
+ "message" => message,
113
+ "success" => true,
114
+ }
115
+ end
16
116
  end
17
117
 
118
+ # Response example (/table_status.json)
119
+ # {
120
+ # "table_status": [
121
+ # {
122
+ # "updated_at": 1462586673,
123
+ # "status": "ready",
124
+ # "table_name": "items",
125
+ # "seq": 0,
126
+ # "created_at": 1462586673,
127
+ # "src_pos": "-",
128
+ # "records_copied": "0",
129
+ # "num_items": 0,
130
+ # "next_item": false
131
+ # },
132
+ # {
133
+ # "updated_at": 1462586706,
134
+ # "status": "ready",
135
+ # "table_name": "users",
136
+ # "seq": 0,
137
+ # "created_at": 1462586674,
138
+ # "src_pos": "-",
139
+ # "records_copied": "0",
140
+ # "records_sent": "8",
141
+ # "num_items": 0,
142
+ # "next_item": false
143
+ # }
144
+ # ]
145
+ # }
146
+
18
147
  def table_status(data_entry_id, options = {})
19
- tables = options[:tables] ? options[:tables].join(',') : ''
20
- @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
148
+ table_array = options[:tables] || ['']
149
+ results = []
150
+ table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
151
+ tables = ta.join(',')
152
+ result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
153
+ results << result
154
+ end
155
+ results.inject({"table_status"=>[], "success"=>true}) do |h, result|
156
+ h["table_status"] += result["table_status"]
157
+ h
158
+ end
21
159
  end
22
160
 
23
161
  def cleanup_sync(data_entry_id, tables, options = {})
@@ -39,6 +177,27 @@ module Flydata
39
177
  def complete_init_sync(data_entry_id, stats_hash)
40
178
  @client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
41
179
  end
180
+
181
+ private
182
+
183
+ # TODO Refactor. This code is copied from web data_entries_controller
184
+ def build_message(stat, complete)
185
+ return "" if complete
186
+
187
+ message = nil
188
+ buffer_size = stat["buffer"].inject(0) {|m, (k, v)| m + v["buffer_size"]}
189
+ chunk_num = stat["queue"].inject(0) {|m, (k, v)| m + v["num_items"]}
190
+
191
+ if buffer_size > 0
192
+ message = " -> %<buffer_size>s bytes remaining..." % {buffer_size: buffer_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
193
+ elsif chunk_num > 0
194
+ message = " -> %<chunk_num>s data chunk(s) remaining..." % {chunk_num: chunk_num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
195
+ else
196
+ message = " -> Calculating remaining bytes..."
197
+ end
198
+
199
+ message
200
+ end
42
201
  end
43
202
  end
44
203
  end
@@ -266,6 +266,7 @@ EOS
266
266
  def self.slop_repair
267
267
  Slop.new do
268
268
  on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
269
+ on 'skip-start', 'Skip auto start after repair is completed.'
269
270
  on 'force-run', 'Run forcefully, ignoring exclusive run info'
270
271
  end
271
272
  end
@@ -274,6 +275,9 @@ EOS
274
275
  # - Entry method
275
276
  def repair
276
277
  _repair
278
+ unless opts.skip_start?
279
+ Flydata::Command::Sender.new.start
280
+ end
277
281
  end
278
282
  run_exclusive :repair
279
283
 
@@ -42,9 +42,10 @@ module Fluent
42
42
  positions_path = @sync_fm.table_positions_dir_path
43
43
  Dir.mkdir positions_path unless File.exists? positions_path
44
44
 
45
- @tables = @tables.split(/(?:\s*,\s*|\s+)/)
45
+ @tables = @tables.to_s.split(/(?:\s*,\s*|\s+)/)
46
46
  @omit_events = Hash.new
47
- @tables_append_only.split(/(?:\s*,\s*|\s+)/).each do |table|
47
+ @tables_append_only = @tables_append_only.to_s.split(/(?:\s*,\s*|\s+)/)
48
+ @tables_append_only.each do |table|
48
49
  @tables << table unless @tables.include?(table)
49
50
  @omit_events[table] = [:delete, :truncate_table]
50
51
  end
@@ -30,7 +30,8 @@ module Fluent
30
30
  end
31
31
 
32
32
  def apply_custom_conf(conf, key, type, option)
33
- if conf[type.to_s] and value = conf[type.to_s][key.to_s]
33
+ if conf[type.to_s] and conf[type.to_s].has_key?(key.to_s)
34
+ value = conf[type.to_s][key.to_s]
34
35
  var_name = option[:var_name] || key
35
36
  instance_variable_set(:"@#{var_name}", value)
36
37
  end
@@ -27,7 +27,7 @@ module Fluent
27
27
 
28
28
  def start
29
29
  if File.exists?(@lock_file)
30
- $log.error "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
30
+ $log.warn "Previous process was terminated abnormally. To start, remove the lock file after checking data integrity."
31
31
  @abort = true
32
32
  Process.kill(:TERM, Process.ppid)
33
33
  return
@@ -57,10 +57,10 @@ class Sync < Component
57
57
  if prefs['tables_append_only']
58
58
  prefs['tables_append_only'] =
59
59
  prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
60
- prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
60
+ prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
61
61
  prefs['tables_append_only']).uniq
62
62
  else
63
- prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
63
+ prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
64
64
  end
65
65
  prefs['invalid_tables'] =
66
66
  prefs['invalid_tables'].kind_of?(String) ?
@@ -211,7 +211,7 @@ EOS
211
211
  def flush_tables_with_read_lock_query(client, database, tbls)
212
212
  tbls ||= []
213
213
  tables = ""
214
- if mysql_server_version(client) >= "5.5"
214
+ if mysql_server_version(client).to_f >= 5.5
215
215
  # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
216
216
  tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
217
217
  end
@@ -629,13 +629,14 @@ SELECT
629
629
  FROM
630
630
  information_schema.tables
631
631
  WHERE
632
- table_schema NOT IN ('information_schema','performance_schema','mysql') AND table_name in (%s);
632
+ table_schema = '%s' AND table_name in (%s);
633
633
  EOT
634
634
 
635
635
  def initialize(de_conf)
636
636
  @de_conf = de_conf
637
+ @database = de_conf['database']
637
638
  @tables = de_conf['tables']
638
- @query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
639
+ @query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
639
640
  end
640
641
 
641
642
  def get_db_bytesize
@@ -36,7 +36,7 @@ Received unsupported alter table query. normalized query:'#{record['normalized_q
36
36
  EOS
37
37
  $log.error(msg)
38
38
  end
39
- $log.info(ret) # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
39
+ $log.info(ret.to_s + " at \"#{binlog_pos(record)}\"") # log all alter table events even if the table is not subject to sync. This is for troubleshooting purpose.
40
40
  ret
41
41
  end
42
42
  end
@@ -1,6 +1,7 @@
1
1
  require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
2
2
  require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
3
3
  require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
4
+ require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
4
5
 
5
6
  module Flydata
6
7
  module SourceMysql
@@ -63,6 +64,7 @@ module PluginSupport
63
64
  AlterTableQueryHandler.new(context),
64
65
  TruncateTableQueryHandler.new(context),
65
66
  DropDatabaseQueryHandler.new(context),
67
+ UnsupportedQueryHandler.new(context), # This must be a last element
66
68
  ]
67
69
  end
68
70
  end
@@ -0,0 +1,34 @@
1
+ require 'flydata/parser/parser_provider'
2
+ require 'flydata/source_mysql/plugin_support/ddl_query_handler'
3
+
4
+ module Flydata
5
+ module SourceMysql
6
+
7
+ module PluginSupport
8
+ class UnsupportedQueryHandler < TableDdlQueryHandler
9
+ PATTERN = /.*/ # Catch all ddl
10
+
11
+ def initialize(context)
12
+ super
13
+ end
14
+
15
+ def pattern
16
+ PATTERN
17
+ end
18
+
19
+ def process(record)
20
+ query = record['normalized_query']
21
+ if acceptable_db?(record) && should_log?(query)
22
+ $log.info({ type: :unsupported_query, query: query.to_s.strip}.to_s + " at \"#{binlog_pos(record)}\"")
23
+ end
24
+ end
25
+
26
+ def should_log?(query)
27
+ return false if query.to_s.empty?
28
+ @context.tables.any?{|tbl| query.include?(tbl)}
29
+ end
30
+ end
31
+ end
32
+
33
+ end
34
+ end
@@ -2,17 +2,345 @@ require 'spec_helper'
2
2
  require 'flydata/api/data_entry'
3
3
 
4
4
  module Flydata
5
- module Api
6
- describe DataEntry do
7
- describe '#create' do
8
- it 'creates data entry' do
9
- api_client = double('api_client')
10
- allow(api_client).to receive(:post)
11
-
12
- api = DataEntry.new(api_client)
13
- api.create(data_port_id: 1, log_path: 'log-path')
5
+ module Api
6
+
7
+ describe DataEntry do
8
+ let(:subject_object) { described_class.new(api_client) }
9
+
10
+ let(:api_client) { double('api_client') }
11
+
12
+ let(:data_entry_id) { 1234 }
13
+ let(:options) { { tables: tables } }
14
+
15
+ describe '#create' do
16
+ subject { subject_object.create(params) }
17
+
18
+ let(:params) { {data_port_id: 1, log_path: 'log-path'} }
19
+ it 'creates data entry' do
20
+ expect(api_client).to receive(:post)
21
+
22
+ subject
23
+ end
24
+ end
25
+
26
+ def build_result(complete, state,
27
+ proxy_size, proxy_length, redshift_size, redshift_length,
28
+ sync_redshift_size, sync_redshift_length,
29
+ copy_num, query_num, message)
30
+ buffer = nil
31
+ queue = nil
32
+ if sync_redshift_size
33
+ buffer = { "proxy" => {
34
+ "buffer_size" => proxy_size,
35
+ "queue_length" => proxy_length,
36
+ },
37
+ "redshift" => {
38
+ "buffer_size" => redshift_size,
39
+ "queue_length" => redshift_length,
40
+ },
41
+ "sync_redshift" => {
42
+ "buffer_size" => sync_redshift_size,
43
+ "queue_length" => sync_redshift_length,
44
+ },
45
+ }
46
+ queue = { "copy_queue" => { "num_items" => copy_num },
47
+ "query_queue" => { "num_items" => query_num },
48
+ }
49
+ else
50
+ buffer = { "redshift" => {
51
+ "buffer_size" => redshift_size,
52
+ "queue_length" => redshift_length,
53
+ },
54
+ }
55
+ queue = {
56
+ "copy_queue" => { "num_items" => copy_num },
57
+ }
58
+ end
59
+
60
+ {
61
+ "complete" => complete,
62
+ "state" => state,
63
+ "stat" => { "buffer" => buffer, "queue" => queue },
64
+ "message" => message,
65
+ "success" => true
66
+ }
67
+ end
68
+
69
+ describe '#buffer_stat' do
70
+ subject { subject_object.buffer_stat(data_entry_id, options) }
71
+
72
+ let(:result1) {
73
+ build_result(complete1, state1, proxy_size1, proxy_length1,
74
+ redshift_size1, redshift_length1,
75
+ sync_redshift_size1, sync_redshift_length1,
76
+ copy_num1, query_num1, message1)
77
+ }
78
+ let(:complete1) { false }
79
+ let(:state1) { "processing" }
80
+ let(:proxy_size1) { 1 }
81
+ let(:proxy_length1) { 3 }
82
+ let(:redshift_size1) { 5 }
83
+ let(:redshift_length1) { 7 }
84
+ let(:sync_redshift_size1) { 11 }
85
+ let(:sync_redshift_length1) { 13 }
86
+ let(:copy_num1) { 17 }
87
+ let(:query_num1) { 19 }
88
+ let(:message1) { double('message1') }
89
+ let(:message2) { double('message2') }
90
+
91
+ shared_examples "calling api once and return expected result" do
92
+ it do
93
+ expect(api_client).to receive(:post).
94
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
95
+ nil, tables: expected_tables).
96
+ and_return(api_result)
97
+
98
+ expect(subject).to eq expected_result
99
+ end
100
+ end
101
+
102
+ context "with no tables" do
103
+ let(:tables) { nil }
104
+ let(:expected_tables) { '' }
105
+ let(:api_result) { result1 }
106
+ let(:expected_result) { result1 }
107
+ let(:message) { message1 }
108
+
109
+ it_behaves_like "calling api once and return expected result"
110
+ end
111
+ context "with empty table list" do
112
+ let(:tables) { [] }
113
+ let(:expected_tables) { '' }
114
+ let(:api_result) { result1 }
115
+ let(:expected_result) { result1 }
116
+ let(:message) { message1 }
117
+
118
+ it_behaves_like "calling api once and return expected result"
119
+ end
120
+ context "with less than 100 tables" do
121
+ let(:tables) { %w[a b] }
122
+ let(:expected_tables) { "a,b" }
123
+ let(:api_result) { result1 }
124
+ let(:expected_result) { result1 }
125
+
126
+ it_behaves_like "calling api once and return expected result"
127
+
128
+ context 'when buffer is empty' do
129
+ let(:proxy_size1) { 0 }
130
+ let(:proxy_length1) { 0 }
131
+ let(:redshift_size1) { 0 }
132
+ let(:redshift_length1) { 0 }
133
+ let(:sync_redshift_size1) { 0 }
134
+ let(:sync_redshift_length1) { 0 }
135
+
136
+ let(:api_result) { result1 }
137
+
138
+ context 'when queue is empty' do
139
+ let(:copy_num1) { 0 }
140
+ let(:query_num1) { 0 }
141
+
142
+ let(:expected_result) {
143
+ build_result(true, "complete", proxy_size1, proxy_length1,
144
+ redshift_size1, redshift_length1,
145
+ sync_redshift_size1, sync_redshift_length1,
146
+ copy_num1, query_num1, message1)
147
+ }
148
+
149
+ it_behaves_like "calling api once and return expected result"
150
+ end
151
+ context 'when queue is not empty' do
152
+ let(:expected_result) {
153
+ build_result(false, "uploading", proxy_size1, proxy_length1,
154
+ redshift_size1, redshift_length1,
155
+ sync_redshift_size1, sync_redshift_length1,
156
+ copy_num1, query_num1, message1)
157
+ }
158
+
159
+ it_behaves_like "calling api once and return expected result"
160
+ end
161
+ end
162
+ context 'when buffer is not empty' do
163
+ context 'when queue is empty' do
164
+ let(:copy_num1) { 0 }
165
+ let(:query_num1) { 0 }
166
+
167
+ let(:expected_result) {
168
+ build_result(false, "processing", proxy_size1, proxy_length1,
169
+ redshift_size1, redshift_length1,
170
+ sync_redshift_size1, sync_redshift_length1,
171
+ copy_num1, query_num1, message1)
172
+ }
173
+
174
+ it_behaves_like "calling api once and return expected result"
175
+ end
176
+ end
177
+ end
178
+ context "with more than 100 tables" do
179
+ let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
180
+ let(:expected_tables1) { tables[0...150].join(",") }
181
+ let(:expected_tables2) { tables[150..-1].join(",") }
182
+
183
+ let(:expected_message) { double('expected_message') }
184
+
185
+ context "when no error" do
186
+ shared_examples "calling api twice and return combined result" do
187
+ it do
188
+ expect(api_client).to receive(:post).
189
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
190
+ nil, tables: expected_tables1).
191
+ and_return(api_result1)
192
+ expect(api_client).to receive(:post).
193
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
194
+ nil, tables: expected_tables2).
195
+ and_return(api_result2)
196
+ expect(subject_object).to receive(:build_message).
197
+ and_return(expected_message)
198
+
199
+ expect(subject).to eq expected_result
200
+ end
201
+ end
202
+
203
+ context "with Sync entry" do
204
+ let(:api_result1) {
205
+ build_result(complete1, state1, proxy_size1, proxy_length1,
206
+ redshift_size1, redshift_length1,
207
+ sync_redshift_size1, sync_redshift_length1,
208
+ copy_num1, query_num1, message1)
209
+ }
210
+ let(:api_result2) {
211
+ build_result(complete1, state1, proxy_size1, proxy_length1,
212
+ redshift_size1, redshift_length1,
213
+ sync_redshift_size1, sync_redshift_length1,
214
+ copy_num1, query_num1, message2)
215
+ }
216
+ let(:expected_result) {
217
+ build_result(complete1, state1,
218
+ proxy_size1,
219
+ proxy_length1,
220
+ redshift_size1 + redshift_size1,
221
+ redshift_length1 + redshift_length1,
222
+ sync_redshift_size1 + sync_redshift_size1,
223
+ sync_redshift_length1 + sync_redshift_length1,
224
+ copy_num1 + copy_num1,
225
+ query_num1 + query_num1, expected_message)
226
+ }
227
+ it_behaves_like "calling api twice and return combined result"
228
+ end
229
+ context "with AutoLoad entry" do
230
+ let(:api_result1) {
231
+ build_result(complete1, state1, nil, nil,
232
+ redshift_size1, redshift_length1,
233
+ nil, nil,
234
+ copy_num1, query_num1, message1)
235
+ }
236
+ let(:api_result2) {
237
+ build_result(complete1, state1, nil, nil,
238
+ redshift_size1, redshift_length1,
239
+ nil, nil,
240
+ copy_num1, query_num1, message2)
241
+ }
242
+ let(:expected_result) {
243
+ build_result(complete1, state1,
244
+ nil,
245
+ nil,
246
+ redshift_size1 + redshift_size1,
247
+ redshift_length1 + redshift_length1,
248
+ nil,
249
+ nil,
250
+ copy_num1 + copy_num1,
251
+ query_num1 + query_num1, expected_message)
252
+ }
253
+ it_behaves_like "calling api twice and return combined result"
254
+ end
255
+ end
256
+ context "with an error with the first api call" do
257
+ let(:api_result1) { {
258
+ "complete" => false,
259
+ "message" => expected_message,
260
+ "success" => true,
261
+ } }
262
+ let(:api_result2) { result1 }
263
+ let(:expected_result) { api_result1 }
264
+ it do
265
+ expect(api_client).to receive(:post).
266
+ with("/data_entries/#{data_entry_id}/buffer_stat/",
267
+ nil, tables: expected_tables1).
268
+ and_return(api_result1)
269
+ expect(subject_object).not_to receive(:build_message)
270
+
271
+ expect(subject).to eq expected_result
14
272
  end
15
273
  end
16
274
  end
17
275
  end
276
+
277
+ describe '#table_status' do
278
+ subject { subject_object.table_status(data_entry_id, options) }
279
+
280
+ let(:result1) {
281
+ { "table_status" => [ status11, status12 ], "success" => true }
282
+ }
283
+ let(:status11) { double('status11') }
284
+ let(:status12) { double('status12') }
285
+
286
+ shared_examples "calling api with tables and return expected result" do
287
+ it do
288
+ expect(api_client).to receive(:post).
289
+ with("/data_entries/#{data_entry_id}/table_status",
290
+ nil, tables: expected_tables).
291
+ and_return(api_result)
292
+
293
+ expect(subject).to eq expected_result
294
+ end
295
+ end
296
+
297
+ context "with no tables" do
298
+ let(:tables) { nil }
299
+ let(:expected_tables) { '' }
300
+ let(:api_result) { result1 }
301
+ let(:expected_result) { result1 }
302
+
303
+ it_behaves_like "calling api with tables and return expected result"
304
+ end
305
+
306
+ context "with less than 100 tables" do
307
+ let(:tables) { %w[a b] }
308
+ let(:expected_tables) { 'a,b' }
309
+ let(:api_result) { result1 }
310
+ let(:expected_result) { result1 }
311
+
312
+ it_behaves_like "calling api with tables and return expected result"
313
+ end
314
+
315
+ let(:result2) {
316
+ { "table_status" => [ status21, status22 ], "success" => true }
317
+ }
318
+ let(:status21) { double('status21') }
319
+ let(:status22) { double('status22') }
320
+
321
+ context "with more than 100 tables" do
322
+ let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
323
+ let(:expected_tables1) { tables[0...150].join(",") }
324
+ let(:expected_tables2) { tables[150..-1].join(",") }
325
+
326
+ it do
327
+ expect(api_client).to receive(:post).
328
+ with("/data_entries/#{data_entry_id}/table_status",
329
+ nil, tables: expected_tables1).
330
+ and_return(result1)
331
+
332
+ expect(api_client).to receive(:post).
333
+ with("/data_entries/#{data_entry_id}/table_status",
334
+ nil, tables: expected_tables2).
335
+ and_return(result2)
336
+
337
+ expect(subject).to eq(
338
+ "table_status" => [ status11, status12, status21, status22 ],
339
+ "success" => true )
340
+ end
341
+ end
342
+ end
343
+ end
344
+
345
+ end
18
346
  end
@@ -264,6 +264,10 @@ EOT
264
264
  expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
265
265
  end
266
266
 
267
+ it 'sets data tables_append_only as array' do
268
+ expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
269
+ end
270
+
267
271
  it 'sets sync_fm' do
268
272
  expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
269
273
  end
@@ -31,10 +31,12 @@ module Flydata
31
31
  end
32
32
 
33
33
  describe MysqlDumpGeneratorNoMasterData do
34
+ let(:subject_object) { described_class.new(default_conf) }
35
+
34
36
  let(:stderr) { double(:stderr) }
35
37
  let(:wait_thr) { double(:wait_thr) }
36
38
 
37
- let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
39
+ let(:default_dump_generator) { subject_object }
38
40
  let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
39
41
  let(:mysql_client) do
40
42
  m = double(:mysql_client)
@@ -80,6 +82,54 @@ module Flydata
80
82
  File.delete(file_path) if File.exists?(file_path)
81
83
  end
82
84
  end
85
+
86
+ describe '#flush_tables_with_read_lock_query' do
87
+ subject { subject_object.send(:flush_tables_with_read_lock_query,
88
+ client, database, tbls) }
89
+
90
+ let(:client) { double('client') }
91
+ let(:database) { "db1" }
92
+ let(:tbls) { ['table1'] }
93
+
94
+ before do
95
+ allow(subject_object).to receive(:mysql_server_version).
96
+ and_return(version)
97
+ end
98
+
99
+ let(:flush_tables_with_tables) { <<EOS.strip }
100
+ FLUSH TABLES `db1`.`table1` WITH READ LOCK;
101
+ EOS
102
+ let(:flush_tables_without_table) { <<EOS.strip }
103
+ FLUSH TABLES WITH READ LOCK;
104
+ EOS
105
+
106
+ context 'with MySQL 5.1' do
107
+ let(:version) { "5.1.67-log" }
108
+ it 'returns flush command without table because the syntax is only supported by MySQL 5.5 or later' do
109
+ is_expected.to eq(flush_tables_without_table)
110
+ end
111
+ end
112
+ context 'with Debian MySQL 5.5' do
113
+ let(:version) { "5.5.24-2+deb.sury.org~precise+2-log" }
114
+ it { is_expected.to eq(flush_tables_with_tables) }
115
+ end
116
+ context 'with Amazon RDS MySQL 5.6' do
117
+ let(:version) { "5.6.19-log" }
118
+ it { is_expected.to eq(flush_tables_with_tables) }
119
+ end
120
+ context 'with MariaDB 10.0' do
121
+ let(:version) { "10.0.24-MariaDB" }
122
+ it 'returns flush command with tables because MariaDB 10.0 is compatible with MySQL 5.6' do
123
+ is_expected.to eq(flush_tables_with_tables)
124
+ end
125
+ end
126
+ context 'with Amazon Aurora' do
127
+ let(:version) { "5.6.10" }
128
+ it do
129
+ is_expected.to eq(flush_tables_with_tables)
130
+ end
131
+ end
132
+ end
83
133
  end
84
134
  end
85
135
 
@@ -64,6 +64,11 @@ EOT
64
64
  allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
65
65
  r
66
66
  end
67
+ let(:unsupported_query_handler) do
68
+ r = double('unsupported_query_handler')
69
+ allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::UnsupportedQueryHandler::PATTERN)
70
+ r
71
+ end
67
72
  let(:context) { double('context') }
68
73
  let(:subject_object) { described_class.new(context) }
69
74
  subject { subject_object }
@@ -79,6 +84,7 @@ EOT
79
84
  expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
80
85
  expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
81
86
  expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
87
+ allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
82
88
  end
83
89
 
84
90
  describe '#dispatch' do
@@ -115,11 +121,26 @@ ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*applic
115
121
  /*!*/;
116
122
  SET TIMESTAMP=1415925954/*!*/;
117
123
  EOS
118
- let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
124
+ let(:normalized_query_1) { 'use `mydatabase` ;' }
125
+ let(:normalized_query_2) { <<EOS.gsub(/\n$/, '') }
119
126
  ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
120
127
  EOS
128
+ let(:normalized_query_3) { 'SET TIMESTAMP=1415925954 ;' }
129
+
130
+ let(:record) do
131
+ r = double('record')
132
+ allow(r).to receive(:[]).with("query").and_return(query)
133
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_1}).and_return(r)
134
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_2}).and_return(r)
135
+ expect(r).to receive(:merge).with({"normalized_query" => normalized_query_3}).and_return(r)
136
+ r
137
+ end
121
138
 
122
- it_behaves_like "a dispatcher that calls query handler with correct query"
139
+ it do
140
+ expect(unsupported_query_handler).to receive(:process).with(record).twice
141
+ expect(alter_query_handler).to receive(:process).with(record).once
142
+ expect(subject.dispatch(record))
143
+ end
123
144
  end
124
145
  context "query with a comment with no space" do
125
146
  let(:query) { <<EOS.gsub(/\n$/, '') }
@@ -0,0 +1,42 @@
1
+ require 'fluent_plugins_spec_helper'
2
+ require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
3
+ require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
4
+
5
+ module Flydata::SourceMysql::PluginSupport
6
+ describe UnsupportedQueryHandler do
7
+ include_context "query handler context"
8
+
9
+ describe '#process' do
10
+ before do
11
+ record['normalized_query'] = normalized_query
12
+ end
13
+
14
+ context 'when the query includes a sync table' do
15
+ let(:normalized_query) { 'drop table foo;' }
16
+ it do
17
+ expect($log).to receive(:info).with("{:type=>:unsupported_query, :query=>\"drop table foo;\"} at \"mysql-bin.000066\t180\"")
18
+ subject.process(record)
19
+ end
20
+ end
21
+
22
+ context 'when the query does not include a sync table' do
23
+ let(:normalized_query) { 'drop table bar;' }
24
+ it do
25
+ expect($log).to receive(:info).never
26
+ subject.process(record)
27
+ end
28
+ end
29
+
30
+ context 'when the query is issued in another database' do
31
+ let(:normalized_query) { 'drop table foo;' }
32
+ before do
33
+ record['db_name'] = 'another_db'
34
+ end
35
+ it do
36
+ expect($log).to receive(:info).never
37
+ subject.process(record)
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.4
4
+ version: 0.7.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-04-28 00:00:00.000000000 Z
15
+ date: 2016-06-02 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -731,6 +731,7 @@ files:
731
731
  - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
732
732
  - lib/flydata/source_mysql/plugin_support/source_position_file.rb
733
733
  - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
734
+ - lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb
734
735
  - lib/flydata/source_mysql/setup.rb
735
736
  - lib/flydata/source_mysql/source_pos.rb
736
737
  - lib/flydata/source_mysql/sync.rb
@@ -828,6 +829,7 @@ files:
828
829
  - spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
829
830
  - spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
830
831
  - spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
832
+ - spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb
831
833
  - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
832
834
  - spec/flydata/source_mysql/table_ddl_spec.rb
833
835
  - spec/flydata/source_mysql/table_meta_spec.rb
@@ -867,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
867
869
  version: '0'
868
870
  requirements: []
869
871
  rubyforge_project:
870
- rubygems_version: 2.4.3
872
+ rubygems_version: 2.0.14.1
871
873
  signing_key:
872
874
  specification_version: 4
873
875
  summary: FlyData Agent