flydata 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +0 -0
- data/lib/flydata/api/data_entry.rb +1 -1
- data/lib/flydata/command/exclusive_runnable.rb +8 -0
- data/lib/flydata/command/sync.rb +15 -1
- data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb +2 -2
- data/lib/flydata/helper/action/check_abnormal_shutdown.rb +7 -2
- data/spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb +4 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 45df6073e7d4b59b307a695058cd1ee70366eb89
|
4
|
+
data.tar.gz: cdac2f4feb8dfdd4531c14f97a38f1dd2df91f48
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df38e5c265ee0b0e4f9cfcde1a6f219db1fa0b4df013d4f79d1a9dc648d4d5684210560153d91ace7a5267cde76c4d26d00d5ee95e757c0acee2867b4ea6be9c
|
7
|
+
data.tar.gz: 7ec85d6e13314f445649d64c1769597f2fd192d35c5a6b76b42043accfca4be1d2fc5d95b7c40de7f00e462876bd67076e0102efa0634f755af4d88ebba2f2f0
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.3
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -12,7 +12,7 @@ module Flydata
|
|
12
12
|
|
13
13
|
def buffer_stat(data_entry_id, options = {})
|
14
14
|
tables = options[:tables] ? options[:tables].join(',') : ''
|
15
|
-
@client.
|
15
|
+
@client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
|
16
16
|
end
|
17
17
|
|
18
18
|
def table_status(data_entry_id, options = {})
|
@@ -74,6 +74,14 @@ module ExclusiveRunnable
|
|
74
74
|
cmd = modules.last.downcase
|
75
75
|
subcommand == :run ? cmd : "#{cmd}:#{subcommand}"
|
76
76
|
end
|
77
|
+
def command_process_exist?
|
78
|
+
exclusive_run_info = load_exclusive_run_info
|
79
|
+
if exclusive_run_info
|
80
|
+
process_exist?(exclusive_run_info['pid'])
|
81
|
+
else
|
82
|
+
false
|
83
|
+
end
|
84
|
+
end
|
77
85
|
end
|
78
86
|
end
|
79
87
|
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -338,6 +338,17 @@ EOS
|
|
338
338
|
|
339
339
|
private
|
340
340
|
|
341
|
+
def generate_kv_pairs(keys, values)
|
342
|
+
h = {}
|
343
|
+
i = 0
|
344
|
+
sz = keys.size
|
345
|
+
while i < sz
|
346
|
+
h[keys[i]] = values[i]
|
347
|
+
i += 1
|
348
|
+
end
|
349
|
+
JSON.generate(h)
|
350
|
+
end
|
351
|
+
|
341
352
|
def _check(options = {})
|
342
353
|
options[:stop_agent] ||= false
|
343
354
|
|
@@ -350,7 +361,7 @@ EOS
|
|
350
361
|
abnormal_shutdown = false
|
351
362
|
begin
|
352
363
|
begin
|
353
|
-
flush_buffer_and_stop(@full_tables, force: false, timeout:
|
364
|
+
flush_buffer_and_stop(@full_tables, force: false, timeout: 7200) # 2 hrs. It can take more but 2 hours are long enough to pull someone's attention.
|
354
365
|
rescue ServerDataProcessingTimeout => e
|
355
366
|
data_stuck_at = e.state
|
356
367
|
end
|
@@ -844,6 +855,9 @@ EOM
|
|
844
855
|
convert_to_flydata_values(mysql_table, values)
|
845
856
|
json = JSON.generate_kv_pairs(mysql_table.column_names, values)
|
846
857
|
values.clear
|
858
|
+
unless json.start_with? '{'
|
859
|
+
raise DumpParseError.new("Broken JSON record json:#{json[0..100]}")
|
860
|
+
end
|
847
861
|
{table_name: mysql_table_name, log: json}
|
848
862
|
end
|
849
863
|
# release resources to prevent the closure from keeping it
|
@@ -3,7 +3,7 @@ require 'flydata/fluent-plugins/mysql/ddl_query_handler'
|
|
3
3
|
module Mysql
|
4
4
|
class DropDatabaseQueryHandler < DatabaseDdlQueryHandler
|
5
5
|
# For MySQL, database and schema are exchangable
|
6
|
-
PATTERN = /^DROP DATABASE|SCHEMA/i
|
6
|
+
PATTERN = /^DROP (DATABASE|SCHEMA)/i
|
7
7
|
|
8
8
|
def initialize(context)
|
9
9
|
super
|
@@ -16,7 +16,7 @@ module Mysql
|
|
16
16
|
def process(record)
|
17
17
|
#Issuing warning message only for the current database.
|
18
18
|
if acceptable_db?(record) #record["db_name"] == @context.database
|
19
|
-
$log.
|
19
|
+
$log.error("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
|
20
20
|
end
|
21
21
|
#NOTE: No emit_record here because this record should not be sent to data servers for now
|
22
22
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'flydata'
|
2
|
+
require 'flydata/command/base'
|
2
3
|
require 'flydata/helper/base_action'
|
3
4
|
|
4
5
|
module Flydata
|
@@ -7,8 +8,12 @@ module Flydata
|
|
7
8
|
class CheckAbnormalShutdown < BaseAction
|
8
9
|
def execute(opts = {})
|
9
10
|
if abnormal_shutdown_occurred?
|
10
|
-
|
11
|
-
|
11
|
+
if Command::Base.command_process_exist?
|
12
|
+
log_info "Abnormal shutdown occurred but another command is stil running. Skip agent restart."
|
13
|
+
else
|
14
|
+
log_info "Abnormal shutdown occurred. Restart the agent."
|
15
|
+
yield :restart_agent, nil
|
16
|
+
end
|
12
17
|
end
|
13
18
|
end
|
14
19
|
|
@@ -21,16 +21,16 @@ module Mysql
|
|
21
21
|
let(:database) { target_database }
|
22
22
|
let(:binlog_pos) { "#{context.current_binlog_file}\t#{record['next_position'] - record['event_length']}" }
|
23
23
|
|
24
|
-
it "shows a
|
25
|
-
expect($log).to receive(:
|
24
|
+
it "shows a error message" do
|
25
|
+
expect($log).to receive(:error).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
|
26
26
|
|
27
27
|
subject
|
28
28
|
end
|
29
29
|
end
|
30
30
|
context "for another db" do
|
31
31
|
let(:database) { "another_db" }
|
32
|
-
it "does not show a
|
33
|
-
expect($log).not_to receive(:
|
32
|
+
it "does not show a error message" do
|
33
|
+
expect($log).not_to receive(:error)
|
34
34
|
subject
|
35
35
|
end
|
36
36
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-01-
|
15
|
+
date: 2016-01-15 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|