flydata 0.6.2 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ff848158232f9a2b6ad6154f56963c64ce6b41a1
4
- data.tar.gz: 1b9083d2ff0f6f8ea9647fa27a6be4b5fede2ec4
3
+ metadata.gz: 45df6073e7d4b59b307a695058cd1ee70366eb89
4
+ data.tar.gz: cdac2f4feb8dfdd4531c14f97a38f1dd2df91f48
5
5
  SHA512:
6
- metadata.gz: b1d53be1bed30298c0f8f3b49b4c9c1882a86044e28b3f3c5d29253d99c6137bcae518d9cc8e22d2e5ab0ee83c91c29cbadb6e4e7188f42a29a02cfac162f4ef
7
- data.tar.gz: 246b84391256eaeb7c97148eb9a31e3a85157b303ebc2538d70896faa175a4b570ca7f75b1a574ff9cf964a81b9c51df6ccff8aad75b8877b002e48f143de049
6
+ metadata.gz: df38e5c265ee0b0e4f9cfcde1a6f219db1fa0b4df013d4f79d1a9dc648d4d5684210560153d91ace7a5267cde76c4d26d00d5ee95e757c0acee2867b4ea6be9c
7
+ data.tar.gz: 7ec85d6e13314f445649d64c1769597f2fd192d35c5a6b76b42043accfca4be1d2fc5d95b7c40de7f00e462876bd67076e0102efa0634f755af4d88ebba2f2f0
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.2
1
+ 0.6.3
data/flydata.gemspec CHANGED
Binary file
@@ -12,7 +12,7 @@ module Flydata
12
12
 
13
13
  def buffer_stat(data_entry_id, options = {})
14
14
  tables = options[:tables] ? options[:tables].join(',') : ''
15
- @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}?tables=#{tables}")
15
+ @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
16
16
  end
17
17
 
18
18
  def table_status(data_entry_id, options = {})
@@ -74,6 +74,14 @@ module ExclusiveRunnable
74
74
  cmd = modules.last.downcase
75
75
  subcommand == :run ? cmd : "#{cmd}:#{subcommand}"
76
76
  end
77
+ def command_process_exist?
78
+ exclusive_run_info = load_exclusive_run_info
79
+ if exclusive_run_info
80
+ process_exist?(exclusive_run_info['pid'])
81
+ else
82
+ false
83
+ end
84
+ end
77
85
  end
78
86
  end
79
87
 
@@ -338,6 +338,17 @@ EOS
338
338
 
339
339
  private
340
340
 
341
+ def generate_kv_pairs(keys, values)
342
+ h = {}
343
+ i = 0
344
+ sz = keys.size
345
+ while i < sz
346
+ h[keys[i]] = values[i]
347
+ i += 1
348
+ end
349
+ JSON.generate(h)
350
+ end
351
+
341
352
  def _check(options = {})
342
353
  options[:stop_agent] ||= false
343
354
 
@@ -350,7 +361,7 @@ EOS
350
361
  abnormal_shutdown = false
351
362
  begin
352
363
  begin
353
- flush_buffer_and_stop(@full_tables, force: false, timeout: 55)
364
+ flush_buffer_and_stop(@full_tables, force: false, timeout: 7200) # 2 hrs. It can take more but 2 hours are long enough to pull someone's attention.
354
365
  rescue ServerDataProcessingTimeout => e
355
366
  data_stuck_at = e.state
356
367
  end
@@ -844,6 +855,9 @@ EOM
844
855
  convert_to_flydata_values(mysql_table, values)
845
856
  json = JSON.generate_kv_pairs(mysql_table.column_names, values)
846
857
  values.clear
858
+ unless json.start_with? '{'
859
+ raise DumpParseError.new("Broken JSON record json:#{json[0..100]}")
860
+ end
847
861
  {table_name: mysql_table_name, log: json}
848
862
  end
849
863
  # release resources to prevent the closure from keeping it
@@ -3,7 +3,7 @@ require 'flydata/fluent-plugins/mysql/ddl_query_handler'
3
3
  module Mysql
4
4
  class DropDatabaseQueryHandler < DatabaseDdlQueryHandler
5
5
  # For MySQL, database and schema are exchangable
6
- PATTERN = /^DROP DATABASE|SCHEMA/i
6
+ PATTERN = /^DROP (DATABASE|SCHEMA)/i
7
7
 
8
8
  def initialize(context)
9
9
  super
@@ -16,7 +16,7 @@ module Mysql
16
16
  def process(record)
17
17
  #Issuing warning message only for the current database.
18
18
  if acceptable_db?(record) #record["db_name"] == @context.database
19
- $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
19
+ $log.error("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record['normalized_query']}' binlog_pos:'#{binlog_pos(record)}'")
20
20
  end
21
21
  #NOTE: No emit_record here because this record should not be sent to data servers for now
22
22
  end
@@ -1,4 +1,5 @@
1
1
  require 'flydata'
2
+ require 'flydata/command/base'
2
3
  require 'flydata/helper/base_action'
3
4
 
4
5
  module Flydata
@@ -7,8 +8,12 @@ module Flydata
7
8
  class CheckAbnormalShutdown < BaseAction
8
9
  def execute(opts = {})
9
10
  if abnormal_shutdown_occurred?
10
- log_info "Abnormal shutdown occurred. Restart the agent."
11
- yield :restart_agent, nil
11
+ if Command::Base.command_process_exist?
12
+ log_info "Abnormal shutdown occurred but another command is stil running. Skip agent restart."
13
+ else
14
+ log_info "Abnormal shutdown occurred. Restart the agent."
15
+ yield :restart_agent, nil
16
+ end
12
17
  end
13
18
  end
14
19
 
@@ -21,16 +21,16 @@ module Mysql
21
21
  let(:database) { target_database }
22
22
  let(:binlog_pos) { "#{context.current_binlog_file}\t#{record['next_position'] - record['event_length']}" }
23
23
 
24
- it "shows a warning message" do
25
- expect($log).to receive(:warn).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
24
+ it "shows a error message" do
25
+ expect($log).to receive(:error).with("DROP DATABASE detected. A full re-sync is required to provide sync consistency. - db_name:'#{record["db_name"]}' query:'#{record["query"]}' normalized query:'#{record["normalized_query"]}' binlog_pos:'#{binlog_pos}'")
26
26
 
27
27
  subject
28
28
  end
29
29
  end
30
30
  context "for another db" do
31
31
  let(:database) { "another_db" }
32
- it "does not show a warning message" do
33
- expect($log).not_to receive(:warn)
32
+ it "does not show a error message" do
33
+ expect($log).not_to receive(:error)
34
34
  subject
35
35
  end
36
36
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.2
4
+ version: 0.6.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-01-08 00:00:00.000000000 Z
15
+ date: 2016-01-15 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client