flydata 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/bin/flydata +1 -0
  4. data/flydata-core/lib/flydata-core/core_ext/module.rb +1 -1
  5. data/flydata-core/lib/flydata-core/core_ext/object.rb +1 -1
  6. data/flydata.gemspec +21 -5
  7. data/lib/flydata.rb +5 -8
  8. data/lib/flydata/api/data_entry.rb +2 -0
  9. data/lib/flydata/api/data_port.rb +2 -0
  10. data/lib/flydata/api/redshift_cluster.rb +2 -0
  11. data/lib/flydata/api_client.rb +3 -0
  12. data/lib/flydata/cli.rb +13 -2
  13. data/lib/flydata/command/base.rb +6 -0
  14. data/lib/flydata/command/conf.rb +3 -0
  15. data/lib/flydata/command/crontab.rb +3 -0
  16. data/lib/flydata/command/encrypt.rb +3 -0
  17. data/lib/flydata/command/kill_all.rb +3 -0
  18. data/lib/flydata/command/login.rb +2 -0
  19. data/lib/flydata/command/restart.rb +3 -0
  20. data/lib/flydata/command/routine.rb +3 -0
  21. data/lib/flydata/command/sender.rb +2 -0
  22. data/lib/flydata/command/setlogdel.rb +4 -1
  23. data/lib/flydata/command/setup.rb +7 -2
  24. data/lib/flydata/command/start.rb +3 -0
  25. data/lib/flydata/command/status.rb +3 -0
  26. data/lib/flydata/command/stop.rb +3 -0
  27. data/lib/flydata/command/sync.rb +10 -3
  28. data/lib/flydata/command/version.rb +2 -0
  29. data/lib/flydata/{command_logger.rb → command_loggable.rb} +0 -0
  30. data/lib/flydata/compatibility_check.rb +1 -1
  31. data/lib/flydata/credentials.rb +2 -0
  32. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +8 -9
  33. data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +1 -1
  34. data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb +1 -1
  35. data/lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb +1 -1
  36. data/lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb +2 -2
  37. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +1 -1
  38. data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +1 -1
  39. data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +1 -1
  40. data/lib/flydata/helpers.rb +0 -10
  41. data/lib/flydata/heroku.rb +3 -0
  42. data/lib/flydata/output/forwarder.rb +1 -1
  43. data/lib/flydata/parser/mysql/dump_parser.rb +29 -31
  44. data/lib/flydata/sync_file_manager.rb +230 -232
  45. data/spec/fly_data_model_spec.rb +1 -0
  46. data/spec/flydata/api/data_entry_spec.rb +1 -0
  47. data/spec/flydata/api_client_spec.rb +18 -0
  48. data/spec/flydata/cli_spec.rb +1 -0
  49. data/spec/flydata/command/base_spec.rb +44 -0
  50. data/spec/flydata/command/conf_spec.rb +21 -0
  51. data/spec/flydata/command/crontab_spec.rb +17 -0
  52. data/spec/flydata/command/encrypt_spec.rb +28 -0
  53. data/spec/flydata/command/kill_all_spec.rb +17 -0
  54. data/spec/flydata/command/login_spec.rb +21 -0
  55. data/spec/flydata/command/restart_spec.rb +17 -0
  56. data/spec/flydata/command/routine_spec.rb +29 -0
  57. data/spec/flydata/command/sender_spec.rb +7 -2
  58. data/spec/flydata/command/setlogdel_spec.rb +18 -0
  59. data/spec/flydata/command/setup_spec.rb +44 -0
  60. data/spec/flydata/command/start_spec.rb +17 -0
  61. data/spec/flydata/command/status_spec.rb +17 -0
  62. data/spec/flydata/command/stop_spec.rb +17 -0
  63. data/spec/flydata/command/sync_spec.rb +1 -0
  64. data/spec/flydata/command/version_spec.rb +14 -0
  65. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +1 -1
  66. data/spec/flydata/parser/mysql/dump_parser_spec.rb +23 -73
  67. data/spec/flydata/sync_file_manager_spec.rb +150 -152
  68. metadata +19 -4
@@ -4,189 +4,187 @@ require 'flydata/sync_file_manager'
4
4
  require 'flydata/parser/mysql/dump_parser'
5
5
 
6
6
  module Flydata
7
- module FileUtil
8
- describe SyncFileManager do
9
- let(:default_mysqldump_dir) do
10
- File.join('/tmp', "sync_dump_#{Time.now.to_i}")
11
- end
12
- let(:default_data_entry) do
13
- {"id"=>93,
14
- "name"=>"flydata_sync_mysql",
15
- "data_port_id"=>52,
16
- "display_name"=>"flydata_sync_mysql",
17
- "enabled"=>true,
18
- "heroku_log_type"=>nil,
19
- "heroku_resource_id"=>nil,
20
- "log_deletion"=>nil,
21
- "log_file_delimiter"=>nil,
22
- "log_file_type"=>nil,
23
- "log_path"=>nil,
24
- "redshift_schema_name"=>"",
25
- "redshift_table_name"=>nil,
26
- "created_at"=>"2014-01-22T18:58:43Z",
27
- "updated_at"=>"2014-01-30T02:42:26Z",
28
- "type"=>"RedshiftMysqlDataEntry",
29
- "tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
30
- "tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
31
- "data_port_key"=>"a458c641",
32
- "mysql_data_entry_preference" =>
33
- { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
34
- "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
35
- "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
36
- "data_servers"=>"localhost:9905" }
37
- }
38
- end
39
- let(:default_sfm) { SyncFileManager.new(default_data_entry) }
7
+ describe SyncFileManager do
8
+ let(:default_mysqldump_dir) do
9
+ File.join('/tmp', "sync_dump_#{Time.now.to_i}")
10
+ end
11
+ let(:default_data_entry) do
12
+ {"id"=>93,
13
+ "name"=>"flydata_sync_mysql",
14
+ "data_port_id"=>52,
15
+ "display_name"=>"flydata_sync_mysql",
16
+ "enabled"=>true,
17
+ "heroku_log_type"=>nil,
18
+ "heroku_resource_id"=>nil,
19
+ "log_deletion"=>nil,
20
+ "log_file_delimiter"=>nil,
21
+ "log_file_type"=>nil,
22
+ "log_path"=>nil,
23
+ "redshift_schema_name"=>"",
24
+ "redshift_table_name"=>nil,
25
+ "created_at"=>"2014-01-22T18:58:43Z",
26
+ "updated_at"=>"2014-01-30T02:42:26Z",
27
+ "type"=>"RedshiftMysqlDataEntry",
28
+ "tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
29
+ "tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
30
+ "data_port_key"=>"a458c641",
31
+ "mysql_data_entry_preference" =>
32
+ { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
33
+ "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
34
+ "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
35
+ "data_servers"=>"localhost:9905" }
36
+ }
37
+ end
38
+ let(:default_sfm) { SyncFileManager.new(default_data_entry) }
40
39
 
41
- let (:status) { 'PARSING' }
42
- let (:table_name) { 'test_table' }
43
- let (:last_pos) { 9999 }
44
- let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
45
- let (:state) { 'CREATE_TABLE' }
46
- let (:substate) { nil }
40
+ let (:status) { 'PARSING' }
41
+ let (:table_name) { 'test_table' }
42
+ let (:last_pos) { 9999 }
43
+ let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
44
+ let (:state) { 'CREATE_TABLE' }
45
+ let (:substate) { nil }
47
46
 
48
- after :each do
49
- if Dir.exists?(default_mysqldump_dir)
50
- Dir.delete(default_mysqldump_dir) rescue nil
51
- end
52
- if File.exists?(default_mysqldump_dir)
53
- File.delete(default_mysqldump_dir) rescue nil
54
- end
47
+ after :each do
48
+ if Dir.exists?(default_mysqldump_dir)
49
+ Dir.delete(default_mysqldump_dir) rescue nil
50
+ end
51
+ if File.exists?(default_mysqldump_dir)
52
+ File.delete(default_mysqldump_dir) rescue nil
55
53
  end
54
+ end
56
55
 
57
- describe '#dump_file_path' do
58
- context 'when mysqldump_dir param is nil' do
59
- before do
60
- default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
61
- end
62
- it do
63
- stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
64
- expect(default_sfm.dump_file_path).to eq(
65
- File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
66
- end
67
- end
68
- context 'when file exists' do
69
- before { `touch #{default_mysqldump_dir}`}
70
- it do
71
- expect{default_sfm.dump_file_path}.to raise_error
72
- end
56
+ describe '#dump_file_path' do
57
+ context 'when mysqldump_dir param is nil' do
58
+ before do
59
+ default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
73
60
  end
74
- context 'when directory exists' do
75
- before { `mkdir -p #{default_mysqldump_dir}`}
76
- it do
77
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
78
- expect(default_sfm.dump_file_path).to eq(
79
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
80
- end
61
+ it do
62
+ stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
63
+ expect(default_sfm.dump_file_path).to eq(
64
+ File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
81
65
  end
82
- context 'when directory or file does not exist' do
83
- it do
84
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
85
- expect(default_sfm.dump_file_path).to eq(
86
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
87
- end
66
+ end
67
+ context 'when file exists' do
68
+ before { `touch #{default_mysqldump_dir}`}
69
+ it do
70
+ expect{default_sfm.dump_file_path}.to raise_error
88
71
  end
89
- context 'when file name includes "~"' do
90
- let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
91
- it do
92
- expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
93
- expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
94
- expect(default_sfm.dump_file_path).to eq(
95
- File.join(expected_dir, 'flydata_sync_mysql.dump'))
96
- end
72
+ end
73
+ context 'when directory exists' do
74
+ before { `mkdir -p #{default_mysqldump_dir}`}
75
+ it do
76
+ expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
77
+ expect(default_sfm.dump_file_path).to eq(
78
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
97
79
  end
98
80
  end
99
-
100
- describe '#dump_pos_path' do
101
- it { expect(default_sfm.dump_pos_path).to eq(
102
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
81
+ context 'when directory or file does not exist' do
82
+ it do
83
+ expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
84
+ expect(default_sfm.dump_file_path).to eq(
85
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
86
+ end
103
87
  end
104
-
105
- describe '#save_dump_pos' do
106
- context 'without mysql marshal data' do
107
- it do
108
- expect{default_sfm.save_dump_pos(
109
- status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
110
- expect(default_sfm.load_dump_pos).to eq({
111
- status: status, table_name: table_name, last_pos: last_pos,
112
- binlog_pos: binlog_pos, state: state, substate: substate,
113
- mysql_table: nil
114
- })
115
- end
88
+ context 'when file name includes "~"' do
89
+ let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
90
+ it do
91
+ expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
92
+ expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
93
+ expect(default_sfm.dump_file_path).to eq(
94
+ File.join(expected_dir, 'flydata_sync_mysql.dump'))
116
95
  end
117
96
  end
97
+ end
118
98
 
119
- describe '#load_dump_pos' do
120
- let (:mysql_table) do
121
- Flydata::Parser::Mysql::MysqlTable.new(
122
- table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
123
- )
124
- end
99
+ describe '#dump_pos_path' do
100
+ it { expect(default_sfm.dump_pos_path).to eq(
101
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
102
+ end
125
103
 
126
- context 'with mysql marshal data' do
127
- before do
128
- default_sfm.save_mysql_table_marshal_dump(mysql_table)
129
- default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
130
- end
131
- it do
132
- ret = default_sfm.load_dump_pos
133
- mt = ret.delete(:mysql_table)
134
- expect(ret).to eq({
104
+ describe '#save_dump_pos' do
105
+ context 'without mysql marshal data' do
106
+ it do
107
+ expect{default_sfm.save_dump_pos(
108
+ status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
109
+ expect(default_sfm.load_dump_pos).to eq({
135
110
  status: status, table_name: table_name, last_pos: last_pos,
136
111
  binlog_pos: binlog_pos, state: state, substate: substate,
112
+ mysql_table: nil
137
113
  })
138
- expect(mt.table_name).to eq(table_name)
139
- expect(mt.columns).to eq({
140
- 'id' => { format_type: 'int' },
141
- 'value' => { format_type: 'text' },
142
- })
143
- end
144
114
  end
145
115
  end
116
+ end
117
+
118
+ describe '#load_dump_pos' do
119
+ let (:mysql_table) do
120
+ Flydata::Parser::Mysql::MysqlTable.new(
121
+ table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
122
+ )
123
+ end
146
124
 
147
- describe '#save_binlog' do
148
- let(:binfile) { 'mysqlbinlog.000001' }
149
- let(:pos) { 107 }
150
- let(:binlog_pos) { {binfile: binfile, pos: pos} }
125
+ context 'with mysql marshal data' do
126
+ before do
127
+ default_sfm.save_mysql_table_marshal_dump(mysql_table)
128
+ default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
129
+ end
151
130
  it do
152
- default_sfm.save_binlog(binlog_pos)
153
- expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
131
+ ret = default_sfm.load_dump_pos
132
+ mt = ret.delete(:mysql_table)
133
+ expect(ret).to eq({
134
+ status: status, table_name: table_name, last_pos: last_pos,
135
+ binlog_pos: binlog_pos, state: state, substate: substate,
136
+ })
137
+ expect(mt.table_name).to eq(table_name)
138
+ expect(mt.columns).to eq({
139
+ 'id' => { format_type: 'int' },
140
+ 'value' => { format_type: 'text' },
141
+ })
154
142
  end
155
143
  end
144
+ end
156
145
 
157
- describe '#binlog_path' do
158
- it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
146
+ describe '#save_binlog' do
147
+ let(:binfile) { 'mysqlbinlog.000001' }
148
+ let(:pos) { 107 }
149
+ let(:binlog_pos) { {binfile: binfile, pos: pos} }
150
+ it do
151
+ default_sfm.save_binlog(binlog_pos)
152
+ expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
159
153
  end
154
+ end
155
+
156
+ describe '#binlog_path' do
157
+ it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
158
+ end
160
159
 
161
- describe '#increment_and_save_table_position' do
162
- let(:test_table) { 'test_table' }
160
+ describe '#increment_and_save_table_position' do
161
+ let(:test_table) { 'test_table' }
162
+ before do
163
+ table_pos_dir = SyncFileManager::TABLE_POSITIONS_DIR
164
+ FileUtils.mkdir_p table_pos_dir
165
+ table_pos_file = "#{table_pos_dir}/#{test_table}.pos"
166
+ FileUtils.rm(table_pos_file) if File.exist?(table_pos_file)
167
+ end
168
+ subject { default_sfm }
169
+ context 'when an exception happens in the block' do
170
+ let(:first_pos) { 1 }
171
+ let(:last_pos) { first_pos + 1 }
163
172
  before do
164
- table_pos_dir = SyncFileManager::TABLE_POSITIONS_DIR
165
- FileUtils.mkdir_p table_pos_dir
166
- table_pos_file = "#{table_pos_dir}/#{test_table}.pos"
167
- FileUtils.rm(table_pos_file) if File.exist?(table_pos_file)
168
- end
169
- subject { default_sfm }
170
- context 'when an exception happens in the block' do
171
- let(:first_pos) { 1 }
172
- let(:last_pos) { first_pos + 1 }
173
- before do
173
+ subject.increment_and_save_table_position(test_table) do |seq|
174
+ expect(seq).to eq first_pos
175
+ end
176
+ begin
174
177
  subject.increment_and_save_table_position(test_table) do |seq|
175
- expect(seq).to eq first_pos
178
+ expect(seq).to eq last_pos
179
+ raise "error"
176
180
  end
177
- begin
178
- subject.increment_and_save_table_position(test_table) do |seq|
179
- expect(seq).to eq last_pos
180
- raise "error"
181
- end
182
- rescue
183
- # intentionally left blank
184
- end
185
- end
186
- it 'yields with the previous value' do
187
- expect{|b| subject.increment_and_save_table_position(test_table, &b)}.to yield_with_args(last_pos)
181
+ rescue
182
+ # intentionally left blank
188
183
  end
189
184
  end
185
+ it 'yields with the previous value' do
186
+ expect{|b| subject.increment_and_save_table_position(test_table, &b)}.to yield_with_args(last_pos)
187
+ end
190
188
  end
191
189
  end
192
190
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-02-28 00:00:00.000000000 Z
15
+ date: 2015-03-04 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -514,7 +514,7 @@ files:
514
514
  - lib/flydata/command/stop.rb
515
515
  - lib/flydata/command/sync.rb
516
516
  - lib/flydata/command/version.rb
517
- - lib/flydata/command_logger.rb
517
+ - lib/flydata/command_loggable.rb
518
518
  - lib/flydata/compatibility_check.rb
519
519
  - lib/flydata/credentials.rb
520
520
  - lib/flydata/cron.rb
@@ -552,9 +552,24 @@ files:
552
552
  - spec/fluent_plugins_spec_helper.rb
553
553
  - spec/fly_data_model_spec.rb
554
554
  - spec/flydata/api/data_entry_spec.rb
555
+ - spec/flydata/api_client_spec.rb
555
556
  - spec/flydata/cli_spec.rb
557
+ - spec/flydata/command/base_spec.rb
558
+ - spec/flydata/command/conf_spec.rb
559
+ - spec/flydata/command/crontab_spec.rb
560
+ - spec/flydata/command/encrypt_spec.rb
561
+ - spec/flydata/command/kill_all_spec.rb
562
+ - spec/flydata/command/login_spec.rb
563
+ - spec/flydata/command/restart_spec.rb
564
+ - spec/flydata/command/routine_spec.rb
556
565
  - spec/flydata/command/sender_spec.rb
566
+ - spec/flydata/command/setlogdel_spec.rb
567
+ - spec/flydata/command/setup_spec.rb
568
+ - spec/flydata/command/start_spec.rb
569
+ - spec/flydata/command/status_spec.rb
570
+ - spec/flydata/command/stop_spec.rb
557
571
  - spec/flydata/command/sync_spec.rb
572
+ - spec/flydata/command/version_spec.rb
558
573
  - spec/flydata/compatibility_check_spec.rb
559
574
  - spec/flydata/fluent-plugins/idle_event_detector_spec.rb
560
575
  - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
@@ -592,7 +607,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
592
607
  version: '0'
593
608
  requirements: []
594
609
  rubyforge_project:
595
- rubygems_version: 2.0.14
610
+ rubygems_version: 2.4.3
596
611
  signing_key:
597
612
  specification_version: 4
598
613
  summary: FlyData Agent