filecluster 0.4.22 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/fc-daemon +1 -1
- data/bin/fc-manage +1 -1
- data/bin/fc-setup-db +1 -1
- data/lib/fc/base.rb +1 -1
- data/lib/fc/db.rb +77 -41
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +2 -2
- data/test/daemon_test.rb +7 -2
- data/test/db_test.rb +55 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0a47f34292388667e2040dfa0cee212503869229
|
4
|
+
data.tar.gz: d69557e79d7ba8365d9aa48c58e678e0ddf6a23e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 83ea94afcc5c96f8066f71310447744685839dbd5ef657928a5212519f0404b9ea4f1036a96ed922918b77cdfc2c6a5630e64a6c6b04c90bac6ae91941182f9b
|
7
|
+
data.tar.gz: 74aaf8eb6378e07202d6bc10a9d94c66d0df54e7290af322fac9836f0ac30e43a3ffd5b63cfb8831251ae5a0d3070fc00dfb78de4f4db9635e0293091c03197d
|
data/bin/fc-daemon
CHANGED
@@ -25,7 +25,7 @@ $update_tasks_thread = nil
|
|
25
25
|
$run_tasks_thread = nil
|
26
26
|
|
27
27
|
args = ARGV.clone
|
28
|
-
default_db_config =
|
28
|
+
default_db_config = FC::DB.options_yml_path
|
29
29
|
descriptions = {
|
30
30
|
:config => {:short => 'c', :full => 'config', :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
|
31
31
|
:log_level => {:short => 'l', :full => 'log_level', :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},
|
data/bin/fc-manage
CHANGED
@@ -8,7 +8,7 @@ require 'filecluster'
|
|
8
8
|
require 'utils'
|
9
9
|
require 'manage'
|
10
10
|
|
11
|
-
default_db_config =
|
11
|
+
default_db_config = FC::DB.options_yml_path
|
12
12
|
descriptions = {
|
13
13
|
:config => {:short => 'c', :full => 'config', :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
|
14
14
|
:curr_host => {:short => 'h', :full => 'host', :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
|
data/bin/fc-setup-db
CHANGED
@@ -27,7 +27,7 @@ options = option_parser_init(descriptions, desc)
|
|
27
27
|
trap("INT", proc {exit})
|
28
28
|
|
29
29
|
if !options[:__keys][:host] && !options[:__keys][:database] && !options[:__keys][:username] && !options[:__keys][:password] && !options[:__keys][:port] && !options[:__keys][:prefix]
|
30
|
-
default_db_config =
|
30
|
+
default_db_config = FC::DB.options_yml_path
|
31
31
|
if File.exists?(default_db_config)
|
32
32
|
db_options = Psych.load(File.read(default_db_config))
|
33
33
|
options.merge!(db_options)
|
data/lib/fc/base.rb
CHANGED
data/lib/fc/db.rb
CHANGED
@@ -1,8 +1,15 @@
|
|
1
1
|
require 'mysql2'
|
2
|
+
require 'psych'
|
2
3
|
|
3
4
|
module FC
|
4
5
|
module DB
|
5
|
-
class << self
|
6
|
+
class << self
|
7
|
+
attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.options_yml_path
|
11
|
+
File.expand_path(File.dirname(__FILE__) + '../../../bin/db.yml')
|
12
|
+
end
|
6
13
|
|
7
14
|
def self.connect_by_config(options)
|
8
15
|
@options = options.clone
|
@@ -11,45 +18,67 @@ module FC
|
|
11
18
|
@connects = {} unless @connects
|
12
19
|
@connects[Thread.current.object_id] = Mysql2::Client.new(@options)
|
13
20
|
end
|
14
|
-
|
15
|
-
def self.
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
else
|
24
|
-
connection = ActiveRecord::Base.connection.instance_variable_get(:@connection)
|
25
|
-
end
|
26
|
-
@options = connection.query_options.clone
|
27
|
-
@options.merge!(options)
|
28
|
-
@prefix = @options[:prefix].to_s if @options[:prefix]
|
29
|
-
@connects = {} unless @connects
|
30
|
-
@connects[Thread.current.object_id] = connection
|
31
|
-
end
|
21
|
+
|
22
|
+
def self.connect_by_yml(options = {})
|
23
|
+
db_options = Psych.load(File.read(options_yml_path))
|
24
|
+
connect_by_config(db_options.merge(options))
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.connect_by_active_record(options = {})
|
28
|
+
if defined?(Octopus::Proxy) && ActiveRecord::Base.connection.is_a?(Octopus::Proxy)
|
29
|
+
connection = ActiveRecord::Base.connection.select_connection.instance_variable_get(:@connection)
|
32
30
|
else
|
33
|
-
|
31
|
+
connection = ActiveRecord::Base.connection.instance_variable_get(:@connection)
|
34
32
|
end
|
35
|
-
|
36
|
-
|
33
|
+
@options = connection.query_options.clone
|
34
|
+
@options.merge!(options)
|
35
|
+
@prefix = @options[:prefix].to_s if @options[:prefix]
|
36
|
+
@connects = {} unless @connects
|
37
|
+
@connects[Thread.current.object_id] = connection
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.lazy_connect(&block)
|
41
|
+
@connect_block = block
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.connect_by_block(options = {})
|
45
|
+
connection = @connect_block.call
|
46
|
+
@options = connection.query_options.clone.merge(options)
|
47
|
+
@prefix = @options[:prefix].to_s if @options[:prefix]
|
48
|
+
@connects = {} unless @connects
|
49
|
+
@connects[Thread.current.object_id] = connection
|
50
|
+
@connect_block = nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.connect
|
54
|
+
connect_by_block if @connect_block
|
55
|
+
return nil unless @options
|
56
|
+
connect_by_config(@options) if @options[:multi_threads] && !@connects[Thread.current.object_id]
|
57
|
+
if @options[:multi_threads]
|
58
|
+
@connects[Thread.current.object_id]
|
37
59
|
else
|
38
|
-
@connects.first[1]
|
60
|
+
@connects.first && @connects.first[1]
|
39
61
|
end
|
40
62
|
end
|
41
63
|
|
42
64
|
def self.connect!(options = {})
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
65
|
+
close if @connects && @connects[Thread.current.object_id]
|
66
|
+
if @connect_block
|
67
|
+
connect_by_block(options)
|
68
|
+
elsif options[:host] || options[:database] || options[:username] || options[:password]
|
69
|
+
connect_by_config(options)
|
70
|
+
elsif @options
|
71
|
+
connect_by_config(@options.merge(options))
|
72
|
+
elsif !@no_active_record && defined?(ActiveRecord::Base) && ActiveRecord::Base.connection
|
73
|
+
connect_by_active_record(options)
|
74
|
+
else
|
75
|
+
connect_by_yml(options)
|
76
|
+
end
|
49
77
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
78
|
+
|
79
|
+
def self.reconnect
|
80
|
+
close if connect
|
81
|
+
connect_by_config(@options)
|
53
82
|
end
|
54
83
|
|
55
84
|
def self.close
|
@@ -60,35 +89,42 @@ module FC
|
|
60
89
|
end
|
61
90
|
else
|
62
91
|
@connects.first[1].close
|
92
|
+
@connects.clear
|
63
93
|
end
|
64
94
|
end
|
65
95
|
|
66
96
|
# connect.query with deadlock solution
|
67
97
|
def self.query(sql)
|
68
98
|
raise 'Too many mysql errors' if FC::DB.err_counter && FC::DB.err_counter > 10
|
99
|
+
t1 = Time.new.to_f
|
69
100
|
r = FC::DB.connect.query(sql)
|
101
|
+
t2 = Time.new.to_f
|
102
|
+
@logger.debug(format('FC SQL (%.1fms) %s', (t2 - t1) * 1000, sql)) if @logger
|
70
103
|
FC::DB.err_counter = 0
|
71
|
-
r = r.each(:as => :hash){} if r
|
104
|
+
r = r.each(:as => :hash) {} if r
|
72
105
|
r
|
73
106
|
rescue Mysql2::Error => e
|
74
107
|
FC::DB.err_counter = FC::DB.err_counter.to_i + 1
|
75
108
|
if e.message.match('Deadlock found when trying to get lock')
|
76
|
-
|
109
|
+
msg = "#{e.message} - retry"
|
110
|
+
@logger ? @logger.error(msg) : puts(msg)
|
77
111
|
sleep 0.1
|
78
|
-
|
112
|
+
query(sql)
|
79
113
|
elsif e.message.match('Lost connection to MySQL server during query')
|
80
|
-
|
114
|
+
msg = "#{e.message} - reconnect"
|
115
|
+
@logger ? @logger.error(msg) : puts(msg)
|
81
116
|
FC::DB.connect.ping
|
82
117
|
sleep 0.1
|
83
|
-
|
118
|
+
query(sql)
|
84
119
|
elsif @options[:reconnect]
|
85
|
-
|
86
|
-
|
87
|
-
|
120
|
+
msg = "#{e.message} - reconnect"
|
121
|
+
@logger ? @logger.info(msg) : puts(msg)
|
122
|
+
reconnect
|
123
|
+
query(sql)
|
88
124
|
else
|
89
125
|
raise e
|
90
126
|
end
|
91
|
-
end
|
127
|
+
end
|
92
128
|
|
93
129
|
def self.server_time
|
94
130
|
FC::DB.query("SELECT UNIX_TIMESTAMP() as curr_time").first['curr_time'].to_i
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -109,7 +109,7 @@ def storages_update_size
|
|
109
109
|
print "Calc current size.. "
|
110
110
|
size = storage.file_size('', true)
|
111
111
|
storage.size = size
|
112
|
-
FC::DB.
|
112
|
+
FC::DB.reconnect
|
113
113
|
begin
|
114
114
|
storage.save
|
115
115
|
rescue Exception => e
|
@@ -244,7 +244,7 @@ def make_storages_sync(storage, make_delete, silent = false, no_reconnect = fals
|
|
244
244
|
process_storage_dir_sync.call
|
245
245
|
|
246
246
|
# rm delete_files
|
247
|
-
FC::DB.
|
247
|
+
FC::DB.reconnect unless no_reconnect
|
248
248
|
if make_delete
|
249
249
|
puts "Deleting files" unless silent
|
250
250
|
delete_files.each do |f|
|
data/test/daemon_test.rb
CHANGED
@@ -77,6 +77,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
77
77
|
end
|
78
78
|
|
79
79
|
should "daemon_all" do
|
80
|
+
puts 'Start' if @debug
|
80
81
|
@@storages.each {|storage| storage.reload}
|
81
82
|
assert @@storages[0].up?, "Storage #{@@storages[0].name} down"
|
82
83
|
assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
|
@@ -90,9 +91,10 @@ class DaemonTest < Test::Unit::TestCase
|
|
90
91
|
|
91
92
|
@@policy.copies = 3
|
92
93
|
@@policy.save
|
93
|
-
|
94
|
-
|
94
|
+
|
95
95
|
# wait for copy
|
96
|
+
sleep 2
|
97
|
+
puts 'Check copy' if @debug
|
96
98
|
[1, 2, 3].each do |i|
|
97
99
|
['b', 'c'].each do |j|
|
98
100
|
assert_equal `du -sb /tmp/host1-sda/bla/bla/test$i 2>&1`.to_i, `du -sb /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
|
@@ -105,7 +107,9 @@ class DaemonTest < Test::Unit::TestCase
|
|
105
107
|
item_storage = FC::ItemStorage.where('item_id = ? AND storage_name = ?', @item1.id, 'host1-sdc').first
|
106
108
|
item_storage.status = 'delete'
|
107
109
|
item_storage.save
|
110
|
+
|
108
111
|
sleep 2
|
112
|
+
puts 'Check delete' if @debug
|
109
113
|
assert_equal 0, `du -sb /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
|
110
114
|
assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
|
111
115
|
|
@@ -118,6 +122,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
118
122
|
@item3.save
|
119
123
|
|
120
124
|
sleep 6
|
125
|
+
puts 'Check mark_deleted' if @debug
|
121
126
|
assert_raise(RuntimeError, "Item not deleted after mark_deleted") {@item1.reload}
|
122
127
|
assert_equal 0, FC::ItemStorage.where('item_id = ?', @item2.id).count, "ItemStorages not deleted after status='error'"
|
123
128
|
@item3.reload
|
data/test/db_test.rb
CHANGED
@@ -51,6 +51,61 @@ class DbTest < Test::Unit::TestCase
|
|
51
51
|
@item_storage = @item_storages.first
|
52
52
|
@item_storage2 = @item_storages[1]
|
53
53
|
end
|
54
|
+
|
55
|
+
should 'sql logger' do
|
56
|
+
FC::DB.logger = mock
|
57
|
+
FC::DB.logger.expects(:debug).at_least_once
|
58
|
+
FC::DB.query 'SELECT 1'
|
59
|
+
FC::DB.logger = nil
|
60
|
+
end
|
61
|
+
|
62
|
+
should 'close, reconnect and connect_by_yml' do
|
63
|
+
db_config_file = File.expand_path(File.dirname(__FILE__)) + '/db_test.yml'
|
64
|
+
File.open(db_config_file, 'w') do |f|
|
65
|
+
f.write(FC::DB.options.to_yaml)
|
66
|
+
end
|
67
|
+
|
68
|
+
connect = FC::DB.connect
|
69
|
+
FC::DB.close
|
70
|
+
assert_false connect.ping, 'Mysql2 connect not closed after close call'
|
71
|
+
assert_nil FC::DB.connect, 'FC::DB.connect is not empty after close call'
|
72
|
+
|
73
|
+
FC::DB.stubs(:options_yml_path).returns(db_config_file)
|
74
|
+
FC::DB.connect_by_yml
|
75
|
+
assert_true FC::DB.connect.ping, 'Not connected after connect_by_yml'
|
76
|
+
|
77
|
+
FC::DB.close
|
78
|
+
FC::DB.reconnect
|
79
|
+
assert_true FC::DB.connect.ping, 'Not connected after reconnect'
|
80
|
+
end
|
81
|
+
|
82
|
+
should 'lazy_connect' do
|
83
|
+
FC::DB.close
|
84
|
+
FC::DB.lazy_connect do
|
85
|
+
Mysql2::Client.new(FC::DB.options)
|
86
|
+
end
|
87
|
+
FC::DB.connect!(:multi_threads => true)
|
88
|
+
assert_true FC::DB.connect.ping, 'Not connected after lazy_connect with Mysql2'
|
89
|
+
assert_true FC::DB.options[:multi_threads], 'Options from connect!(options) was not setted'
|
90
|
+
|
91
|
+
FC::DB.close
|
92
|
+
FC::DB.lazy_connect do
|
93
|
+
FC::DB.connect_by_config(FC::DB.options)
|
94
|
+
end
|
95
|
+
assert_true FC::DB.connect.ping, 'Not connected after lazy_connect with FC::DB.connect_by_config'
|
96
|
+
end
|
97
|
+
|
98
|
+
should 'multi threads' do
|
99
|
+
FC::DB.connect!(:multi_threads => true)
|
100
|
+
threads = Array.new(5) do
|
101
|
+
Thread.new do
|
102
|
+
assert_nothing_raised { FC::DB.query('select sleep(0.05)') }
|
103
|
+
end
|
104
|
+
end
|
105
|
+
threads.each(&:join)
|
106
|
+
assert_equal FC::DB.instance_variable_get(:@connects).keys.count, 6
|
107
|
+
FC::DB.connect!(:multi_threads => false)
|
108
|
+
end
|
54
109
|
|
55
110
|
should "items" do
|
56
111
|
assert @items.count > 0, 'Items not loaded'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-04-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|