filecluster 0.3.12 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/fc/db.rb +6 -1
- data/lib/fc/item.rb +3 -3
- data/lib/fc/policy.rb +21 -3
- data/lib/fc/storage.rb +12 -0
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +8 -3
- data/test/functional_test.rb +13 -6
- data/test/policy_test.rb +40 -20
- data/test/storage_test.rb +8 -0
- metadata +4 -4
data/lib/fc/db.rb
CHANGED
@@ -43,7 +43,12 @@ module FC
|
|
43
43
|
puts "Deadlock"
|
44
44
|
sleep 0.1
|
45
45
|
self.query(sql)
|
46
|
-
|
46
|
+
elsif e.message.match('Lost connection to MySQL server during query')
|
47
|
+
puts "Lost connection to MySQL server during query"
|
48
|
+
FC::DB.connect.ping
|
49
|
+
sleep 0.1
|
50
|
+
self.query(sql)
|
51
|
+
else
|
47
52
|
raise e
|
48
53
|
end
|
49
54
|
end
|
data/lib/fc/item.rb
CHANGED
@@ -26,7 +26,7 @@ module FC
|
|
26
26
|
|
27
27
|
if local_path.include?(item_name) && !options[:not_local]
|
28
28
|
storage = policy.get_create_storages.detect do |s|
|
29
|
-
s.host == FC::Storage.curr_host && local_path.index(s.path) == 0 && local_path.sub(s.path, '') == item_params[:name]
|
29
|
+
s.host == FC::Storage.curr_host && local_path.index(s.path) == 0 && local_path.sub(s.path, '').sub(/\/$/, '').sub(/^\//, '') == item_params[:name]
|
30
30
|
end
|
31
31
|
FC::Error.raise "local_path #{local_path} is not valid path for policy ##{policy.id}" unless storage
|
32
32
|
end
|
@@ -48,8 +48,8 @@ module FC
|
|
48
48
|
if storage
|
49
49
|
item_storage = item.make_item_storage(storage, 'ready')
|
50
50
|
item.reload
|
51
|
-
else
|
52
|
-
storage = policy.get_proper_storage_for_create(item.size)
|
51
|
+
else
|
52
|
+
storage = policy.get_proper_storage_for_create(item.size, File.realdirpath(local_path))
|
53
53
|
FC::Error.raise 'No available storage', :item_id => item.id unless storage
|
54
54
|
|
55
55
|
# mark delete item_storages on replace
|
data/lib/fc/policy.rb
CHANGED
@@ -35,9 +35,27 @@ module FC
|
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
-
# get available storage for create by size
|
39
|
-
def get_proper_storage_for_create(size,
|
40
|
-
get_proper_storages_for_create(size
|
38
|
+
# get available storage for create by size and local item path
|
39
|
+
def get_proper_storage_for_create(size, local_path = nil)
|
40
|
+
storages = get_proper_storages_for_create(size)
|
41
|
+
# sort by current_host and free size
|
42
|
+
storages.sort do |a, b|
|
43
|
+
if FC::Storage.curr_host == a.host && FC::Storage.curr_host == b.host
|
44
|
+
if local_path && local_path.index(a.path) == 0
|
45
|
+
1
|
46
|
+
elsif local_path && local_path.index(b.path) == 0
|
47
|
+
-1
|
48
|
+
else
|
49
|
+
a.free_rate <=> b.free_rate
|
50
|
+
end
|
51
|
+
elsif FC::Storage.curr_host == a.host
|
52
|
+
1
|
53
|
+
elsif FC::Storage.curr_host == b.host
|
54
|
+
-1
|
55
|
+
else
|
56
|
+
a.free_rate <=> b.free_rate
|
57
|
+
end
|
58
|
+
end.last
|
41
59
|
end
|
42
60
|
end
|
43
61
|
end
|
data/lib/fc/storage.rb
CHANGED
@@ -26,6 +26,18 @@ module FC
|
|
26
26
|
super params
|
27
27
|
end
|
28
28
|
|
29
|
+
def free
|
30
|
+
size_limit - size
|
31
|
+
end
|
32
|
+
|
33
|
+
def size_rate
|
34
|
+
size.to_f / size_limit
|
35
|
+
end
|
36
|
+
|
37
|
+
def free_rate
|
38
|
+
free.to_f / size_limit
|
39
|
+
end
|
40
|
+
|
29
41
|
def get_copy_storages
|
30
42
|
self.class.get_copy_storages_mutex.synchronize do
|
31
43
|
unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -8,6 +8,7 @@ def storages_list
|
|
8
8
|
else
|
9
9
|
storages.each do |storage|
|
10
10
|
str = "#{colorize_string(storage.host, :yellow)} #{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
|
11
|
+
str += "#{(storage.free_rate*100).to_i}% free "
|
11
12
|
str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
|
12
13
|
str += " #{storage.check_time_delay} seconds ago" if storage.check_time
|
13
14
|
puts str
|
@@ -23,7 +24,8 @@ def storages_show
|
|
23
24
|
Host: #{storage.host}
|
24
25
|
Path: #{storage.path}
|
25
26
|
Url: #{storage.url}
|
26
|
-
Size: #{size_to_human storage.size}
|
27
|
+
Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
|
28
|
+
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
27
29
|
Size limit: #{size_to_human storage.size_limit}
|
28
30
|
Copy storages: #{storage.copy_storages}
|
29
31
|
Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
|
@@ -53,12 +55,14 @@ def storages_add
|
|
53
55
|
puts "Error: #{e.message}"
|
54
56
|
exit
|
55
57
|
end
|
58
|
+
free = size_limit - size
|
56
59
|
puts %Q{\nStorage
|
57
60
|
Name: #{name}
|
58
61
|
Host: #{host}
|
59
62
|
Path: #{path}
|
60
63
|
Url: #{url}
|
61
|
-
Size: #{size_to_human size}
|
64
|
+
Size: #{size_to_human size} (#{(size.to_f*100 / size_limit).to_i}%)
|
65
|
+
Free: #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
|
62
66
|
Size limit: #{size_to_human size_limit}
|
63
67
|
Copy storages #{copy_storages}}
|
64
68
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
@@ -135,7 +139,8 @@ def storages_change
|
|
135
139
|
Host: #{storage.host}
|
136
140
|
Path: #{storage.path}
|
137
141
|
Url: #{storage.url}
|
138
|
-
Size: #{size_to_human storage.size}
|
142
|
+
Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
|
143
|
+
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
139
144
|
Size limit: #{size_to_human storage.size_limit}
|
140
145
|
Copy storages: #{storage.copy_storages}}
|
141
146
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
data/test/functional_test.rb
CHANGED
@@ -16,11 +16,11 @@ class FunctionalTest < Test::Unit::TestCase
|
|
16
16
|
`cp #{@@test_file_path.shellescape} #{@@test_dir_path.shellescape}/bbb/test2`
|
17
17
|
|
18
18
|
@@storages = []
|
19
|
-
@@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
|
20
|
-
@@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
|
21
|
-
@@storages << FC::Storage.new(:name => 'host2-sda', :host => 'host2', :path => '/tmp/host2-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
|
22
|
-
@@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
|
23
|
-
@@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size_limit => 1000000)
|
19
|
+
@@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
|
20
|
+
@@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
|
21
|
+
@@storages << FC::Storage.new(:name => 'host2-sda', :host => 'host2', :path => '/tmp/host2-sda/', :size => 10, :size_limit => 1000000, :check_time => Time.new.to_i)
|
22
|
+
@@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size => 10, :size_limit => 1000000, :check_time => Time.new.to_i)
|
23
|
+
@@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size => 100, :size_limit => 1000000)
|
24
24
|
@@storages.each { |storage| storage.save}
|
25
25
|
|
26
26
|
@@policies = []
|
@@ -113,6 +113,13 @@ class FunctionalTest < Test::Unit::TestCase
|
|
113
113
|
should "item create_from_local inplace" do
|
114
114
|
tmp_file_path = "/tmp/host2-sda/inplace test"
|
115
115
|
`cp #{@@test_file_path.shellescape} #{tmp_file_path.shellescape}`
|
116
|
-
assert_nothing_raised {
|
116
|
+
assert_nothing_raised { FC::Item.create_from_local(tmp_file_path, 'inplace test', @@policies[0]) }
|
117
|
+
end
|
118
|
+
|
119
|
+
should "item create_from_local inplace for dir" do
|
120
|
+
tmp_dir_path = "/tmp/host2-sda/inplace test dir/"
|
121
|
+
`mkdir #{tmp_dir_path.shellescape}`
|
122
|
+
`cp #{@@test_file_path.shellescape} #{tmp_dir_path.shellescape}`
|
123
|
+
assert_nothing_raised { FC::Item.create_from_local(tmp_dir_path, '/inplace test dir/', @@policies[0]) }
|
117
124
|
end
|
118
125
|
end
|
data/test/policy_test.rb
CHANGED
@@ -4,14 +4,16 @@ class PolicyTest < Test::Unit::TestCase
|
|
4
4
|
class << self
|
5
5
|
def startup
|
6
6
|
@@storages = []
|
7
|
-
@@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 10)
|
8
|
-
@@storages << FC::Storage.new(:name => '
|
9
|
-
@@storages << FC::Storage.new(:name => 'rec2-
|
7
|
+
@@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :path => '/tmp/host1-sda/', :size => 0, :size_limit => 10)
|
8
|
+
@@storages << FC::Storage.new(:name => 'rec1-sdb', :host => 'rec1', :path => '/tmp/host1-sdb/', :size => 0, :size_limit => 10)
|
9
|
+
@@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :path => '/tmp/host2-sda/', :size => 30, :size_limit => 100)
|
10
|
+
@@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :path => '/tmp/host2-sdb/', :size => 20, :size_limit => 100)
|
11
|
+
@@storages << FC::Storage.new(:name => 'rec2-sdc', :host => 'rec2', :path => '/tmp/host2-sdc/', :size => 10, :size_limit => 100)
|
12
|
+
@@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :path => '/tmp/host3-sda/', :size => 99, :size_limit => 1000)
|
13
|
+
@@storages << FC::Storage.new(:name => 'rec3-sdb', :host => 'rec3', :path => '/tmp/host3-sdb/', :size => 199, :size_limit => 1000)
|
10
14
|
@@storages.each {|storage| storage.save}
|
11
|
-
|
12
|
-
@@
|
13
|
-
|
14
|
-
@@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
|
15
|
+
|
16
|
+
@@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb,rec2-sdc,rec3-sda,rec3-sdb', :copies => 1, :name => 'policy 1')
|
15
17
|
@@policy.save
|
16
18
|
end
|
17
19
|
def shutdown
|
@@ -22,32 +24,50 @@ class PolicyTest < Test::Unit::TestCase
|
|
22
24
|
|
23
25
|
should "get_create_storages" do
|
24
26
|
FC::Policy.storages_cache_time = 10
|
25
|
-
|
27
|
+
assert_equal 6, @@policy.get_create_storages.size
|
26
28
|
@@policy.create_storages = 'rec1-sda,rec2-sda'
|
27
29
|
@@policy.save
|
28
|
-
assert_equal
|
30
|
+
assert_equal 6, @@policy.get_create_storages.size
|
29
31
|
FC::Policy.storages_cache_time = 0
|
30
32
|
assert_equal 2, @@policy.get_create_storages.size
|
33
|
+
@@policy.create_storages = 'rec1-sda,rec2-sda,rec2-sdb,rec2-sdc,rec3-sda,rec3-sdb'
|
34
|
+
@@policy.save
|
35
|
+
end
|
36
|
+
|
37
|
+
should "filter_by_host" do
|
38
|
+
FC::Storage.stubs(:curr_host).returns('rec2')
|
39
|
+
FC::Policy.new(:create_storages => 'rec3-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 2').save
|
40
|
+
FC::Policy.new(:create_storages => 'rec1-sda,rec3-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 3').save
|
41
|
+
assert_same_elements ['policy 1', 'policy 2'], FC::Policy.filter_by_host.map{|p| p.name}
|
31
42
|
end
|
32
43
|
|
33
44
|
should "get_proper_storage_for_create" do
|
34
|
-
@@storages.each {|storage| storage.check_time = 0; storage.save}
|
35
|
-
@@storage3.check_time = 0
|
36
|
-
@@storage3.save
|
37
45
|
FC::Policy.storages_cache_time = 0
|
46
|
+
@@storages.each {|storage| storage.check_time = 0; storage.save}
|
38
47
|
assert_nil @@policy.get_proper_storage_for_create(1), 'all storages down'
|
39
48
|
@@storages[0].update_check_time
|
40
|
-
|
49
|
+
assert_kind_of FC::Storage, @@policy.get_proper_storage_for_create(5), 'first storages up'
|
50
|
+
assert_equal 'rec1-sda', @@policy.get_proper_storage_for_create(5).name, 'first storages up'
|
41
51
|
assert_nil @@policy.get_proper_storage_for_create(20), 'first storage full'
|
42
|
-
@@storages[
|
43
|
-
assert_equal
|
52
|
+
@@storages[2].update_check_time
|
53
|
+
assert_equal 'rec2-sda', @@policy.get_proper_storage_for_create(20).name, 'second storages up'
|
44
54
|
assert_nil @@policy.get_proper_storage_for_create(1000), 'all storages full'
|
45
55
|
end
|
46
56
|
|
47
|
-
should "
|
57
|
+
should "get_proper_storage_for_create for local path" do
|
58
|
+
FC::Policy.storages_cache_time = 0
|
59
|
+
@@storages.each {|storage| storage.update_check_time}
|
48
60
|
FC::Storage.stubs(:curr_host).returns('rec2')
|
49
|
-
|
50
|
-
|
51
|
-
|
61
|
+
assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_create(9, '/tmp/host2-sdb/test').name, 'current host, local path match'
|
62
|
+
@@storages[3].check_time = 0;
|
63
|
+
@@storages[3].save
|
64
|
+
assert_equal 'rec2-sdc', @@policy.get_proper_storage_for_create(9, '/tmp/host2-sdb/test').name, 'current host, most free storage'
|
65
|
+
FC::Storage.stubs(:curr_host).returns('rec3')
|
66
|
+
@@storages[5].check_time = 0;
|
67
|
+
@@storages[5].save
|
68
|
+
assert_equal 'rec3-sdb', @@policy.get_proper_storage_for_create(9, '/tmp/host3-sdc/test').name, 'current host, single storage'
|
69
|
+
FC::Storage.stubs(:curr_host).returns('rec5')
|
70
|
+
assert_equal 'rec1-sda', @@policy.get_proper_storage_for_create(9, '/tmp/host3-sdc/test').name, 'not current host, most free storage'
|
71
|
+
assert_equal 'rec2-sdc', @@policy.get_proper_storage_for_create(10, '/tmp/host3-sdc/test').name, 'not current host, big file, most free storage with free space'
|
52
72
|
end
|
53
|
-
end
|
73
|
+
end
|
data/test/storage_test.rb
CHANGED
@@ -69,4 +69,12 @@ class StorageTest < Test::Unit::TestCase
|
|
69
69
|
assert_equal 'rec2-sda', @@storages[2].get_proper_storage_for_copy(20).name, 'second storages up, big file'
|
70
70
|
assert_nil @@storages[2].get_proper_storage_for_copy(1000), 'second storages up, huge file'
|
71
71
|
end
|
72
|
+
|
73
|
+
should "free and rate" do
|
74
|
+
storage = @@storages[0]
|
75
|
+
storage.size = 1
|
76
|
+
assert_equal 9, storage.free, 'first storage free must be size_limit - size'
|
77
|
+
assert_equal 9.to_f/10, storage.free_rate, 'first storage free_rate must be free/size_limit'
|
78
|
+
assert_equal 1.to_f/10, storage.size_rate, 'first storage free must be size/size_limit'
|
79
|
+
end
|
72
80
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-11-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mysql2
|
@@ -181,7 +181,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
181
181
|
version: '0'
|
182
182
|
segments:
|
183
183
|
- 0
|
184
|
-
hash: -
|
184
|
+
hash: -1129513891376945174
|
185
185
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
186
186
|
none: false
|
187
187
|
requirements:
|
@@ -190,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
190
190
|
version: '0'
|
191
191
|
segments:
|
192
192
|
- 0
|
193
|
-
hash: -
|
193
|
+
hash: -1129513891376945174
|
194
194
|
requirements: []
|
195
195
|
rubyforge_project:
|
196
196
|
rubygems_version: 1.8.24
|