filecluster 0.5.4 → 0.5.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/daemon/check_thread.rb +2 -0
- data/lib/fc/db.rb +7 -2
- data/lib/fc/storage.rb +28 -3
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +30 -5
- data/lib/utils.rb +3 -2
- data/test/storage_test.rb +28 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 09926dfab03ca82fe41a029a554098bd80e7d596
|
4
|
+
data.tar.gz: 584db18e46de8e00dede1833ab4b206d02774d3d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d7a502bcd878cb1847fb7c168fbfc8239d0e1a3050b6938b86b1311fada3e2595d16004bee2025166e086f90b9bb61dd77c8c8c3f1113cbceb0b6aa89ddf0d2
|
7
|
+
data.tar.gz: e863eb81db7bde65539315d19f14585d54f083f66524a25051e1c7f0205728fb8aec346824e14cc503453ff5b219c0da167d68e95fe2f70774b8fbeafc2f7d20
|
data/lib/daemon/check_thread.rb
CHANGED
@@ -3,6 +3,8 @@ class CheckThread < BaseThread
|
|
3
3
|
$log.debug("CheckThread: Run stotage check for #{storage_name}")
|
4
4
|
storage = $storages.detect{|s| s.name == storage_name}
|
5
5
|
if File.writable?(storage.path)
|
6
|
+
storage.size_limit = storage.get_real_size if storage.auto_size?
|
7
|
+
# also saves size_limit
|
6
8
|
storage.update_check_time
|
7
9
|
else
|
8
10
|
error "Storage #{storage.name} with path #{storage.path} not writable"
|
data/lib/fc/db.rb
CHANGED
@@ -90,11 +90,11 @@ module FC
|
|
90
90
|
def self.close
|
91
91
|
if @options[:multi_threads]
|
92
92
|
if @connects[Thread.current.object_id]
|
93
|
-
@connects[Thread.current.object_id].close
|
93
|
+
@connects[Thread.current.object_id].close if @connects[Thread.current.object_id]
|
94
94
|
@connects.delete(Thread.current.object_id)
|
95
95
|
end
|
96
96
|
else
|
97
|
-
@connects.first[1].close
|
97
|
+
@connects.first[1].close if @connects.first
|
98
98
|
@connects.clear
|
99
99
|
end
|
100
100
|
end
|
@@ -329,5 +329,10 @@ module FC
|
|
329
329
|
FC::DB.query("ALTER TABLE #{@prefix}items MODIFY COLUMN status ENUM('new', 'ready', 'error', 'delete', 'deferred_delete') NOT NULL DEFAULT 'new'")
|
330
330
|
FC::DB.query("ALTER TABLE #{@prefix}policies ADD COLUMN delete_deferred_time int NOT NULL DEFAULT 0")
|
331
331
|
end
|
332
|
+
|
333
|
+
def self.migrate_4
|
334
|
+
FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN auto_size bigint(20) DEFAULT 0")
|
335
|
+
end
|
336
|
+
|
332
337
|
end
|
333
338
|
end
|
data/lib/fc/storage.rb
CHANGED
@@ -4,7 +4,7 @@ require 'fileutils'
|
|
4
4
|
|
5
5
|
module FC
|
6
6
|
class Storage < DbBase
|
7
|
-
set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight'
|
7
|
+
set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size'
|
8
8
|
|
9
9
|
class << self
|
10
10
|
attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
|
@@ -39,7 +39,11 @@ module FC
|
|
39
39
|
end
|
40
40
|
super params
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
|
+
def auto_size?
|
44
|
+
(auto_size || 0) > 0
|
45
|
+
end
|
46
|
+
|
43
47
|
def free
|
44
48
|
size_limit - size
|
45
49
|
end
|
@@ -52,7 +56,28 @@ module FC
|
|
52
56
|
rate = free.to_f / size_limit
|
53
57
|
rate < 0 ? 0.0 : rate
|
54
58
|
end
|
55
|
-
|
59
|
+
|
60
|
+
def size_in_status(status)
|
61
|
+
FC::DB.query(%{SELECT sum(i.size) as isize
|
62
|
+
FROM #{FC::ItemStorage.table_name} its
|
63
|
+
join #{FC::Item.table_name} i on i.id = its.item_id
|
64
|
+
where its.storage_name = '#{self.name}'
|
65
|
+
and its.status = '#{status}'}).first['isize'].to_i
|
66
|
+
end
|
67
|
+
|
68
|
+
def get_real_size
|
69
|
+
size_in_copy = size_in_status 'copy'
|
70
|
+
self.size_limit = get_disk_free_space - size_in_copy - auto_size + size
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_disk_free_space
|
74
|
+
cmd = "df #{self.path.shellescape}"
|
75
|
+
cmd = "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"df #{self.path.shellescape}\"" unless self.class.curr_host == host
|
76
|
+
r = `#{cmd} 2>&1`
|
77
|
+
raise r if $?.exitstatus != 0
|
78
|
+
r.split("\n").last.split(/\s+/)[3].to_i * 1024
|
79
|
+
end
|
80
|
+
|
56
81
|
def get_copy_storages
|
57
82
|
self.class.get_copy_storages_mutex.synchronize do
|
58
83
|
unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -30,6 +30,7 @@ def storages_show
|
|
30
30
|
Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
|
31
31
|
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
32
32
|
Size limit: #{size_to_human storage.size_limit}
|
33
|
+
Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
|
33
34
|
Copy storages: #{storage.copy_storages}
|
34
35
|
Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
|
35
36
|
Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
|
@@ -46,21 +47,34 @@ def storages_add
|
|
46
47
|
url = stdin_read_val('Url')
|
47
48
|
url_weight = stdin_read_val('URL weight', true).to_i
|
48
49
|
write_weight = stdin_read_val('Write weight', true).to_i
|
49
|
-
|
50
|
+
is_auto_size = %(y yes).include?(stdin_read_val('Auto size (y/n)?').downcase)
|
51
|
+
if is_auto_size
|
52
|
+
auto_size = human_to_size stdin_read_val('Minimal free disk space') {|val| "Minimal free disk space not is valid size." unless human_to_size(val) || human_to_size(val) < 1 }
|
53
|
+
size_limit = 0
|
54
|
+
else
|
55
|
+
auto_size = 0
|
56
|
+
size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
|
57
|
+
end
|
58
|
+
|
50
59
|
copy_storages = stdin_read_val('Copy storages', true)
|
51
60
|
storages = FC::Storage.where.map(&:name)
|
52
61
|
copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
|
53
62
|
begin
|
54
63
|
path = path +'/' unless path[-1] == '/'
|
55
64
|
path = '/' + path unless path[0] == '/'
|
56
|
-
storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight)
|
57
|
-
print
|
65
|
+
storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
|
66
|
+
print 'Calc current size.. '
|
58
67
|
size = storage.file_size('', true)
|
59
68
|
puts "ok"
|
60
69
|
rescue Exception => e
|
61
70
|
puts "Error: #{e.message}"
|
62
71
|
exit
|
63
72
|
end
|
73
|
+
|
74
|
+
if storage.auto_size?
|
75
|
+
storage.size = size
|
76
|
+
size_limit = storage.get_real_size
|
77
|
+
end
|
64
78
|
free = size_limit - size
|
65
79
|
puts %Q{\nStorage
|
66
80
|
Name: #{name}
|
@@ -72,6 +86,7 @@ def storages_add
|
|
72
86
|
Write weight: #{write_weight}
|
73
87
|
Size: #{size_to_human size} (#{(size.to_f*100 / size_limit).to_i}%)
|
74
88
|
Free: #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
|
89
|
+
Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
|
75
90
|
Size limit: #{size_to_human size_limit}
|
76
91
|
Copy storages #{copy_storages}}
|
77
92
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
@@ -129,7 +144,14 @@ def storages_change
|
|
129
144
|
url = stdin_read_val("Url (now #{storage.url})", true)
|
130
145
|
url_weight = stdin_read_val("URL weight (now #{storage.url_weight})", true)
|
131
146
|
write_weight = stdin_read_val("Write weight (now #{storage.write_weight})", true)
|
132
|
-
|
147
|
+
is_auto_size = %(y yes).include?(stdin_read_val("Auto size (now #{storage.auto_size? ? 'yes' : 'no'})", true, storage.auto_size? ? 'yes' : 'no').downcase)
|
148
|
+
if is_auto_size
|
149
|
+
auto_size = human_to_size stdin_read_val("Minimal free disk space (now #{size_to_human(storage.auto_size)})", true, size_to_human(storage.auto_size)) {|val| "Minimal free disk space not is valid size." if !human_to_size(val) || human_to_size(val) < 1}
|
150
|
+
size_limit = 0
|
151
|
+
else
|
152
|
+
auto_size = 0
|
153
|
+
size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
|
154
|
+
end
|
133
155
|
copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
|
134
156
|
|
135
157
|
storage.dc = dc unless dc.empty?
|
@@ -142,6 +164,8 @@ def storages_change
|
|
142
164
|
storage.size = storage.file_size('', true)
|
143
165
|
puts "ok"
|
144
166
|
end
|
167
|
+
storage.auto_size = auto_size
|
168
|
+
size_limit = size_to_human(storage.get_real_size) if storage.auto_size?
|
145
169
|
storage.url = url unless url.empty?
|
146
170
|
storage.url_weight = url_weight.to_i unless url_weight.empty?
|
147
171
|
storage.write_weight = write_weight.to_i unless write_weight.empty?
|
@@ -159,7 +183,8 @@ def storages_change
|
|
159
183
|
Write weight: #{storage.write_weight}
|
160
184
|
Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
|
161
185
|
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
162
|
-
Size
|
186
|
+
Size type: #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
|
187
|
+
Size limit: #{size_to_human storage.size_limit }
|
163
188
|
Copy storages: #{storage.copy_storages}}
|
164
189
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
165
190
|
puts ""
|
data/lib/utils.rb
CHANGED
@@ -47,11 +47,12 @@ def human_to_size(size)
|
|
47
47
|
result.to_i
|
48
48
|
end
|
49
49
|
|
50
|
-
def stdin_read_val(name, can_empty = false)
|
50
|
+
def stdin_read_val(name, can_empty = false, default_value = nil)
|
51
51
|
while val = Readline.readline("#{name}: ", false).strip.downcase
|
52
52
|
if val.empty? && !can_empty
|
53
53
|
puts "Input non empty #{name}."
|
54
|
-
else
|
54
|
+
else
|
55
|
+
val = default_value if default_value && val.empty?
|
55
56
|
if block_given?
|
56
57
|
if err = yield(val)
|
57
58
|
puts err
|
data/test/storage_test.rb
CHANGED
@@ -8,6 +8,7 @@ class StorageTest < Test::Unit::TestCase
|
|
8
8
|
@@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_storages => 'rec1-sda,rec3-sda', :size_limit => 100)
|
9
9
|
@@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_storages => 'rec1-sda,rec2-sda', :size_limit => 10)
|
10
10
|
@@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
|
11
|
+
@@storages << FC::Storage.new(:name => 'rec1-sdc', :host => 'rec1', :size => 100, :auto_size => 50)
|
11
12
|
@@storages.each {|storage| storage.save}
|
12
13
|
|
13
14
|
@@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
|
@@ -15,6 +16,8 @@ class StorageTest < Test::Unit::TestCase
|
|
15
16
|
end
|
16
17
|
def shutdown
|
17
18
|
FC::DB.query("DELETE FROM policies")
|
19
|
+
FC::DB.query("DELETE FROM items_storages")
|
20
|
+
FC::DB.query("DELETE FROM items")
|
18
21
|
FC::DB.query("DELETE FROM storages")
|
19
22
|
end
|
20
23
|
end
|
@@ -79,4 +82,29 @@ class StorageTest < Test::Unit::TestCase
|
|
79
82
|
assert_equal 9.to_f/10, storage.free_rate, 'first storage free_rate must be free/size_limit'
|
80
83
|
assert_equal 1.to_f/10, storage.size_rate, 'first storage free must be size/size_limit'
|
81
84
|
end
|
85
|
+
|
86
|
+
should 'size_in_status copy' do
|
87
|
+
# create fc_item_storage in status 'copy'
|
88
|
+
@item = FC::Item.new(:name => '/test item', :policy_id => 1, :size => 100)
|
89
|
+
@item.save
|
90
|
+
@item_storage = FC::ItemStorage.new(:item_id => @item.id, :storage_name => @@storages[0].name, :status => 'copy')
|
91
|
+
@item_storage.save
|
92
|
+
size_in_copy_status = @@storages[0].size_in_status 'copy'
|
93
|
+
assert_equal 100, size_in_copy_status, 'items sum size in status copy must be correct'
|
94
|
+
end
|
95
|
+
|
96
|
+
should 'get real size with items status in copy' do # most worst case
|
97
|
+
disk_free = 1024
|
98
|
+
item_size = 100
|
99
|
+
|
100
|
+
# mock real disk free method
|
101
|
+
@@storages[4].define_singleton_method(:get_disk_free_space) do
|
102
|
+
disk_free
|
103
|
+
end
|
104
|
+
@item = FC::Item.new(:name => '/test item rec1-sdc', :policy_id => 1, :size => item_size)
|
105
|
+
@item.save
|
106
|
+
@item_storage = FC::ItemStorage.new(:item_id => @item.id, :storage_name => @@storages[4].name, :status => 'copy')
|
107
|
+
@item_storage.save
|
108
|
+
assert_equal disk_free - item_size - @@storages[4].auto_size + @@storages[4].size, @@storages[4].get_real_size, 'storage size must be correct with items in copy status'
|
109
|
+
end
|
82
110
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|