filecluster 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +1 -0
- data/bin/fc-daemon +1 -0
- data/lib/daemon.rb +1 -0
- data/lib/daemon/task_thread.rb +5 -0
- data/lib/fc/db.rb +1 -0
- data/lib/fc/policy.rb +14 -10
- data/lib/fc/storage.rb +2 -2
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +3 -3
- data/test/policy_test.rb +14 -9
- metadata +4 -4
data/README.md
CHANGED
@@ -45,6 +45,7 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
|
|
45
45
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
46
46
|
|daemon_global_tasks_per_thread|10|tasks count for one task thread|
|
47
47
|
|daemon_global_tasks_threads_limit|5|tasks threads count limit for one storage|
|
48
|
+
|daemon_copy_tasks_limit|5|copy tasks count limit for one host|
|
48
49
|
|
49
50
|
## Usage
|
50
51
|
|
data/bin/fc-daemon
CHANGED
@@ -13,6 +13,7 @@ $tasks = {} # tasks by storage name
|
|
13
13
|
$curr_tasks = [] # current tasks
|
14
14
|
$tasks_threads = {} # threads by storage name
|
15
15
|
$check_threads = {} # threads by storage name
|
16
|
+
$copy_count = 0 # copy tasks count
|
16
17
|
$exit_signal = false
|
17
18
|
$global_daemon_thread = nil
|
18
19
|
|
data/lib/daemon.rb
CHANGED
@@ -100,6 +100,7 @@ def run_tasks
|
|
100
100
|
run_threads_count = run_threads_count - threads_count
|
101
101
|
|
102
102
|
$log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
|
103
|
+
$log.debug("copy_count: #{$copy_count}")
|
103
104
|
run_threads_count.times do
|
104
105
|
$log.debug("spawn TaskThread for #{storage.name}")
|
105
106
|
$tasks_threads[storage.name] << TaskThread.new(storage.name)
|
data/lib/daemon/task_thread.rb
CHANGED
@@ -29,6 +29,9 @@ class TaskThread < BaseThread
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def make_copy(task)
|
32
|
+
limit = FC::Var.get('daemon_copy_tasks_limit', 10).to_i
|
33
|
+
sleep 0.1 while $copy_count > limit
|
34
|
+
$copy_count += 1
|
32
35
|
item_storage = task[:item_storage]
|
33
36
|
storage = $storages.detect{|s| s.name == item_storage.storage_name}
|
34
37
|
item = FC::Item.find(item_storage.item_id)
|
@@ -44,5 +47,7 @@ class TaskThread < BaseThread
|
|
44
47
|
rescue Exception => e
|
45
48
|
error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
|
46
49
|
$curr_tasks.delete(task)
|
50
|
+
ensure
|
51
|
+
$copy_count -= 1 if item_storage && $copy_count > 0
|
47
52
|
end
|
48
53
|
end
|
data/lib/fc/db.rb
CHANGED
@@ -203,6 +203,7 @@ module FC
|
|
203
203
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
|
204
204
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
|
205
205
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='5', descr='tasks threads count limit for one storage'")
|
206
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='5', descr='copy tasks count limit for one host'")
|
206
207
|
end
|
207
208
|
end
|
208
209
|
end
|
data/lib/fc/policy.rb
CHANGED
@@ -40,17 +40,21 @@ module FC
|
|
40
40
|
# get available storage for copy by copy_id and size
|
41
41
|
def get_proper_storage_for_copy(size, copy_id = nil, exclude = [])
|
42
42
|
storages = get_copy_storages
|
43
|
-
|
44
|
-
|
45
|
-
|
43
|
+
storage_index = 0
|
44
|
+
storage_host = nil
|
45
|
+
while s = storages[storage_index]
|
46
|
+
if copy_id.to_i == s.copy_id.to_i && !exclude.include?(s.name)
|
47
|
+
storage_index -= 1 while storage_index > 0 && storages[storage_index-1].host == s.host
|
48
|
+
storage_host = s.host
|
49
|
+
break
|
50
|
+
end
|
51
|
+
storage_index += 1
|
52
|
+
end
|
53
|
+
storages = (storages[storage_index..-1]+storages[0..storage_index-1]).select do |s|
|
54
|
+
!exclude.include?(s.name) && s.up? && s.size + size < s.size_limit
|
46
55
|
end
|
47
|
-
|
48
|
-
|
49
|
-
!exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
|
50
|
-
end
|
51
|
-
storage = storages.detect{|s| copy_id.to_i == s.copy_id.to_i}
|
52
|
-
storage = storages.detect{|s| copy_id.to_i < s.copy_id.to_i} unless storage
|
53
|
-
storage = storages.first unless storage
|
56
|
+
storage = storages.select{|s| storage_host == s.host}.sort{|a,b| b.copy_id.to_i <=> a.copy_id.to_i}.first
|
57
|
+
storage = storages.sort{|a,b| b.copy_id.to_i <=> a.copy_id.to_i}.first unless storage
|
54
58
|
storage
|
55
59
|
end
|
56
60
|
end
|
data/lib/fc/storage.rb
CHANGED
@@ -84,13 +84,13 @@ module FC
|
|
84
84
|
end
|
85
85
|
|
86
86
|
# return object size on storage
|
87
|
-
def file_size(file_name)
|
87
|
+
def file_size(file_name, ignore_errors = false)
|
88
88
|
dst_path = "#{self.path}#{file_name}"
|
89
89
|
|
90
90
|
cmd = self.class.curr_host == host ?
|
91
91
|
"du -sb #{dst_path}" :
|
92
92
|
"ssh -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} 'du -sb #{dst_path}'"
|
93
|
-
r = `#{cmd} 2>&1`
|
93
|
+
r = ignore_errors ? `#{cmd} 2>/dev/null` : `#{cmd} 2>&1`
|
94
94
|
raise r if $?.exitstatus != 0
|
95
95
|
r.to_i
|
96
96
|
end
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -42,7 +42,7 @@ def storages_add
|
|
42
42
|
path = '/' + path unless path[0] == '/'
|
43
43
|
storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_id => copy_id)
|
44
44
|
print "Calc current size.. "
|
45
|
-
size = storage.file_size('')
|
45
|
+
size = storage.file_size('', true)
|
46
46
|
puts "ok"
|
47
47
|
rescue Exception => e
|
48
48
|
puts "Error: #{e.message}"
|
@@ -88,7 +88,7 @@ end
|
|
88
88
|
def storages_update_size
|
89
89
|
if storage = find_storage
|
90
90
|
print "Calc current size.. "
|
91
|
-
size = storage.file_size('')
|
91
|
+
size = storage.file_size('', true)
|
92
92
|
storage.size = size
|
93
93
|
begin
|
94
94
|
storage.save
|
@@ -115,7 +115,7 @@ def storages_change
|
|
115
115
|
path = '/' + path unless path[0] == '/'
|
116
116
|
storage.path = path
|
117
117
|
print "Calc current size.. "
|
118
|
-
storage.size = storage.file_size('')
|
118
|
+
storage.size = storage.file_size('', true)
|
119
119
|
puts "ok"
|
120
120
|
end
|
121
121
|
storage.url = url unless url.empty?
|
data/test/policy_test.rb
CHANGED
@@ -6,11 +6,12 @@ class PolicyTest < Test::Unit::TestCase
|
|
6
6
|
@@storages = []
|
7
7
|
@@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_id => 1, :size_limit => 10)
|
8
8
|
@@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_id => 2 , :size_limit => 100)
|
9
|
+
@@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :copy_id => 4 , :size_limit => 100)
|
9
10
|
@@storages.each {|storage| storage.save}
|
10
11
|
@@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_id => 3 , :size_limit => 10)
|
11
12
|
@@storage3.save
|
12
13
|
|
13
|
-
@@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
|
14
|
+
@@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
|
14
15
|
@@policy.save
|
15
16
|
end
|
16
17
|
def shutdown
|
@@ -22,23 +23,23 @@ class PolicyTest < Test::Unit::TestCase
|
|
22
23
|
should "get_create_storages" do
|
23
24
|
FC::Policy.storages_cache_time = 10
|
24
25
|
assert_same_elements @@storages.map(&:id), @@policy.get_create_storages.map(&:id)
|
25
|
-
@@policy.create_storages = 'rec1-sda,rec2-sda
|
26
|
+
@@policy.create_storages = 'rec1-sda,rec2-sda'
|
26
27
|
@@policy.save
|
27
28
|
assert_equal @@storages.size, @@policy.get_create_storages.size
|
28
29
|
FC::Policy.storages_cache_time = 0
|
29
|
-
assert_equal
|
30
|
+
assert_equal 2, @@policy.get_create_storages.size
|
30
31
|
end
|
31
32
|
|
32
33
|
should "get_copy_storages" do
|
33
|
-
@@policy.copy_storages = 'rec1-sda,rec2-sda'
|
34
|
+
@@policy.copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
|
34
35
|
@@policy.save
|
35
36
|
FC::Policy.storages_cache_time = 10
|
36
37
|
assert_same_elements @@storages.map(&:id), @@policy.get_copy_storages.map(&:id)
|
37
|
-
@@policy.copy_storages = 'rec1-sda,rec2-sda
|
38
|
+
@@policy.copy_storages = 'rec1-sda,rec2-sda'
|
38
39
|
@@policy.save
|
39
40
|
assert_equal @@storages.size, @@policy.get_copy_storages.size
|
40
41
|
FC::Policy.storages_cache_time = 0
|
41
|
-
assert_equal
|
42
|
+
assert_equal 2, @@policy.get_copy_storages.size
|
42
43
|
end
|
43
44
|
|
44
45
|
should "get_proper_storage_for_create" do
|
@@ -74,15 +75,19 @@ class PolicyTest < Test::Unit::TestCase
|
|
74
75
|
@@policy.copy_storages = 'rec1-sda,rec3-sda'
|
75
76
|
@@policy.save
|
76
77
|
assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
|
77
|
-
@@policy.copy_storages = '
|
78
|
+
@@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda,rec2-sdb'
|
78
79
|
@@policy.save
|
79
80
|
assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
|
81
|
+
@@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda,rec2-sdb'
|
82
|
+
@@policy.save
|
83
|
+
@@storages[2].update_check_time
|
84
|
+
assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
|
80
85
|
|
81
|
-
@@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda
|
86
|
+
@@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
|
82
87
|
@@policy.save
|
83
88
|
@@storages[0].check_time = 0
|
84
89
|
@@storages[0].save
|
85
|
-
assert_equal '
|
90
|
+
assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1, 1).name, 'storage by copy_id'
|
86
91
|
end
|
87
92
|
|
88
93
|
should "filter_by_host" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
12
|
+
date: 2013-07-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rb-readline
|
@@ -190,7 +190,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
190
190
|
version: '0'
|
191
191
|
segments:
|
192
192
|
- 0
|
193
|
-
hash:
|
193
|
+
hash: -3114920766113706520
|
194
194
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
195
195
|
none: false
|
196
196
|
requirements:
|
@@ -199,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
199
|
version: '0'
|
200
200
|
segments:
|
201
201
|
- 0
|
202
|
-
hash:
|
202
|
+
hash: -3114920766113706520
|
203
203
|
requirements: []
|
204
204
|
rubyforge_project:
|
205
205
|
rubygems_version: 1.8.24
|