filecluster 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +1 -1
- data/lib/daemon.rb +7 -7
- data/lib/daemon/global_daemon_thread.rb +21 -18
- data/lib/daemon/task_thread.rb +1 -0
- data/lib/fc/db.rb +1 -1
- data/lib/fc/version.rb +1 -1
- metadata +4 -4
data/README.md
CHANGED
@@ -44,7 +44,7 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
|
|
44
44
|
|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
|
45
45
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
46
46
|
|daemon_global_tasks_per_thread|10|tasks count for one task thread|
|
47
|
-
|daemon_global_tasks_threads_limit|
|
47
|
+
|daemon_global_tasks_threads_limit|5|tasks threads count limit for one storage|
|
48
48
|
|
49
49
|
## Usage
|
50
50
|
|
data/lib/daemon.rb
CHANGED
@@ -30,11 +30,9 @@ def run_global_daemon
|
|
30
30
|
$log.debug("spawn GlobalDaemonThread")
|
31
31
|
$global_daemon_thread = GlobalDaemonThread.new(timeout)
|
32
32
|
end
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
$global_daemon_thread.exit
|
37
|
-
end
|
33
|
+
elsif $global_daemon_thread && $global_daemon_thread.alive?
|
34
|
+
$log.warn("Kill global daemon thread (new host = #{r['val']})")
|
35
|
+
$global_daemon_thread.exit
|
38
36
|
end
|
39
37
|
end
|
40
38
|
|
@@ -94,11 +92,13 @@ def run_tasks
|
|
94
92
|
threads_count = $tasks_threads[storage.name].count
|
95
93
|
|
96
94
|
# <max_threads> tasks per thread, maximum <tasks_per_thread> threads
|
97
|
-
max_threads = FC::Var.get('daemon_global_tasks_threads_limit',
|
95
|
+
max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 5).to_i
|
98
96
|
tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
|
99
97
|
|
100
|
-
run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
|
98
|
+
run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
|
101
99
|
run_threads_count = max_threads if run_threads_count > max_threads
|
100
|
+
run_threads_count = run_threads_count - threads_count
|
101
|
+
|
102
102
|
$log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
|
103
103
|
run_threads_count.times do
|
104
104
|
$log.debug("spawn TaskThread for #{storage.name}")
|
@@ -38,25 +38,28 @@ class GlobalDaemonThread < BaseThread
|
|
38
38
|
end
|
39
39
|
|
40
40
|
limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
$log.
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy
|
56
|
-
if storage
|
57
|
-
FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
|
41
|
+
all_policies.each do |policy|
|
42
|
+
next if policy.copies.to_i < 2
|
43
|
+
copies = (1..policy.copies.to_i-1).to_a.join(',')
|
44
|
+
sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages "+
|
45
|
+
"FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE i.policy_id = #{policy.id} AND "+
|
46
|
+
"ist.item_id = i.id AND i.copies IN (#{copies}) AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
|
47
|
+
r = FC::DB.query(sql)
|
48
|
+
r.each do |row|
|
49
|
+
$log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
|
50
|
+
item_storages = row['storages'].split(',')
|
51
|
+
if row['item_copies'] != item_storages.count
|
52
|
+
$log.warn("GlobalDaemonThread: ItemStorage count <> item.copies for item #{row['item_id']}")
|
53
|
+
elsif item_storages.count >= policy.copies.to_i
|
54
|
+
$log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
|
58
55
|
else
|
59
|
-
|
56
|
+
src_storage = all_storages.detect{|s| item_storages.first == s.name}
|
57
|
+
storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy
|
58
|
+
if storage
|
59
|
+
FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
|
60
|
+
else
|
61
|
+
error 'No available storage', :item_id => row['item_id']
|
62
|
+
end
|
60
63
|
end
|
61
64
|
end
|
62
65
|
end
|
data/lib/daemon/task_thread.rb
CHANGED
data/lib/fc/db.rb
CHANGED
@@ -202,7 +202,7 @@ module FC
|
|
202
202
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
|
203
203
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
|
204
204
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
|
205
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='
|
205
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='5', descr='tasks threads count limit for one storage'")
|
206
206
|
end
|
207
207
|
end
|
208
208
|
end
|
data/lib/fc/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
12
|
+
date: 2013-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rb-readline
|
@@ -190,7 +190,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
190
190
|
version: '0'
|
191
191
|
segments:
|
192
192
|
- 0
|
193
|
-
hash:
|
193
|
+
hash: 4310476469501781465
|
194
194
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
195
195
|
none: false
|
196
196
|
requirements:
|
@@ -199,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
199
|
version: '0'
|
200
200
|
segments:
|
201
201
|
- 0
|
202
|
-
hash:
|
202
|
+
hash: 4310476469501781465
|
203
203
|
requirements: []
|
204
204
|
rubyforge_project:
|
205
205
|
rubygems_version: 1.8.24
|