filecluster 0.2.3 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +2 -2
- data/bin/fc-daemon +2 -0
- data/lib/daemon.rb +9 -45
- data/lib/daemon/run_tasks_thread.rb +25 -0
- data/lib/daemon/task_thread.rb +1 -2
- data/lib/daemon/update_tasks_thread.rb +27 -0
- data/lib/fc/db.rb +2 -2
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +1 -1
- metadata +6 -4
data/README.md
CHANGED
@@ -44,8 +44,8 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
|
|
44
44
|
|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
|
45
45
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
46
46
|
|daemon_global_tasks_per_thread|10|tasks count for one task thread|
|
47
|
-
|daemon_global_tasks_threads_limit|
|
48
|
-
|daemon_copy_tasks_limit|
|
47
|
+
|daemon_global_tasks_threads_limit|10|tasks threads count limit for one storage|
|
48
|
+
|daemon_copy_tasks_limit|10|copy tasks count limit for one host|
|
49
49
|
|
50
50
|
## Usage
|
51
51
|
|
data/bin/fc-daemon
CHANGED
@@ -16,6 +16,8 @@ $check_threads = {} # threads by storage name
|
|
16
16
|
$copy_count = 0 # copy tasks count
|
17
17
|
$exit_signal = false
|
18
18
|
$global_daemon_thread = nil
|
19
|
+
$update_tasks_thread = nil
|
20
|
+
$run_tasks_thread = nil
|
19
21
|
|
20
22
|
default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
|
21
23
|
descriptions = {
|
data/lib/daemon.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
require "date"
|
2
2
|
require "daemon/base_thread"
|
3
|
-
require "daemon/global_daemon_thread"
|
4
3
|
require "daemon/check_thread"
|
4
|
+
require "daemon/global_daemon_thread"
|
5
|
+
require "daemon/run_tasks_thread"
|
6
|
+
require "daemon/update_tasks_thread"
|
5
7
|
require "daemon/task_thread"
|
6
8
|
|
7
9
|
def error(msg, options = {})
|
@@ -57,53 +59,15 @@ def storages_check
|
|
57
59
|
end
|
58
60
|
|
59
61
|
def update_tasks
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
def check_tasks(type)
|
64
|
-
storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
|
65
|
-
cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
|
66
|
-
ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
|
67
|
-
flatten.map{|task| task[:item_storage].id}
|
68
|
-
ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
|
69
|
-
|
70
|
-
limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
|
71
|
-
cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
|
72
|
-
cond << " LIMIT #{limit}"
|
73
|
-
FC::ItemStorage.where(cond).each do |item_storage|
|
74
|
-
unless ids.include?(item_storage.id)
|
75
|
-
$tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
|
76
|
-
$tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
|
77
|
-
$log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
|
78
|
-
end
|
79
|
-
end
|
62
|
+
if !$update_tasks_thread || !$update_tasks_thread.alive?
|
63
|
+
$log.debug("spawn UpdateTasksThread")
|
64
|
+
$update_tasks_thread = UpdateTasksThread.new
|
80
65
|
end
|
81
|
-
|
82
|
-
check_tasks(:delete)
|
83
|
-
check_tasks(:copy)
|
84
66
|
end
|
85
67
|
|
86
68
|
def run_tasks
|
87
|
-
|
88
|
-
|
89
|
-
$
|
90
|
-
$tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
|
91
|
-
tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
|
92
|
-
threads_count = $tasks_threads[storage.name].count
|
93
|
-
|
94
|
-
# <max_threads> tasks per thread, maximum <tasks_per_thread> threads
|
95
|
-
max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 5).to_i
|
96
|
-
tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
|
97
|
-
|
98
|
-
run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
|
99
|
-
run_threads_count = max_threads if run_threads_count > max_threads
|
100
|
-
run_threads_count = run_threads_count - threads_count
|
101
|
-
|
102
|
-
$log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
|
103
|
-
$log.debug("copy_count: #{$copy_count}")
|
104
|
-
run_threads_count.times do
|
105
|
-
$log.debug("spawn TaskThread for #{storage.name}")
|
106
|
-
$tasks_threads[storage.name] << TaskThread.new(storage.name)
|
107
|
-
end
|
69
|
+
if !$run_tasks_thread || !$run_tasks_thread.alive?
|
70
|
+
$log.debug("spawn RunTasksThread")
|
71
|
+
$run_tasks_thread = RunTasksThread.new
|
108
72
|
end
|
109
73
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class RunTasksThread < BaseThread
|
2
|
+
def go
|
3
|
+
$log.debug('Run tasks (copy_count: #{$copy_count})')
|
4
|
+
$storages.each do |storage|
|
5
|
+
$tasks_threads[storage.name] = [] unless $tasks_threads[storage.name]
|
6
|
+
$tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
|
7
|
+
tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
|
8
|
+
threads_count = $tasks_threads[storage.name].count
|
9
|
+
|
10
|
+
# <max_threads> tasks per thread, maximum <tasks_per_thread> threads
|
11
|
+
max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 10).to_i
|
12
|
+
tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
|
13
|
+
|
14
|
+
run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
|
15
|
+
run_threads_count = max_threads if run_threads_count > max_threads
|
16
|
+
run_threads_count = run_threads_count - threads_count
|
17
|
+
|
18
|
+
$log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
|
19
|
+
run_threads_count.times do
|
20
|
+
$log.debug("spawn TaskThread for #{storage.name}")
|
21
|
+
$tasks_threads[storage.name] << TaskThread.new(storage.name)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/daemon/task_thread.rb
CHANGED
@@ -29,8 +29,7 @@ class TaskThread < BaseThread
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def make_copy(task)
|
32
|
-
|
33
|
-
sleep 0.1 while $copy_count > limit
|
32
|
+
sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_limit', 10).to_i
|
34
33
|
$copy_count += 1
|
35
34
|
item_storage = task[:item_storage]
|
36
35
|
storage = $storages.detect{|s| s.name == item_storage.storage_name}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
class UpdateTasksThread < BaseThread
|
2
|
+
def go
|
3
|
+
$log.debug('Update tasks')
|
4
|
+
check_tasks(:delete)
|
5
|
+
check_tasks(:copy)
|
6
|
+
end
|
7
|
+
|
8
|
+
def check_tasks(type)
|
9
|
+
storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
|
10
|
+
return if storages_names.empty?
|
11
|
+
cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
|
12
|
+
ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
|
13
|
+
flatten.map{|task| task[:item_storage].id}
|
14
|
+
ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
|
15
|
+
|
16
|
+
limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
|
17
|
+
cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
|
18
|
+
cond << " LIMIT #{limit}"
|
19
|
+
FC::ItemStorage.where(cond).each do |item_storage|
|
20
|
+
unless ids.include?(item_storage.id)
|
21
|
+
$tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
|
22
|
+
$tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
|
23
|
+
$log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/fc/db.rb
CHANGED
@@ -202,8 +202,8 @@ module FC
|
|
202
202
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
|
203
203
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
|
204
204
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
|
205
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='
|
206
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='
|
205
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='10', descr='tasks threads count limit for one storage'")
|
206
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='10', descr='copy tasks count limit for one host'")
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -120,7 +120,7 @@ def storages_change
|
|
120
120
|
end
|
121
121
|
storage.url = url unless url.empty?
|
122
122
|
storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
|
123
|
-
storage.copy_id = copy_id.to_i
|
123
|
+
storage.copy_id = copy_id.to_i unless copy_id.empty?
|
124
124
|
|
125
125
|
puts %Q{\nStorage
|
126
126
|
Name: #{storage.name}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.4
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
12
|
+
date: 2013-07-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rb-readline
|
@@ -147,7 +147,9 @@ files:
|
|
147
147
|
- lib/daemon/base_thread.rb
|
148
148
|
- lib/daemon/check_thread.rb
|
149
149
|
- lib/daemon/global_daemon_thread.rb
|
150
|
+
- lib/daemon/run_tasks_thread.rb
|
150
151
|
- lib/daemon/task_thread.rb
|
152
|
+
- lib/daemon/update_tasks_thread.rb
|
151
153
|
- lib/fc/base.rb
|
152
154
|
- lib/fc/db.rb
|
153
155
|
- lib/fc/error.rb
|
@@ -190,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
190
192
|
version: '0'
|
191
193
|
segments:
|
192
194
|
- 0
|
193
|
-
hash: -
|
195
|
+
hash: -4358767098397896012
|
194
196
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
195
197
|
none: false
|
196
198
|
requirements:
|
@@ -199,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
201
|
version: '0'
|
200
202
|
segments:
|
201
203
|
- 0
|
202
|
-
hash: -
|
204
|
+
hash: -4358767098397896012
|
203
205
|
requirements: []
|
204
206
|
rubyforge_project:
|
205
207
|
rubygems_version: 1.8.24
|