filecluster 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +6 -4
- data/bin/fc-daemon +12 -9
- data/lib/daemon.rb +2 -1
- data/lib/daemon/copy_task_thread.rb +45 -0
- data/lib/daemon/delete_task_thread.rb +35 -0
- data/lib/daemon/run_tasks_thread.rb +21 -13
- data/lib/daemon/update_tasks_thread.rb +13 -14
- data/lib/fc/db.rb +6 -4
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +2 -2
- metadata +6 -5
- data/lib/daemon/task_thread.rb +0 -53
data/README.md
CHANGED
@@ -40,12 +40,14 @@ Selecting available storage to copy item by storage.copy_storages (from left to
|
|
40
40
|
|global_daemon_host||set fc_daemon when run global daemon task (only in one instance)|
|
41
41
|
|daemon_cycle_time|30|time between global daemon checks and storages available checks|
|
42
42
|
|daemon_global_wait_time|120|time between runs global daemon if it does not running|
|
43
|
-
|
|
43
|
+
|daemon_tasks_copy_group_limit|1000|select limit for copy tasks|
|
44
|
+
|daemon_tasks_delete_group_limit|10000|select limit for delete tasks|
|
45
|
+
|daemon_tasks_copy_threads_limit|10|copy tasks threads count limit for one storage|
|
46
|
+
|daemon_tasks_delete_threads_limit|10|delete tasks threads count limit for one storage|
|
47
|
+
|daemon_copy_tasks_per_host_limit|10|copy tasks count limit for one host|
|
48
|
+
|daemon_global_tasks_group_limit|1000|select limit for create copy tasks|
|
44
49
|
|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
|
45
50
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
46
|
-
|daemon_global_tasks_per_thread|10|tasks count for one task thread|
|
47
|
-
|daemon_global_tasks_threads_limit|10|tasks threads count limit for one storage|
|
48
|
-
|daemon_copy_tasks_limit|10|copy tasks count limit for one host|
|
49
51
|
|daemon_restart_period|86400|time between fc-daemon self restart|
|
50
52
|
|
51
53
|
## Usage
|
data/bin/fc-daemon
CHANGED
@@ -8,16 +8,19 @@ require 'filecluster'
|
|
8
8
|
require 'utils'
|
9
9
|
require 'daemon'
|
10
10
|
|
11
|
-
$
|
12
|
-
$
|
13
|
-
$
|
14
|
-
$
|
15
|
-
$
|
16
|
-
$
|
17
|
-
$
|
11
|
+
$all_storages = [] # all storages
|
12
|
+
$storages = [] # storages on current host
|
13
|
+
$tasks_copy = {} # copy tasks by storage name
|
14
|
+
$tasks_delete = {} # delete tasks by storage name
|
15
|
+
$curr_tasks = [] # current tasks
|
16
|
+
$tasks_copy_threads = {} # copy threads by storage name
|
17
|
+
$tasks_delete_threads = {} # delete threads by storage name
|
18
|
+
$check_threads = {} # check threads by storage name
|
19
|
+
$copy_count = 0 # copy tasks count for current host
|
20
|
+
$exit_signal = false
|
18
21
|
$global_daemon_thread = nil
|
19
|
-
$update_tasks_thread
|
20
|
-
$run_tasks_thread
|
22
|
+
$update_tasks_thread = nil
|
23
|
+
$run_tasks_thread = nil
|
21
24
|
|
22
25
|
default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
|
23
26
|
descriptions = {
|
data/lib/daemon.rb
CHANGED
@@ -4,7 +4,8 @@ require "daemon/check_thread"
|
|
4
4
|
require "daemon/global_daemon_thread"
|
5
5
|
require "daemon/run_tasks_thread"
|
6
6
|
require "daemon/update_tasks_thread"
|
7
|
-
require "daemon/
|
7
|
+
require "daemon/copy_task_thread"
|
8
|
+
require "daemon/delete_task_thread"
|
8
9
|
|
9
10
|
def error(msg, options = {})
|
10
11
|
$log.error(msg)
|
@@ -0,0 +1,45 @@
|
|
1
|
+
class CopyTaskThread < BaseThread
|
2
|
+
def go(storage_name)
|
3
|
+
return unless $tasks_copy[storage_name]
|
4
|
+
Thread.current[:tasks_processed] = 0 unless Thread.current[:tasks_processed]
|
5
|
+
while task = $tasks_copy[storage_name].shift do
|
6
|
+
$curr_tasks << task
|
7
|
+
$log.debug("CopyTaskThread(#{storage_name}): run task for item_storage ##{task.id}")
|
8
|
+
make_copy(task)
|
9
|
+
$curr_tasks.delete(task)
|
10
|
+
$log.debug("CopyTaskThread(#{storage_name}): finish task for item_storage ##{task.id}")
|
11
|
+
Thread.current[:tasks_processed] += 1
|
12
|
+
exit if $exit_signal
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def make_copy(task)
|
17
|
+
sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_per_host_limit', 10).to_i
|
18
|
+
$copy_count += 1
|
19
|
+
storage = $storages.detect{|s| s.name == task.storage_name}
|
20
|
+
begin
|
21
|
+
item = FC::Item.find(task.item_id)
|
22
|
+
rescue Exception => e
|
23
|
+
if e.message.match('Record not found')
|
24
|
+
$log.warn("Item ##{task.item_id} not found before copy")
|
25
|
+
return nil
|
26
|
+
else
|
27
|
+
raise e
|
28
|
+
end
|
29
|
+
end
|
30
|
+
return nil unless item && item.status == 'ready'
|
31
|
+
src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
|
32
|
+
unless src_item_storage
|
33
|
+
$log.warn("Item ##{item.id} #{item.name} has no ready item_storage")
|
34
|
+
return nil
|
35
|
+
end
|
36
|
+
src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
|
37
|
+
$log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
|
38
|
+
item.copy_item_storage(src_storage, storage, task)
|
39
|
+
rescue Exception => e
|
40
|
+
error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => task.item_id, :item_storage_id => task.id
|
41
|
+
$curr_tasks.delete(task)
|
42
|
+
ensure
|
43
|
+
$copy_count -= 1 if $copy_count > 0
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
class DeleteTaskThread < BaseThread
|
2
|
+
def go(storage_name)
|
3
|
+
return unless $tasks_delete[storage_name]
|
4
|
+
Thread.current[:tasks_processed] = 0 unless Thread.current[:tasks_processed]
|
5
|
+
while task = $tasks_delete[storage_name].shift do
|
6
|
+
$curr_tasks << task
|
7
|
+
$log.debug("DeleteTaskThread(#{storage_name}): run task for item_storage ##{task.id}")
|
8
|
+
make_delete(task)
|
9
|
+
$curr_tasks.delete(task)
|
10
|
+
$log.debug("DeleteTaskThread(#{storage_name}): finish task for item_storage ##{task.id}")
|
11
|
+
Thread.current[:tasks_processed] += 1
|
12
|
+
exit if $exit_signal
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def make_delete(task)
|
17
|
+
storage = $storages.detect{|s| s.name == task.storage_name}
|
18
|
+
begin
|
19
|
+
item = FC::Item.find(task.item_id)
|
20
|
+
rescue Exception => e
|
21
|
+
if e.message.match('Record not found')
|
22
|
+
$log.warn("Item ##{task.item_id} not found before delete")
|
23
|
+
return nil
|
24
|
+
else
|
25
|
+
raise e
|
26
|
+
end
|
27
|
+
end
|
28
|
+
$log.debug("Delete #{storage.path}#{item.name}")
|
29
|
+
storage.delete_file(item.name)
|
30
|
+
task.delete
|
31
|
+
rescue Exception => e
|
32
|
+
error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => task.item_id, :item_storage_id => task.id
|
33
|
+
$curr_tasks.delete(task)
|
34
|
+
end
|
35
|
+
end
|
@@ -1,24 +1,32 @@
|
|
1
1
|
class RunTasksThread < BaseThread
|
2
2
|
def go
|
3
|
-
$log.debug(
|
3
|
+
$log.debug('Run tasks')
|
4
|
+
run_tasks(:delete)
|
5
|
+
run_tasks(:copy)
|
6
|
+
end
|
7
|
+
|
8
|
+
def run_tasks(type)
|
9
|
+
tasks = (type == :copy ? $tasks_copy : $tasks_delete)
|
10
|
+
tasks_threads = (type == :copy ? $tasks_copy_threads : $tasks_delete_threads)
|
4
11
|
$storages.each do |storage|
|
5
|
-
|
6
|
-
|
7
|
-
tasks_count =
|
8
|
-
threads_count =
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
12
|
+
tasks_threads[storage.name] = [] unless tasks_threads[storage.name]
|
13
|
+
tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
|
14
|
+
tasks_count = tasks[storage.name] ? tasks[storage.name].size : 0
|
15
|
+
threads_count = tasks_threads[storage.name].count
|
16
|
+
thread_avg_performance = 0
|
17
|
+
thread_avg_performance = tasks_threads[storage.name].inject(0){|sum, thread| sum+thread[:tasks_processed]} / threads_count if threads_count > 0
|
18
|
+
thread_avg_performance = 10 if thread_avg_performance < 10
|
19
|
+
max_threads = FC::Var.get("daemon_tasks_#{type}_threads_limit", 10).to_i
|
13
20
|
|
14
|
-
run_threads_count = (tasks_count/
|
21
|
+
run_threads_count = (tasks_count/thread_avg_performance.to_f).ceil
|
15
22
|
run_threads_count = max_threads if run_threads_count > max_threads
|
16
23
|
run_threads_count = run_threads_count - threads_count
|
24
|
+
run_threads_count = 0 if run_threads_count < 0
|
17
25
|
|
18
|
-
$log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
|
26
|
+
$log.debug("#{storage.name} #{type} tasks_count: #{tasks_count}, threads_count: #{threads_count}, avg_performance: #{thread_avg_performance}, run_threads_count: #{run_threads_count}")
|
27
|
+
tasks_threads[storage.name].each{|thread| thread[:tasks_processed] = 0}
|
19
28
|
run_threads_count.times do
|
20
|
-
|
21
|
-
$tasks_threads[storage.name] << TaskThread.new(storage.name)
|
29
|
+
tasks_threads[storage.name] << (type == :copy ? CopyTaskThread.new(storage.name) : DeleteTaskThread.new(storage.name))
|
22
30
|
end
|
23
31
|
end
|
24
32
|
end
|
@@ -6,22 +6,21 @@ class UpdateTasksThread < BaseThread
|
|
6
6
|
end
|
7
7
|
|
8
8
|
def check_tasks(type)
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
unless ids.include?(item_storage.id)
|
21
|
-
$tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
|
22
|
-
$tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
|
9
|
+
count = 0
|
10
|
+
tasks = (type == :copy ? $tasks_copy : $tasks_delete)
|
11
|
+
$storages.each do |storage|
|
12
|
+
tasks[storage.name] = [] unless tasks[storage.name]
|
13
|
+
cond = "storage_name = '#{storage.name}' AND status='#{type.to_s}'"
|
14
|
+
ids = tasks[storage.name].map(&:id) + $curr_tasks.map(&:id)
|
15
|
+
limit = FC::Var.get("daemon_tasks_#{type}_group_limit", 1000).to_i
|
16
|
+
cond << "AND id not in (#{ids.join(',')})" if ids.length > 0
|
17
|
+
cond << " LIMIT #{limit}"
|
18
|
+
FC::ItemStorage.where(cond).each do |item_storage|
|
19
|
+
tasks[storage.name] << item_storage
|
23
20
|
$log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
|
21
|
+
count +=1
|
24
22
|
end
|
25
23
|
end
|
24
|
+
$log.debug("Add #{count} #{type} tasks")
|
26
25
|
end
|
27
26
|
end
|
data/lib/fc/db.rb
CHANGED
@@ -191,12 +191,14 @@ module FC
|
|
191
191
|
FC::DB.connect.query("CREATE TRIGGER fc_vars_before_update BEFORE UPDATE on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
|
192
192
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_cycle_time', val='30', descr='time between global daemon checks and storages available checks'")
|
193
193
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_wait_time', val='120', descr='time between runs global daemon if it does not running'")
|
194
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='
|
194
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_copy_group_limit', val='1000', descr='select limit for copy tasks'")
|
195
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_delete_group_limit', val='10000', descr='select limit for delete tasks'")
|
196
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_copy_threads_limit', val='10', descr='copy tasks threads count limit for one storage'")
|
197
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_delete_threads_limit', val='10', descr='delete tasks threads count limit for one storage'")
|
198
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_per_host_limit', val='10', descr='copy tasks count limit for one host'")
|
199
|
+
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_group_limit', val='1000', descr='select limit for create copy tasks'")
|
195
200
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
|
196
201
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
|
197
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
|
198
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='10', descr='tasks threads count limit for one storage'")
|
199
|
-
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='10', descr='copy tasks count limit for one host'")
|
200
202
|
FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_restart_period', val='86400', descr='time between fc-daemon self restart'")
|
201
203
|
end
|
202
204
|
end
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -128,7 +128,7 @@ def storages_change
|
|
128
128
|
storage.url = url unless url.empty?
|
129
129
|
storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
|
130
130
|
storages = FC::Storage.where.map(&:name)
|
131
|
-
copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
|
131
|
+
storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
|
132
132
|
|
133
133
|
puts %Q{\nStorage
|
134
134
|
Name: #{storage.name}
|
@@ -137,7 +137,7 @@ def storages_change
|
|
137
137
|
Url: #{storage.url}
|
138
138
|
Size: #{size_to_human storage.size}
|
139
139
|
Size limit: #{size_to_human storage.size_limit}
|
140
|
-
Copy storages: #{copy_storages}}
|
140
|
+
Copy storages: #{storage.copy_storages}}
|
141
141
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
142
142
|
puts ""
|
143
143
|
if s == "y" || s == "yes"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-11-
|
12
|
+
date: 2013-11-12 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rb-readline
|
@@ -146,9 +146,10 @@ files:
|
|
146
146
|
- lib/daemon.rb
|
147
147
|
- lib/daemon/base_thread.rb
|
148
148
|
- lib/daemon/check_thread.rb
|
149
|
+
- lib/daemon/copy_task_thread.rb
|
150
|
+
- lib/daemon/delete_task_thread.rb
|
149
151
|
- lib/daemon/global_daemon_thread.rb
|
150
152
|
- lib/daemon/run_tasks_thread.rb
|
151
|
-
- lib/daemon/task_thread.rb
|
152
153
|
- lib/daemon/update_tasks_thread.rb
|
153
154
|
- lib/fc/base.rb
|
154
155
|
- lib/fc/db.rb
|
@@ -192,7 +193,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
192
193
|
version: '0'
|
193
194
|
segments:
|
194
195
|
- 0
|
195
|
-
hash:
|
196
|
+
hash: -4221257389535843985
|
196
197
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
197
198
|
none: false
|
198
199
|
requirements:
|
@@ -201,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
201
202
|
version: '0'
|
202
203
|
segments:
|
203
204
|
- 0
|
204
|
-
hash:
|
205
|
+
hash: -4221257389535843985
|
205
206
|
requirements: []
|
206
207
|
rubyforge_project:
|
207
208
|
rubygems_version: 1.8.24
|
data/lib/daemon/task_thread.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
class TaskThread < BaseThread
|
2
|
-
def go(storage_name)
|
3
|
-
return unless $tasks[storage_name]
|
4
|
-
while task = $tasks[storage_name].shift do
|
5
|
-
$curr_tasks << task
|
6
|
-
$log.debug("TaskThread(#{storage_name}): run task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
|
7
|
-
if task[:action] == :delete
|
8
|
-
make_delete(task)
|
9
|
-
elsif task[:action] == :copy
|
10
|
-
make_copy(task)
|
11
|
-
else
|
12
|
-
error "Unknown task action: #{task[:action]}"
|
13
|
-
end
|
14
|
-
$curr_tasks.delete(task)
|
15
|
-
$log.debug("TaskThread(#{storage_name}): Finish task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
|
16
|
-
exit if $exit_signal
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def make_delete(task)
|
21
|
-
item_storage = task[:item_storage]
|
22
|
-
storage = $storages.detect{|s| s.name == item_storage.storage_name}
|
23
|
-
item = FC::Item.find(item_storage.item_id)
|
24
|
-
$log.debug("Delete #{storage.path}#{item.name}")
|
25
|
-
storage.delete_file(item.name)
|
26
|
-
item_storage.delete
|
27
|
-
rescue Exception => e
|
28
|
-
error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
|
29
|
-
$curr_tasks.delete(task)
|
30
|
-
end
|
31
|
-
|
32
|
-
def make_copy(task)
|
33
|
-
sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_limit', 10).to_i
|
34
|
-
$copy_count += 1
|
35
|
-
item_storage = task[:item_storage]
|
36
|
-
storage = $storages.detect{|s| s.name == item_storage.storage_name}
|
37
|
-
item = FC::Item.find(item_storage.item_id)
|
38
|
-
return nil unless item && item.status == 'ready'
|
39
|
-
src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
|
40
|
-
unless src_item_storage
|
41
|
-
$log.info("Item ##{item.id} #{item.name} has no ready item_storage")
|
42
|
-
return nil
|
43
|
-
end
|
44
|
-
src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
|
45
|
-
$log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
|
46
|
-
item.copy_item_storage(src_storage, storage, item_storage)
|
47
|
-
rescue Exception => e
|
48
|
-
error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
|
49
|
-
$curr_tasks.delete(task)
|
50
|
-
ensure
|
51
|
-
$copy_count -= 1 if item_storage && $copy_count > 0
|
52
|
-
end
|
53
|
-
end
|