filecluster 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +18 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +29 -0
- data/Rakefile +11 -0
- data/TODO +9 -0
- data/bin/fc-daemon +70 -0
- data/bin/fc-manage +102 -0
- data/bin/fc-setup-db +47 -0
- data/filecluster.gemspec +24 -0
- data/lib/daemon.rb +91 -0
- data/lib/daemon/base_thread.rb +13 -0
- data/lib/daemon/check_thread.rb +12 -0
- data/lib/daemon/global_daemon_thread.rb +60 -0
- data/lib/daemon/task_thread.rb +37 -0
- data/lib/fc/base.rb +82 -0
- data/lib/fc/db.rb +168 -0
- data/lib/fc/error.rb +12 -0
- data/lib/fc/item.rb +102 -0
- data/lib/fc/item_storage.rb +8 -0
- data/lib/fc/policy.rb +18 -0
- data/lib/fc/storage.rb +80 -0
- data/lib/fc/version.rb +3 -0
- data/lib/filecluster.rb +13 -0
- data/lib/manage.rb +4 -0
- data/lib/manage/policies.rb +70 -0
- data/lib/manage/show.rb +57 -0
- data/lib/manage/storages.rb +87 -0
- data/lib/utils.rb +76 -0
- data/test/base_test.rb +74 -0
- data/test/daemon_test.rb +98 -0
- data/test/db_test.rb +182 -0
- data/test/error_test.rb +15 -0
- data/test/functional_test.rb +97 -0
- data/test/helper.rb +17 -0
- data/test/item_test.rb +49 -0
- data/test/policy_test.rb +34 -0
- data/test/storage_test.rb +29 -0
- data/test/version_test.rb +7 -0
- metadata +199 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 sh
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Dstorage
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'dstorage'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install dstorage
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Contributing
|
24
|
+
|
25
|
+
1. Fork it
|
26
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
28
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/TODO
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
добавить в check storage проверку на дорступность урла
|
2
|
+
|
3
|
+
алерт на доступность в каждоый политике стораджа на запись
|
4
|
+
алерт на доступность стораджей
|
5
|
+
алерт: проверка раз в сутки на количество is задержавшихся в статусе delete и copy дольше суток (NOW - time > 86400)
|
6
|
+
|
7
|
+
bin:
|
8
|
+
управление из командной строки: добавление, изменение и получение статуса объектов
|
9
|
+
в ручную запускаемая задача синхронизации фс и бд
|
data/bin/fc-daemon
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
|
4
|
+
require 'psych'
|
5
|
+
require 'logger'
|
6
|
+
require 'optparse'
|
7
|
+
require 'filecluster'
|
8
|
+
require 'utils'
|
9
|
+
require 'daemon'
|
10
|
+
|
11
|
+
$storages = []
|
12
|
+
$tasks = {} # tasks by storage name
|
13
|
+
$curr_task = {} # task by storage name
|
14
|
+
$tasks_threads = {} # threads by storage name
|
15
|
+
$check_threads = {} # threads by storage name
|
16
|
+
$exit_signal = false
|
17
|
+
$global_daemon_thread = nil
|
18
|
+
|
19
|
+
default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
|
20
|
+
descriptions = {
|
21
|
+
:config => {:short => 'c', :full => 'config', :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
|
22
|
+
:log_level => {:short => 'l', :full => 'log_level', :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},
|
23
|
+
:cycle_time => {:short => 't', :full => 'time', :default => 30, :text => 'Time between checks database and storages available, default 30'},
|
24
|
+
:global_wait => {:short => 'g', :full => 'wait', :default => 120, :text => 'Time between runs global daemon if it does not running, default 120'},
|
25
|
+
:curr_host => {:short => 'h', :full => 'host', :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
|
26
|
+
}
|
27
|
+
desc = %q{Run FileCluster daemon.
|
28
|
+
Usage: fc-daemon [options]}
|
29
|
+
options = option_parser_init(descriptions, desc)
|
30
|
+
FC::Storage.instance_variable_set(:@uname, options[:curr_host]) if options[:curr_host] && options[:curr_host] != FC::Storage.curr_host
|
31
|
+
|
32
|
+
STDOUT.sync = true
|
33
|
+
$log = Logger.new(STDOUT)
|
34
|
+
$log.formatter = proc do |severity, datetime, progname, msg|
|
35
|
+
"[#{datetime}] [#{severity}] [#{Thread.current.object_id}] #{msg}\n"
|
36
|
+
end
|
37
|
+
$log.level = Logger.const_get(options[:log_level].upcase)
|
38
|
+
$log.info('Started')
|
39
|
+
|
40
|
+
db_options = Psych.load(File.read(options[:config]))
|
41
|
+
FC::DB.connect_by_config(db_options.merge(:reconnect => true, :multi_threads => true))
|
42
|
+
$log.info('Connected to database')
|
43
|
+
|
44
|
+
def quit_on_quit
|
45
|
+
$log.info('Exit signal')
|
46
|
+
$exit_signal = true
|
47
|
+
end
|
48
|
+
trap("TERM") {quit_on_quit}
|
49
|
+
trap("INT") {quit_on_quit}
|
50
|
+
|
51
|
+
while true do
|
52
|
+
if $exit_signal
|
53
|
+
$log.debug('wait tasks_threads')
|
54
|
+
$tasks_threads.each {|t| t.join}
|
55
|
+
if $global_daemon_thread
|
56
|
+
$log.debug('wait global_daemon_thread')
|
57
|
+
$global_daemon_thread.join
|
58
|
+
end
|
59
|
+
$log.info('Exit')
|
60
|
+
exit
|
61
|
+
else
|
62
|
+
run_global_daemon options[:global_wait].to_i
|
63
|
+
update_storages
|
64
|
+
storages_check
|
65
|
+
update_tasks
|
66
|
+
run_tasks
|
67
|
+
end
|
68
|
+
$log.debug('sleep')
|
69
|
+
sleep options[:cycle_time].to_i
|
70
|
+
end
|
data/bin/fc-manage
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
|
4
|
+
require 'psych'
|
5
|
+
require 'logger'
|
6
|
+
require 'optparse'
|
7
|
+
require 'filecluster'
|
8
|
+
require 'utils'
|
9
|
+
require 'manage'
|
10
|
+
|
11
|
+
default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
|
12
|
+
descriptions = {
|
13
|
+
:config => {:short => 'c', :full => 'config', :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
|
14
|
+
:curr_host => {:short => 'h', :full => 'host', :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
|
15
|
+
}
|
16
|
+
commands_help = {
|
17
|
+
'storages' => [
|
18
|
+
'show list and manage storages',
|
19
|
+
%q{Usage: fc-manage [options] storages <command>
|
20
|
+
Command:
|
21
|
+
list show all stotages for all hosts
|
22
|
+
show <name> show full info for storage
|
23
|
+
add add new storage
|
24
|
+
rm <name> delete storage
|
25
|
+
}],
|
26
|
+
'policies' => [
|
27
|
+
'show list and manage plicies',
|
28
|
+
%q{Usage: fc-manage [options] plicies <command>
|
29
|
+
Command:
|
30
|
+
list show all plicies
|
31
|
+
show <id> show full info for policy
|
32
|
+
add add new policy
|
33
|
+
rm <id> delete policy
|
34
|
+
}],
|
35
|
+
'show' => [
|
36
|
+
'show variable',
|
37
|
+
%q{Usage: fc-manage [options] show <variable>
|
38
|
+
Variable:
|
39
|
+
current_host show current host
|
40
|
+
global_daemon show host and uptime where run global daemon
|
41
|
+
errors [<count>] show last count (default 10) errors
|
42
|
+
host_info [<host>] show info for host (default current host)
|
43
|
+
items_info show items statistics
|
44
|
+
}]
|
45
|
+
}
|
46
|
+
desc = %q{Get info and manage for storages, policies and items.
|
47
|
+
Usage: fc-manage [options] <command> [<args>]
|
48
|
+
Commands:
|
49
|
+
}
|
50
|
+
commands_help.each{|key, val| desc << " #{key}#{" "*(10-key.size)}#{val[0]}\n"}
|
51
|
+
desc << " help show help for commands ('fc-manage help <command>')\n"
|
52
|
+
$options = option_parser_init(descriptions, desc)
|
53
|
+
FC::Storage.instance_variable_set(:@uname, $options[:curr_host]) if $options[:curr_host] && $options[:curr_host] != FC::Storage.curr_host
|
54
|
+
trap("INT", proc {exit})
|
55
|
+
|
56
|
+
STDOUT.sync = true
|
57
|
+
db_options = Psych.load(File.read($options[:config]))
|
58
|
+
FC::DB.connect_by_config(db_options.merge(:reconnect => true, :multi_threads => true))
|
59
|
+
|
60
|
+
command = ARGV[0]
|
61
|
+
if ARGV.empty?
|
62
|
+
puts $options['optparse']
|
63
|
+
exit
|
64
|
+
end
|
65
|
+
|
66
|
+
case command
|
67
|
+
when 'help'
|
68
|
+
if !ARGV[1]
|
69
|
+
puts $options['optparse']
|
70
|
+
elsif commands_help[ARGV[1]]
|
71
|
+
puts commands_help[ARGV[1]][1]
|
72
|
+
else
|
73
|
+
puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
|
74
|
+
end
|
75
|
+
when 'show'
|
76
|
+
if !ARGV[1]
|
77
|
+
puts "Need variable name. See 'fc-manage help show'."
|
78
|
+
elsif self.private_methods.member?("show_#{ARGV[1]}".to_sym)
|
79
|
+
send "show_#{ARGV[1]}"
|
80
|
+
else
|
81
|
+
puts "Unknown variable. See 'fc-manage help show'."
|
82
|
+
end
|
83
|
+
when 'storages'
|
84
|
+
if !ARGV[1]
|
85
|
+
puts "Need command. See 'fc-manage help storages'."
|
86
|
+
elsif self.private_methods.member?("storages_#{ARGV[1]}".to_sym)
|
87
|
+
send "storages_#{ARGV[1]}"
|
88
|
+
else
|
89
|
+
puts "Unknown command. See 'fc-manage help storages'."
|
90
|
+
end
|
91
|
+
when 'policies'
|
92
|
+
if !ARGV[1]
|
93
|
+
puts "Need command. See 'fc-manage help policies'."
|
94
|
+
elsif self.private_methods.member?("policies_#{ARGV[1]}".to_sym)
|
95
|
+
send "policies_#{ARGV[1]}"
|
96
|
+
else
|
97
|
+
puts "Unknown command. See 'fc-manage help policies'."
|
98
|
+
end
|
99
|
+
else
|
100
|
+
puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
|
101
|
+
end
|
102
|
+
|
data/bin/fc-setup-db
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
|
4
|
+
require 'optparse'
|
5
|
+
require 'psych'
|
6
|
+
require 'filecluster'
|
7
|
+
require 'utils'
|
8
|
+
require 'readline'
|
9
|
+
|
10
|
+
descriptions = {
|
11
|
+
:host => {:short => 'h', :full => 'host', :default => 'localhost', :text => 'mysql host name, default "localhost"', :save => true},
|
12
|
+
:database => {:short => 'd', :full => 'db', :default => 'fc', :text => 'mysql database, default "fc"', :save => true},
|
13
|
+
:username => {:short => 'u', :full => 'user', :default => 'root', :text => 'mysql user, default "root"', :save => true},
|
14
|
+
:password => {:short => 'p', :full => 'password', :default => '', :text => 'mysql password, default ""', :save => true},
|
15
|
+
:port => {:short => 'P', :full => 'port', :default => '3306', :text => 'mysql port, default "3306"', :save => true},
|
16
|
+
:prefix => {:short => 't', :full => 'prefix', :default => '', :text => 'tables prefix, default ""', :save => true},
|
17
|
+
:init_tables =>{:short => 'i', :full => 'init', :default => false, :text => 'init tables, default no', :no_val => true}
|
18
|
+
}
|
19
|
+
desc = %q{Setup FileCluster database connection options.
|
20
|
+
Create tables if nessary.
|
21
|
+
Usage: fc-init-db [options]}
|
22
|
+
options = option_parser_init(descriptions, desc)
|
23
|
+
options.delete('optparse')
|
24
|
+
trap("INT", proc {exit})
|
25
|
+
|
26
|
+
puts options.inspect.gsub(/[\{\}\:]/, "").gsub(", ", "\n").gsub(/(.{7,})=>/, "\\1:\t").gsub("=>", ":\t\t")
|
27
|
+
|
28
|
+
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
29
|
+
puts ""
|
30
|
+
if s == "y" || s == "yes"
|
31
|
+
print "Test connection.. "
|
32
|
+
FC::DB.connect_by_config(options)
|
33
|
+
puts "ok"
|
34
|
+
if options[:init_tables]
|
35
|
+
print "Make tables.. "
|
36
|
+
FC::DB.init_db
|
37
|
+
puts "ok"
|
38
|
+
end
|
39
|
+
print "Save to config.. "
|
40
|
+
options.select!{|key, val| descriptions[key][:save]}
|
41
|
+
File.open(File.expand_path(File.dirname(__FILE__))+'/db.yml', 'w') do |f|
|
42
|
+
f.write(options.to_yaml)
|
43
|
+
end
|
44
|
+
puts "ok"
|
45
|
+
else
|
46
|
+
puts "Canceled."
|
47
|
+
end
|
data/filecluster.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/fc/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["sh"]
|
6
|
+
gem.email = ["cntyrf@gmail.com"]
|
7
|
+
gem.description = %q{Distributed storage}
|
8
|
+
gem.summary = %q{Distributed storage}
|
9
|
+
gem.homepage = ""
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "filecluster"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = FC::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency "bundler"
|
19
|
+
gem.add_development_dependency "test-unit"
|
20
|
+
gem.add_development_dependency "rake"
|
21
|
+
gem.add_development_dependency "mysql2"
|
22
|
+
gem.add_development_dependency "shoulda-context"
|
23
|
+
gem.add_development_dependency "mocha", ">= 0.13.3"
|
24
|
+
end
|
data/lib/daemon.rb
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
require "date"
|
2
|
+
require "daemon/base_thread"
|
3
|
+
require "daemon/global_daemon_thread"
|
4
|
+
require "daemon/check_thread"
|
5
|
+
require "daemon/task_thread"
|
6
|
+
|
7
|
+
def error(msg, options = {})
|
8
|
+
$log.error(msg)
|
9
|
+
FC::Error.new(options.merge(:host => FC::Storage.curr_host, :message => msg)).save
|
10
|
+
end
|
11
|
+
|
12
|
+
class << FC::Error
|
13
|
+
def raise(msg, options = {})
|
14
|
+
error(msg, options)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def run_global_daemon(timeout)
|
19
|
+
$log.debug('Run global daemon check')
|
20
|
+
r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
|
21
|
+
if !r || r['curr_time'].to_i - r['time'].to_i > timeout
|
22
|
+
$log.debug('Set global daemon host to current')
|
23
|
+
FC::DB.connect.query("REPLACE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}', name='global_daemon_host'")
|
24
|
+
sleep 1
|
25
|
+
r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
|
26
|
+
end
|
27
|
+
if r['val'] == FC::Storage.curr_host
|
28
|
+
if !$global_daemon_thread || !$global_daemon_thread.alive?
|
29
|
+
$log.debug("spawn GlobalDaemonThread")
|
30
|
+
$global_daemon_thread = GlobalDaemonThread.new(timeout)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
if $global_daemon_thread
|
34
|
+
$log.warn("Kill global daemon thread (new host = #{r['host']})")
|
35
|
+
$global_daemon_thread.exit
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def update_storages
|
41
|
+
$log.debug('Update storages')
|
42
|
+
$storages = FC::Storage.where('host = ?', FC::Storage.curr_host)
|
43
|
+
end
|
44
|
+
|
45
|
+
def storages_check
|
46
|
+
$log.debug('Run storages check')
|
47
|
+
$check_threads.each do |storage_name, thread|
|
48
|
+
if thread.alive?
|
49
|
+
error "Storage #{storage_name} check timeout"
|
50
|
+
thread.exit
|
51
|
+
end
|
52
|
+
end
|
53
|
+
$storages.each do|storage|
|
54
|
+
$log.debug("spawn CheckThread for #{storage.name}")
|
55
|
+
$check_threads[storage.name] = CheckThread.new(storage.name)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def update_tasks
|
60
|
+
$log.debug('Update tasks')
|
61
|
+
return if $storages.length == 0
|
62
|
+
|
63
|
+
def check_tasks(type)
|
64
|
+
storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
|
65
|
+
cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
|
66
|
+
ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
|
67
|
+
flatten.map{|task| task[:item_storage].id}
|
68
|
+
$curr_task.map{|storage_name, task| ids << task[:item_storage].id if task && task[:action] == type}
|
69
|
+
|
70
|
+
cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
|
71
|
+
FC::ItemStorage.where(cond).each do |item_storage|
|
72
|
+
$tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
|
73
|
+
$tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
|
74
|
+
$log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
check_tasks(:delete)
|
79
|
+
check_tasks(:copy)
|
80
|
+
end
|
81
|
+
|
82
|
+
def run_tasks
|
83
|
+
$log.debug('Run tasks')
|
84
|
+
$storages.each do |storage|
|
85
|
+
thread = $tasks_threads[storage.name]
|
86
|
+
if (!thread || !thread.alive?) && $tasks[storage.name] && $tasks[storage.name].size > 0
|
87
|
+
$log.debug("spawn TaskThread for #{storage.name}")
|
88
|
+
$tasks_threads[storage.name] = TaskThread.new(storage.name)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class BaseThread < Thread
|
2
|
+
def initialize(*args)
|
3
|
+
super(*args) do |*p|
|
4
|
+
begin
|
5
|
+
self.go(*p)
|
6
|
+
rescue Exception => e
|
7
|
+
error "#{self.class}: #{e.message}; #{e.backtrace.join(', ')}"
|
8
|
+
end
|
9
|
+
FC::DB.close
|
10
|
+
$log.debug("close #{self.class}")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|