filecluster 0.5.21 → 0.5.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +2 -0
- data/lib/daemon/check_thread.rb +8 -2
- data/lib/daemon/global_daemon_thread.rb +6 -2
- data/lib/fc/db.rb +18 -4
- data/lib/fc/storage.rb +8 -3
- data/lib/fc/version.rb +1 -1
- data/lib/manage/storages.rb +22 -17
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 1ee01d84ce7df5b8ebe6279a92c0c14c488a5c82
|
4
|
+
data.tar.gz: 7e2f785e88144ed619e55cdec6bf038264278052
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7735a60395e14dac92d0078d60c63458ac795847e55292a6b4616e29f072d50b7417f74a6cf2bb0e775c0452b2706363f10b6d82aaafe41cd5b8d371bf242c89
|
7
|
+
data.tar.gz: 1e670216069549f82614300d99c53422e73782ab96c5e3ed459c32bc4d8b27dcc77eaa47b813a68e6a3a5d3fda0cc60f498e48435d19c1fab30ece49f0eee530
|
data/README.md
CHANGED
@@ -64,6 +64,8 @@ Can be used the following variables:
|
|
64
64
|
|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
|
65
65
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
66
66
|
|daemon_restart_period|86400|time between fc-daemon self restart|
|
67
|
+
|daemon_global_delete_limit|1000|limits number of deleted items per query|
|
68
|
+
|daemon_global_delete_dela|1|delay in seconds(float) between items delete query|
|
67
69
|
|
68
70
|
## Usage
|
69
71
|
|
data/lib/daemon/check_thread.rb
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
class CheckThread < BaseThread
|
2
2
|
require "net/http"
|
3
|
+
|
4
|
+
@@storage_http_check = {}
|
5
|
+
HTTP_RETRIES = 3
|
6
|
+
|
3
7
|
def go(storage_name)
|
4
8
|
$log.debug("CheckThread: Run stotage check for #{storage_name}")
|
5
9
|
storage = $storages.detect{|s| s.name == storage_name}
|
@@ -10,7 +14,7 @@ class CheckThread < BaseThread
|
|
10
14
|
else
|
11
15
|
error "Storage #{storage.name} with path #{storage.path} not writable"
|
12
16
|
end
|
13
|
-
check_http(storage)
|
17
|
+
check_http(storage) if storage.http_check_enabled?
|
14
18
|
$log.debug("CheckThread: Finish stotage check for #{storage_name}")
|
15
19
|
end
|
16
20
|
|
@@ -22,8 +26,10 @@ class CheckThread < BaseThread
|
|
22
26
|
resp = request.start { |http| http.get(uri.path) } rescue nil
|
23
27
|
if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
|
24
28
|
storage.update_http_check_time
|
29
|
+
@@storage_http_check[storage.name] = 0
|
25
30
|
else
|
26
|
-
|
31
|
+
@@storage_http_check[storage.name] = @@storage_http_check[storage.name].to_i + 1
|
32
|
+
error("Storage #{storage.name} with url #{storage.url} not readable") if @@storage_http_check[storage.name] > HTTP_RETRIES
|
27
33
|
end
|
28
34
|
rescue => err
|
29
35
|
$log.error("CheckThread: check_http error: #{err}")
|
@@ -96,11 +96,15 @@ class GlobalDaemonThread < BaseThread
|
|
96
96
|
$log.debug("GlobalDaemonThread: delete_deleted_items")
|
97
97
|
|
98
98
|
r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
|
99
|
-
|
100
|
-
|
99
|
+
item_ids = r.map{|row| row['id']}
|
100
|
+
limit = FC::Var.get('daemon_global_delete_limit', 1000).to_i
|
101
|
+
limit = 1000 if limit < 2
|
102
|
+
delay = FC::Var.get('daemon_global_delete_delay', 1).to_f
|
103
|
+
item_ids.each_slice(limit) do |ids|
|
101
104
|
ids = ids.join(',')
|
102
105
|
FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
|
103
106
|
$log.info("GlobalDaemonThread: delete items #{ids}")
|
107
|
+
sleep delay if delay > 0
|
104
108
|
end
|
105
109
|
end
|
106
110
|
|
data/lib/fc/db.rb
CHANGED
@@ -4,7 +4,7 @@ require 'psych'
|
|
4
4
|
module FC
|
5
5
|
module DB
|
6
6
|
class << self
|
7
|
-
attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting
|
7
|
+
attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting, :reconnect_block
|
8
8
|
end
|
9
9
|
|
10
10
|
def self.options_yml_path
|
@@ -20,7 +20,7 @@ module FC
|
|
20
20
|
@prefix = @options[:prefix].to_s if @options[:prefix]
|
21
21
|
connection = Mysql2::Client.new(@options)
|
22
22
|
@reconnecting = false
|
23
|
-
@connect_block = nil
|
23
|
+
@connect_block = nil unless @options[:keep_lazy_connection]
|
24
24
|
@connects = {} unless @connects
|
25
25
|
@connects[Thread.current.object_id] = connection
|
26
26
|
end
|
@@ -48,13 +48,22 @@ module FC
|
|
48
48
|
@connect_block = block
|
49
49
|
end
|
50
50
|
|
51
|
+
def self.lazy_reconnect(&block)
|
52
|
+
@reconnect_block = block
|
53
|
+
end
|
54
|
+
|
51
55
|
def self.connect_by_block(options = {})
|
52
56
|
connection = @connect_block.call
|
53
57
|
@options = connection.query_options.clone.merge(symbolize_keys(options))
|
54
58
|
@prefix = @options[:prefix].to_s if @options[:prefix]
|
55
59
|
@connects = {} unless @connects
|
56
60
|
@connects[Thread.current.object_id] = connection
|
57
|
-
@connect_block = nil
|
61
|
+
@connect_block = nil unless @options[:keep_lazy_connection]
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.reconnect_by_block
|
65
|
+
@connects[Thread.current.object_id] = @reconnect_block.call
|
66
|
+
@reconnecting = false
|
58
67
|
end
|
59
68
|
|
60
69
|
def self.connect
|
@@ -86,7 +95,7 @@ module FC
|
|
86
95
|
def self.reconnect
|
87
96
|
close if connect
|
88
97
|
@reconnecting = true
|
89
|
-
connect_by_config(@options)
|
98
|
+
@reconnect_block ? reconnect_by_block : connect_by_config(@options)
|
90
99
|
end
|
91
100
|
|
92
101
|
def self.close
|
@@ -343,5 +352,10 @@ module FC
|
|
343
352
|
def self.migrate_6
|
344
353
|
FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
|
345
354
|
end
|
355
|
+
|
356
|
+
def self.migrate_7
|
357
|
+
FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_limit', val='1000', descr='limits number of deleted items per query'")
|
358
|
+
FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_delay', val='1', descr='delay in seconds between items delete query'")
|
359
|
+
end
|
346
360
|
end
|
347
361
|
end
|
data/lib/fc/storage.rb
CHANGED
@@ -95,6 +95,7 @@ module FC
|
|
95
95
|
end
|
96
96
|
|
97
97
|
def update_http_check_time
|
98
|
+
return unless http_check_enabled?
|
98
99
|
self.http_check_time = Time.new.to_i
|
99
100
|
save
|
100
101
|
end
|
@@ -104,9 +105,13 @@ module FC
|
|
104
105
|
end
|
105
106
|
|
106
107
|
def http_check_time_delay
|
107
|
-
Time.new.to_i - http_check_time.to_i
|
108
|
+
http_check_enabled? ? Time.new.to_i - http_check_time.to_i : 0
|
108
109
|
end
|
109
110
|
|
111
|
+
def http_check_enabled?
|
112
|
+
http_check_time.to_i >= 0
|
113
|
+
end
|
114
|
+
|
110
115
|
def up?
|
111
116
|
check_time_delay < self.class.check_time_limit
|
112
117
|
end
|
@@ -137,7 +142,7 @@ module FC
|
|
137
142
|
raise r if $?.exitstatus != 0
|
138
143
|
else
|
139
144
|
local_path += '/' if File.stat(local_path).directory?
|
140
|
-
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a
|
145
|
+
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
|
141
146
|
r = `#{cmd} 2>&1`
|
142
147
|
raise r if $?.exitstatus != 0
|
143
148
|
end
|
@@ -155,7 +160,7 @@ module FC
|
|
155
160
|
r = `#{cmd} 2>&1`
|
156
161
|
src_path += '/' if $?.exitstatus == 0
|
157
162
|
|
158
|
-
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a
|
163
|
+
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
|
159
164
|
r = `#{cmd} 2>&1`
|
160
165
|
raise r if $?.exitstatus != 0
|
161
166
|
end
|
data/lib/fc/version.rb
CHANGED
data/lib/manage/storages.rb
CHANGED
@@ -20,21 +20,22 @@ def storages_show
|
|
20
20
|
if storage = find_storage
|
21
21
|
count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
|
22
22
|
puts %Q{Storage
|
23
|
-
Name:
|
24
|
-
Host:
|
25
|
-
DC:
|
26
|
-
Path:
|
27
|
-
Url:
|
28
|
-
Url weight:
|
29
|
-
Write weight
|
30
|
-
Size:
|
31
|
-
Free:
|
32
|
-
Size limit:
|
33
|
-
Size type:
|
34
|
-
Copy storages:
|
35
|
-
Check time:
|
36
|
-
|
37
|
-
|
23
|
+
Name: #{storage.name}
|
24
|
+
Host: #{storage.host}
|
25
|
+
DC: #{storage.dc}
|
26
|
+
Path: #{storage.path}
|
27
|
+
Url: #{storage.url}
|
28
|
+
Url weight: #{storage.url_weight}
|
29
|
+
Write weight #{storage.write_weight}
|
30
|
+
Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
|
31
|
+
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
32
|
+
Size limit: #{size_to_human storage.size_limit}
|
33
|
+
Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
|
34
|
+
Copy storages: #{storage.copy_storages}
|
35
|
+
Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
|
36
|
+
Check http time: #{storage.http_check_enabled? ? "#{Time.at(storage.http_check_time)} (#{storage.http_check_time_delay} seconds ago)" : 'disabled'}
|
37
|
+
Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
|
38
|
+
Items storages: #{count}}
|
38
39
|
end
|
39
40
|
end
|
40
41
|
|
@@ -56,13 +57,14 @@ def storages_add
|
|
56
57
|
size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
|
57
58
|
end
|
58
59
|
|
60
|
+
check_http = %(y yes).include?(stdin_read_val('Check http (y/n)?').downcase) ? 0 : -1
|
59
61
|
copy_storages = stdin_read_val('Copy storages', true)
|
60
62
|
storages = FC::Storage.where.map(&:name)
|
61
63
|
copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
|
62
64
|
begin
|
63
65
|
path = path +'/' unless path[-1] == '/'
|
64
66
|
path = '/' + path unless path[0] == '/'
|
65
|
-
storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
|
67
|
+
storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size, :http_check_time => check_http)
|
66
68
|
print 'Calc current size.. '
|
67
69
|
size = storage.file_size('', true)
|
68
70
|
puts "ok"
|
@@ -88,6 +90,7 @@ def storages_add
|
|
88
90
|
Free: #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
|
89
91
|
Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
|
90
92
|
Size limit: #{size_to_human size_limit}
|
93
|
+
Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
|
91
94
|
Copy storages #{copy_storages}}
|
92
95
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
93
96
|
puts ""
|
@@ -152,6 +155,7 @@ def storages_change
|
|
152
155
|
auto_size = 0
|
153
156
|
size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
|
154
157
|
end
|
158
|
+
check_http = %(y yes).include?(stdin_read_val("Check http (now #{storage.http_check_enabled? ? 'yes' : 'no'})", true, storage.http_check_enabled? ? 'yes' : 'no').downcase)
|
155
159
|
copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
|
156
160
|
|
157
161
|
storage.dc = dc unless dc.empty?
|
@@ -172,7 +176,7 @@ def storages_change
|
|
172
176
|
storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
|
173
177
|
storages = FC::Storage.where.map(&:name)
|
174
178
|
storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
|
175
|
-
|
179
|
+
storage.http_check_time = (check_http ? 0 : -1) if storage.http_check_enabled? != check_http
|
176
180
|
puts %Q{\nStorage
|
177
181
|
Name: #{storage.name}
|
178
182
|
DC: #{storage.dc}
|
@@ -185,6 +189,7 @@ def storages_change
|
|
185
189
|
Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
|
186
190
|
Size type: #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
|
187
191
|
Size limit: #{size_to_human storage.size_limit }
|
192
|
+
Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
|
188
193
|
Copy storages: #{storage.copy_storages}}
|
189
194
|
s = Readline.readline("Continue? (y/n) ", false).strip.downcase
|
190
195
|
puts ""
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
217
217
|
version: '0'
|
218
218
|
requirements: []
|
219
219
|
rubyforge_project:
|
220
|
-
rubygems_version: 2.
|
220
|
+
rubygems_version: 2.6.14
|
221
221
|
signing_key:
|
222
222
|
specification_version: 4
|
223
223
|
summary: Distributed storage
|