filecluster 0.5.19 → 0.5.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +2 -0
- data/lib/daemon/check_thread.rb +1 -0
- data/lib/daemon/global_daemon_thread.rb +6 -2
- data/lib/fc/db.rb +18 -4
- data/lib/fc/storage.rb +2 -2
- data/lib/fc/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8d483e4b277c06ca5555620bc5ae7f23f4e524d6
|
4
|
+
data.tar.gz: e7ee71236466346353c3ef599bd3bd29ac47551c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4520da860c08b62f5815e39625c27c2e95d132dc1f3f135883ad6d83c2f4c8eff494544aa5874f96f0fbc8b578f7404f3f065613e8838bef858c518c70b440f4
|
7
|
+
data.tar.gz: 402c406c37d2c560df3da5a66f0a3c231c89b89edb542e19468fd2e151c3868ce338b223f3dcf364b14022ae34d28c0270b4b6d3c0610b81dd08935943c1b31c
|
data/README.md
CHANGED
@@ -64,6 +64,8 @@ Can be used the following variables:
|
|
64
64
|
|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
|
65
65
|
|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
|
66
66
|
|daemon_restart_period|86400|time between fc-daemon self restart|
|
67
|
+
|daemon_global_delete_limit|1000|limits number of deleted items per query|
|
68
|
+
|daemon_global_delete_dela|1|delay in seconds(float) between items delete query|
|
67
69
|
|
68
70
|
## Usage
|
69
71
|
|
data/lib/daemon/check_thread.rb
CHANGED
@@ -96,11 +96,15 @@ class GlobalDaemonThread < BaseThread
|
|
96
96
|
$log.debug("GlobalDaemonThread: delete_deleted_items")
|
97
97
|
|
98
98
|
r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
|
99
|
-
|
100
|
-
|
99
|
+
item_ids = r.map{|row| row['id']}
|
100
|
+
limit = FC::Var.get('daemon_global_delete_limit', 1000).to_i
|
101
|
+
limit = 1000 if limit < 2
|
102
|
+
delay = FC::Var.get('daemon_global_delete_delay', 1).to_f
|
103
|
+
item_ids.each_slice(limit) do |ids|
|
101
104
|
ids = ids.join(',')
|
102
105
|
FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
|
103
106
|
$log.info("GlobalDaemonThread: delete items #{ids}")
|
107
|
+
sleep delay if delay > 0
|
104
108
|
end
|
105
109
|
end
|
106
110
|
|
data/lib/fc/db.rb
CHANGED
@@ -4,7 +4,7 @@ require 'psych'
|
|
4
4
|
module FC
|
5
5
|
module DB
|
6
6
|
class << self
|
7
|
-
attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting
|
7
|
+
attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting, :reconnect_block
|
8
8
|
end
|
9
9
|
|
10
10
|
def self.options_yml_path
|
@@ -20,7 +20,7 @@ module FC
|
|
20
20
|
@prefix = @options[:prefix].to_s if @options[:prefix]
|
21
21
|
connection = Mysql2::Client.new(@options)
|
22
22
|
@reconnecting = false
|
23
|
-
@connect_block = nil
|
23
|
+
@connect_block = nil unless @options[:keep_lazy_connection]
|
24
24
|
@connects = {} unless @connects
|
25
25
|
@connects[Thread.current.object_id] = connection
|
26
26
|
end
|
@@ -48,13 +48,22 @@ module FC
|
|
48
48
|
@connect_block = block
|
49
49
|
end
|
50
50
|
|
51
|
+
def self.lazy_reconnect(&block)
|
52
|
+
@reconnect_block = block
|
53
|
+
end
|
54
|
+
|
51
55
|
def self.connect_by_block(options = {})
|
52
56
|
connection = @connect_block.call
|
53
57
|
@options = connection.query_options.clone.merge(symbolize_keys(options))
|
54
58
|
@prefix = @options[:prefix].to_s if @options[:prefix]
|
55
59
|
@connects = {} unless @connects
|
56
60
|
@connects[Thread.current.object_id] = connection
|
57
|
-
@connect_block = nil
|
61
|
+
@connect_block = nil unless @options[:keep_lazy_connection]
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.reconnect_by_block
|
65
|
+
@connects[Thread.current.object_id] = @reconnect_block.call
|
66
|
+
@reconnecting = false
|
58
67
|
end
|
59
68
|
|
60
69
|
def self.connect
|
@@ -86,7 +95,7 @@ module FC
|
|
86
95
|
def self.reconnect
|
87
96
|
close if connect
|
88
97
|
@reconnecting = true
|
89
|
-
connect_by_config(@options)
|
98
|
+
@reconnect_block ? reconnect_by_block : connect_by_config(@options)
|
90
99
|
end
|
91
100
|
|
92
101
|
def self.close
|
@@ -343,5 +352,10 @@ module FC
|
|
343
352
|
def self.migrate_6
|
344
353
|
FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
|
345
354
|
end
|
355
|
+
|
356
|
+
def self.migrate_7
|
357
|
+
FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_limit', val='1000', descr='limits number of deleted items per query'")
|
358
|
+
FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_delay', val='1', descr='delay in seconds between items delete query'")
|
359
|
+
end
|
346
360
|
end
|
347
361
|
end
|
data/lib/fc/storage.rb
CHANGED
@@ -137,7 +137,7 @@ module FC
|
|
137
137
|
raise r if $?.exitstatus != 0
|
138
138
|
else
|
139
139
|
local_path += '/' if File.stat(local_path).directory?
|
140
|
-
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a
|
140
|
+
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
|
141
141
|
r = `#{cmd} 2>&1`
|
142
142
|
raise r if $?.exitstatus != 0
|
143
143
|
end
|
@@ -155,7 +155,7 @@ module FC
|
|
155
155
|
r = `#{cmd} 2>&1`
|
156
156
|
src_path += '/' if $?.exitstatus == 0
|
157
157
|
|
158
|
-
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a
|
158
|
+
cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
|
159
159
|
r = `#{cmd} 2>&1`
|
160
160
|
raise r if $?.exitstatus != 0
|
161
161
|
end
|
data/lib/fc/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.25
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-06-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
217
217
|
version: '0'
|
218
218
|
requirements: []
|
219
219
|
rubyforge_project:
|
220
|
-
rubygems_version: 2.
|
220
|
+
rubygems_version: 2.6.14
|
221
221
|
signing_key:
|
222
222
|
specification_version: 4
|
223
223
|
summary: Distributed storage
|