filecluster 0.5.21 → 0.5.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA256:
3
- metadata.gz: 4bca4b52453d8184c6b3aa8f6915a4077c92fdefbc5a8c3690031d0dbfeeebdc
4
- data.tar.gz: 7387acb1f8c92b285311425b8ccb26da2ea05ad5fb35759d2e110fb6035da4a7
2
+ SHA1:
3
+ metadata.gz: 1ee01d84ce7df5b8ebe6279a92c0c14c488a5c82
4
+ data.tar.gz: 7e2f785e88144ed619e55cdec6bf038264278052
5
5
  SHA512:
6
- metadata.gz: 77502723c7312da226e1207527fa57654646077cffc5fed30ec2c83c99330e30851b4951e79dbab09374d95ff49904f1db7a442225a393b5d9dd58c39de8f4cc
7
- data.tar.gz: e814ddae4b981950bb78f585704d5b5c9fe7d2801cac3f807c9f42ff1e85a8709202ee6b8ace7cc3f38294a11688fca27fc66f00f52e00dc69581c9f9bfb9607
6
+ metadata.gz: 7735a60395e14dac92d0078d60c63458ac795847e55292a6b4616e29f072d50b7417f74a6cf2bb0e775c0452b2706363f10b6d82aaafe41cd5b8d371bf242c89
7
+ data.tar.gz: 1e670216069549f82614300d99c53422e73782ab96c5e3ed459c32bc4d8b27dcc77eaa47b813a68e6a3a5d3fda0cc60f498e48435d19c1fab30ece49f0eee530
data/README.md CHANGED
@@ -64,6 +64,8 @@ Can be used the following variables:
64
64
  |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
65
65
  |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
66
66
  |daemon_restart_period|86400|time between fc-daemon self restart|
67
+ |daemon_global_delete_limit|1000|limits number of deleted items per query|
68
+ |daemon_global_delete_dela|1|delay in seconds(float) between items delete query|
67
69
 
68
70
  ## Usage
69
71
 
@@ -1,5 +1,9 @@
1
1
  class CheckThread < BaseThread
2
2
  require "net/http"
3
+
4
+ @@storage_http_check = {}
5
+ HTTP_RETRIES = 3
6
+
3
7
  def go(storage_name)
4
8
  $log.debug("CheckThread: Run stotage check for #{storage_name}")
5
9
  storage = $storages.detect{|s| s.name == storage_name}
@@ -10,7 +14,7 @@ class CheckThread < BaseThread
10
14
  else
11
15
  error "Storage #{storage.name} with path #{storage.path} not writable"
12
16
  end
13
- check_http(storage)
17
+ check_http(storage) if storage.http_check_enabled?
14
18
  $log.debug("CheckThread: Finish stotage check for #{storage_name}")
15
19
  end
16
20
 
@@ -22,8 +26,10 @@ class CheckThread < BaseThread
22
26
  resp = request.start { |http| http.get(uri.path) } rescue nil
23
27
  if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
24
28
  storage.update_http_check_time
29
+ @@storage_http_check[storage.name] = 0
25
30
  else
26
- error "Storage #{storage.name} with url #{storage.url} not readable"
31
+ @@storage_http_check[storage.name] = @@storage_http_check[storage.name].to_i + 1
32
+ error("Storage #{storage.name} with url #{storage.url} not readable") if @@storage_http_check[storage.name] > HTTP_RETRIES
27
33
  end
28
34
  rescue => err
29
35
  $log.error("CheckThread: check_http error: #{err}")
@@ -96,11 +96,15 @@ class GlobalDaemonThread < BaseThread
96
96
  $log.debug("GlobalDaemonThread: delete_deleted_items")
97
97
 
98
98
  r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
99
- ids = r.map{|row| row['id']}
100
- if ids.count > 0
99
+ item_ids = r.map{|row| row['id']}
100
+ limit = FC::Var.get('daemon_global_delete_limit', 1000).to_i
101
+ limit = 1000 if limit < 2
102
+ delay = FC::Var.get('daemon_global_delete_delay', 1).to_f
103
+ item_ids.each_slice(limit) do |ids|
101
104
  ids = ids.join(',')
102
105
  FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
103
106
  $log.info("GlobalDaemonThread: delete items #{ids}")
107
+ sleep delay if delay > 0
104
108
  end
105
109
  end
106
110
 
@@ -4,7 +4,7 @@ require 'psych'
4
4
  module FC
5
5
  module DB
6
6
  class << self
7
- attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting
7
+ attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting, :reconnect_block
8
8
  end
9
9
 
10
10
  def self.options_yml_path
@@ -20,7 +20,7 @@ module FC
20
20
  @prefix = @options[:prefix].to_s if @options[:prefix]
21
21
  connection = Mysql2::Client.new(@options)
22
22
  @reconnecting = false
23
- @connect_block = nil
23
+ @connect_block = nil unless @options[:keep_lazy_connection]
24
24
  @connects = {} unless @connects
25
25
  @connects[Thread.current.object_id] = connection
26
26
  end
@@ -48,13 +48,22 @@ module FC
48
48
  @connect_block = block
49
49
  end
50
50
 
51
+ def self.lazy_reconnect(&block)
52
+ @reconnect_block = block
53
+ end
54
+
51
55
  def self.connect_by_block(options = {})
52
56
  connection = @connect_block.call
53
57
  @options = connection.query_options.clone.merge(symbolize_keys(options))
54
58
  @prefix = @options[:prefix].to_s if @options[:prefix]
55
59
  @connects = {} unless @connects
56
60
  @connects[Thread.current.object_id] = connection
57
- @connect_block = nil
61
+ @connect_block = nil unless @options[:keep_lazy_connection]
62
+ end
63
+
64
+ def self.reconnect_by_block
65
+ @connects[Thread.current.object_id] = @reconnect_block.call
66
+ @reconnecting = false
58
67
  end
59
68
 
60
69
  def self.connect
@@ -86,7 +95,7 @@ module FC
86
95
  def self.reconnect
87
96
  close if connect
88
97
  @reconnecting = true
89
- connect_by_config(@options)
98
+ @reconnect_block ? reconnect_by_block : connect_by_config(@options)
90
99
  end
91
100
 
92
101
  def self.close
@@ -343,5 +352,10 @@ module FC
343
352
  def self.migrate_6
344
353
  FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
345
354
  end
355
+
356
+ def self.migrate_7
357
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_limit', val='1000', descr='limits number of deleted items per query'")
358
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_delay', val='1', descr='delay in seconds between items delete query'")
359
+ end
346
360
  end
347
361
  end
@@ -95,6 +95,7 @@ module FC
95
95
  end
96
96
 
97
97
  def update_http_check_time
98
+ return unless http_check_enabled?
98
99
  self.http_check_time = Time.new.to_i
99
100
  save
100
101
  end
@@ -104,9 +105,13 @@ module FC
104
105
  end
105
106
 
106
107
  def http_check_time_delay
107
- Time.new.to_i - http_check_time.to_i
108
+ http_check_enabled? ? Time.new.to_i - http_check_time.to_i : 0
108
109
  end
109
110
 
111
+ def http_check_enabled?
112
+ http_check_time.to_i >= 0
113
+ end
114
+
110
115
  def up?
111
116
  check_time_delay < self.class.check_time_limit
112
117
  end
@@ -137,7 +142,7 @@ module FC
137
142
  raise r if $?.exitstatus != 0
138
143
  else
139
144
  local_path += '/' if File.stat(local_path).directory?
140
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
145
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
141
146
  r = `#{cmd} 2>&1`
142
147
  raise r if $?.exitstatus != 0
143
148
  end
@@ -155,7 +160,7 @@ module FC
155
160
  r = `#{cmd} 2>&1`
156
161
  src_path += '/' if $?.exitstatus == 0
157
162
 
158
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
163
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
159
164
  r = `#{cmd} 2>&1`
160
165
  raise r if $?.exitstatus != 0
161
166
  end
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = '0.5.21'.freeze
2
+ VERSION = '0.5.26'.freeze
3
3
  end
@@ -20,21 +20,22 @@ def storages_show
20
20
  if storage = find_storage
21
21
  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
22
22
  puts %Q{Storage
23
- Name: #{storage.name}
24
- Host: #{storage.host}
25
- DC: #{storage.dc}
26
- Path: #{storage.path}
27
- Url: #{storage.url}
28
- Url weight: #{storage.url_weight}
29
- Write weight #{storage.write_weight}
30
- Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
31
- Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
32
- Size limit: #{size_to_human storage.size_limit}
33
- Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
34
- Copy storages: #{storage.copy_storages}
35
- Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
36
- Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
37
- Items storages: #{count}}
23
+ Name: #{storage.name}
24
+ Host: #{storage.host}
25
+ DC: #{storage.dc}
26
+ Path: #{storage.path}
27
+ Url: #{storage.url}
28
+ Url weight: #{storage.url_weight}
29
+ Write weight #{storage.write_weight}
30
+ Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
31
+ Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
32
+ Size limit: #{size_to_human storage.size_limit}
33
+ Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
34
+ Copy storages: #{storage.copy_storages}
35
+ Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
36
+ Check http time: #{storage.http_check_enabled? ? "#{Time.at(storage.http_check_time)} (#{storage.http_check_time_delay} seconds ago)" : 'disabled'}
37
+ Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
38
+ Items storages: #{count}}
38
39
  end
39
40
  end
40
41
 
@@ -56,13 +57,14 @@ def storages_add
56
57
  size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
57
58
  end
58
59
 
60
+ check_http = %(y yes).include?(stdin_read_val('Check http (y/n)?').downcase) ? 0 : -1
59
61
  copy_storages = stdin_read_val('Copy storages', true)
60
62
  storages = FC::Storage.where.map(&:name)
61
63
  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
62
64
  begin
63
65
  path = path +'/' unless path[-1] == '/'
64
66
  path = '/' + path unless path[0] == '/'
65
- storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
67
+ storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size, :http_check_time => check_http)
66
68
  print 'Calc current size.. '
67
69
  size = storage.file_size('', true)
68
70
  puts "ok"
@@ -88,6 +90,7 @@ def storages_add
88
90
  Free: #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
89
91
  Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
90
92
  Size limit: #{size_to_human size_limit}
93
+ Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
91
94
  Copy storages #{copy_storages}}
92
95
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
93
96
  puts ""
@@ -152,6 +155,7 @@ def storages_change
152
155
  auto_size = 0
153
156
  size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
154
157
  end
158
+ check_http = %(y yes).include?(stdin_read_val("Check http (now #{storage.http_check_enabled? ? 'yes' : 'no'})", true, storage.http_check_enabled? ? 'yes' : 'no').downcase)
155
159
  copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
156
160
 
157
161
  storage.dc = dc unless dc.empty?
@@ -172,7 +176,7 @@ def storages_change
172
176
  storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
173
177
  storages = FC::Storage.where.map(&:name)
174
178
  storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
175
-
179
+ storage.http_check_time = (check_http ? 0 : -1) if storage.http_check_enabled? != check_http
176
180
  puts %Q{\nStorage
177
181
  Name: #{storage.name}
178
182
  DC: #{storage.dc}
@@ -185,6 +189,7 @@ def storages_change
185
189
  Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
186
190
  Size type: #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
187
191
  Size limit: #{size_to_human storage.size_limit }
192
+ Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
188
193
  Copy storages: #{storage.copy_storages}}
189
194
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
190
195
  puts ""
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.21
4
+ version: 0.5.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - sh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-11 00:00:00.000000000 Z
11
+ date: 2020-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mysql2
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
217
217
  version: '0'
218
218
  requirements: []
219
219
  rubyforge_project:
220
- rubygems_version: 2.7.9
220
+ rubygems_version: 2.6.14
221
221
  signing_key:
222
222
  specification_version: 4
223
223
  summary: Distributed storage