filecluster 0.5.21 → 0.5.26

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA256:
3
- metadata.gz: 4bca4b52453d8184c6b3aa8f6915a4077c92fdefbc5a8c3690031d0dbfeeebdc
4
- data.tar.gz: 7387acb1f8c92b285311425b8ccb26da2ea05ad5fb35759d2e110fb6035da4a7
2
+ SHA1:
3
+ metadata.gz: 1ee01d84ce7df5b8ebe6279a92c0c14c488a5c82
4
+ data.tar.gz: 7e2f785e88144ed619e55cdec6bf038264278052
5
5
  SHA512:
6
- metadata.gz: 77502723c7312da226e1207527fa57654646077cffc5fed30ec2c83c99330e30851b4951e79dbab09374d95ff49904f1db7a442225a393b5d9dd58c39de8f4cc
7
- data.tar.gz: e814ddae4b981950bb78f585704d5b5c9fe7d2801cac3f807c9f42ff1e85a8709202ee6b8ace7cc3f38294a11688fca27fc66f00f52e00dc69581c9f9bfb9607
6
+ metadata.gz: 7735a60395e14dac92d0078d60c63458ac795847e55292a6b4616e29f072d50b7417f74a6cf2bb0e775c0452b2706363f10b6d82aaafe41cd5b8d371bf242c89
7
+ data.tar.gz: 1e670216069549f82614300d99c53422e73782ab96c5e3ed459c32bc4d8b27dcc77eaa47b813a68e6a3a5d3fda0cc60f498e48435d19c1fab30ece49f0eee530
data/README.md CHANGED
@@ -64,6 +64,8 @@ Can be used the following variables:
64
64
  |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
65
65
  |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
66
66
  |daemon_restart_period|86400|time between fc-daemon self restart|
67
+ |daemon_global_delete_limit|1000|limits number of deleted items per query|
68
+ |daemon_global_delete_dela|1|delay in seconds(float) between items delete query|
67
69
 
68
70
  ## Usage
69
71
 
@@ -1,5 +1,9 @@
1
1
  class CheckThread < BaseThread
2
2
  require "net/http"
3
+
4
+ @@storage_http_check = {}
5
+ HTTP_RETRIES = 3
6
+
3
7
  def go(storage_name)
4
8
  $log.debug("CheckThread: Run stotage check for #{storage_name}")
5
9
  storage = $storages.detect{|s| s.name == storage_name}
@@ -10,7 +14,7 @@ class CheckThread < BaseThread
10
14
  else
11
15
  error "Storage #{storage.name} with path #{storage.path} not writable"
12
16
  end
13
- check_http(storage)
17
+ check_http(storage) if storage.http_check_enabled?
14
18
  $log.debug("CheckThread: Finish stotage check for #{storage_name}")
15
19
  end
16
20
 
@@ -22,8 +26,10 @@ class CheckThread < BaseThread
22
26
  resp = request.start { |http| http.get(uri.path) } rescue nil
23
27
  if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
24
28
  storage.update_http_check_time
29
+ @@storage_http_check[storage.name] = 0
25
30
  else
26
- error "Storage #{storage.name} with url #{storage.url} not readable"
31
+ @@storage_http_check[storage.name] = @@storage_http_check[storage.name].to_i + 1
32
+ error("Storage #{storage.name} with url #{storage.url} not readable") if @@storage_http_check[storage.name] > HTTP_RETRIES
27
33
  end
28
34
  rescue => err
29
35
  $log.error("CheckThread: check_http error: #{err}")
@@ -96,11 +96,15 @@ class GlobalDaemonThread < BaseThread
96
96
  $log.debug("GlobalDaemonThread: delete_deleted_items")
97
97
 
98
98
  r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
99
- ids = r.map{|row| row['id']}
100
- if ids.count > 0
99
+ item_ids = r.map{|row| row['id']}
100
+ limit = FC::Var.get('daemon_global_delete_limit', 1000).to_i
101
+ limit = 1000 if limit < 2
102
+ delay = FC::Var.get('daemon_global_delete_delay', 1).to_f
103
+ item_ids.each_slice(limit) do |ids|
101
104
  ids = ids.join(',')
102
105
  FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
103
106
  $log.info("GlobalDaemonThread: delete items #{ids}")
107
+ sleep delay if delay > 0
104
108
  end
105
109
  end
106
110
 
@@ -4,7 +4,7 @@ require 'psych'
4
4
  module FC
5
5
  module DB
6
6
  class << self
7
- attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting
7
+ attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting, :reconnect_block
8
8
  end
9
9
 
10
10
  def self.options_yml_path
@@ -20,7 +20,7 @@ module FC
20
20
  @prefix = @options[:prefix].to_s if @options[:prefix]
21
21
  connection = Mysql2::Client.new(@options)
22
22
  @reconnecting = false
23
- @connect_block = nil
23
+ @connect_block = nil unless @options[:keep_lazy_connection]
24
24
  @connects = {} unless @connects
25
25
  @connects[Thread.current.object_id] = connection
26
26
  end
@@ -48,13 +48,22 @@ module FC
48
48
  @connect_block = block
49
49
  end
50
50
 
51
+ def self.lazy_reconnect(&block)
52
+ @reconnect_block = block
53
+ end
54
+
51
55
  def self.connect_by_block(options = {})
52
56
  connection = @connect_block.call
53
57
  @options = connection.query_options.clone.merge(symbolize_keys(options))
54
58
  @prefix = @options[:prefix].to_s if @options[:prefix]
55
59
  @connects = {} unless @connects
56
60
  @connects[Thread.current.object_id] = connection
57
- @connect_block = nil
61
+ @connect_block = nil unless @options[:keep_lazy_connection]
62
+ end
63
+
64
+ def self.reconnect_by_block
65
+ @connects[Thread.current.object_id] = @reconnect_block.call
66
+ @reconnecting = false
58
67
  end
59
68
 
60
69
  def self.connect
@@ -86,7 +95,7 @@ module FC
86
95
  def self.reconnect
87
96
  close if connect
88
97
  @reconnecting = true
89
- connect_by_config(@options)
98
+ @reconnect_block ? reconnect_by_block : connect_by_config(@options)
90
99
  end
91
100
 
92
101
  def self.close
@@ -343,5 +352,10 @@ module FC
343
352
  def self.migrate_6
344
353
  FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
345
354
  end
355
+
356
+ def self.migrate_7
357
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_limit', val='1000', descr='limits number of deleted items per query'")
358
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_delay', val='1', descr='delay in seconds between items delete query'")
359
+ end
346
360
  end
347
361
  end
@@ -95,6 +95,7 @@ module FC
95
95
  end
96
96
 
97
97
  def update_http_check_time
98
+ return unless http_check_enabled?
98
99
  self.http_check_time = Time.new.to_i
99
100
  save
100
101
  end
@@ -104,9 +105,13 @@ module FC
104
105
  end
105
106
 
106
107
  def http_check_time_delay
107
- Time.new.to_i - http_check_time.to_i
108
+ http_check_enabled? ? Time.new.to_i - http_check_time.to_i : 0
108
109
  end
109
110
 
111
+ def http_check_enabled?
112
+ http_check_time.to_i >= 0
113
+ end
114
+
110
115
  def up?
111
116
  check_time_delay < self.class.check_time_limit
112
117
  end
@@ -137,7 +142,7 @@ module FC
137
142
  raise r if $?.exitstatus != 0
138
143
  else
139
144
  local_path += '/' if File.stat(local_path).directory?
140
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
145
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
141
146
  r = `#{cmd} 2>&1`
142
147
  raise r if $?.exitstatus != 0
143
148
  end
@@ -155,7 +160,7 @@ module FC
155
160
  r = `#{cmd} 2>&1`
156
161
  src_path += '/' if $?.exitstatus == 0
157
162
 
158
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
163
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
159
164
  r = `#{cmd} 2>&1`
160
165
  raise r if $?.exitstatus != 0
161
166
  end
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = '0.5.21'.freeze
2
+ VERSION = '0.5.26'.freeze
3
3
  end
@@ -20,21 +20,22 @@ def storages_show
20
20
  if storage = find_storage
21
21
  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
22
22
  puts %Q{Storage
23
- Name: #{storage.name}
24
- Host: #{storage.host}
25
- DC: #{storage.dc}
26
- Path: #{storage.path}
27
- Url: #{storage.url}
28
- Url weight: #{storage.url_weight}
29
- Write weight #{storage.write_weight}
30
- Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
31
- Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
32
- Size limit: #{size_to_human storage.size_limit}
33
- Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
34
- Copy storages: #{storage.copy_storages}
35
- Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
36
- Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
37
- Items storages: #{count}}
23
+ Name: #{storage.name}
24
+ Host: #{storage.host}
25
+ DC: #{storage.dc}
26
+ Path: #{storage.path}
27
+ Url: #{storage.url}
28
+ Url weight: #{storage.url_weight}
29
+ Write weight #{storage.write_weight}
30
+ Size: #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
31
+ Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
32
+ Size limit: #{size_to_human storage.size_limit}
33
+ Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
34
+ Copy storages: #{storage.copy_storages}
35
+ Check time: #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
36
+ Check http time: #{storage.http_check_enabled? ? "#{Time.at(storage.http_check_time)} (#{storage.http_check_time_delay} seconds ago)" : 'disabled'}
37
+ Status: #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
38
+ Items storages: #{count}}
38
39
  end
39
40
  end
40
41
 
@@ -56,13 +57,14 @@ def storages_add
56
57
  size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
57
58
  end
58
59
 
60
+ check_http = %(y yes).include?(stdin_read_val('Check http (y/n)?').downcase) ? 0 : -1
59
61
  copy_storages = stdin_read_val('Copy storages', true)
60
62
  storages = FC::Storage.where.map(&:name)
61
63
  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
62
64
  begin
63
65
  path = path +'/' unless path[-1] == '/'
64
66
  path = '/' + path unless path[0] == '/'
65
- storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
67
+ storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size, :http_check_time => check_http)
66
68
  print 'Calc current size.. '
67
69
  size = storage.file_size('', true)
68
70
  puts "ok"
@@ -88,6 +90,7 @@ def storages_add
88
90
  Free: #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
89
91
  Size type: #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
90
92
  Size limit: #{size_to_human size_limit}
93
+ Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
91
94
  Copy storages #{copy_storages}}
92
95
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
93
96
  puts ""
@@ -152,6 +155,7 @@ def storages_change
152
155
  auto_size = 0
153
156
  size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
154
157
  end
158
+ check_http = %(y yes).include?(stdin_read_val("Check http (now #{storage.http_check_enabled? ? 'yes' : 'no'})", true, storage.http_check_enabled? ? 'yes' : 'no').downcase)
155
159
  copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
156
160
 
157
161
  storage.dc = dc unless dc.empty?
@@ -172,7 +176,7 @@ def storages_change
172
176
  storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
173
177
  storages = FC::Storage.where.map(&:name)
174
178
  storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
175
-
179
+ storage.http_check_time = (check_http ? 0 : -1) if storage.http_check_enabled? != check_http
176
180
  puts %Q{\nStorage
177
181
  Name: #{storage.name}
178
182
  DC: #{storage.dc}
@@ -185,6 +189,7 @@ def storages_change
185
189
  Free: #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
186
190
  Size type: #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
187
191
  Size limit: #{size_to_human storage.size_limit }
192
+ Check http: #{storage.http_check_enabled? ? 'yes' : 'no' }
188
193
  Copy storages: #{storage.copy_storages}}
189
194
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
190
195
  puts ""
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.21
4
+ version: 0.5.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - sh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-11 00:00:00.000000000 Z
11
+ date: 2020-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mysql2
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
217
217
  version: '0'
218
218
  requirements: []
219
219
  rubyforge_project:
220
- rubygems_version: 2.7.9
220
+ rubygems_version: 2.6.14
221
221
  signing_key:
222
222
  specification_version: 4
223
223
  summary: Distributed storage