filecluster 0.5.19 → 0.5.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA256:
3
- metadata.gz: cc6dcb8fbcf81a564a087d60e5af17e9e636e262b942452945980b4bb2be8475
4
- data.tar.gz: d8b18f330e1701ba93c9c05fdee1f31cf769bf319d46a3376e60ab72b3e2aa07
2
+ SHA1:
3
+ metadata.gz: 8d483e4b277c06ca5555620bc5ae7f23f4e524d6
4
+ data.tar.gz: e7ee71236466346353c3ef599bd3bd29ac47551c
5
5
  SHA512:
6
- metadata.gz: 566242919a5a563dcdeb80824875b10361b0a3cc4475a90b380f33157b4f7bb5dcdbd454143e8eacd01fcdb77ff720b8fbcdf4ba9e69b995433f7e8ed9da1b3a
7
- data.tar.gz: 6d9e8bd6c13d062be2131f97e3b51c71f09ddf09a7e4a3852c16b7a0b14ae6977df6b99c32ccf1cf924f4bd8c9658824b6b4443cb8e68e21379d8cd110a1eace
6
+ metadata.gz: 4520da860c08b62f5815e39625c27c2e95d132dc1f3f135883ad6d83c2f4c8eff494544aa5874f96f0fbc8b578f7404f3f065613e8838bef858c518c70b440f4
7
+ data.tar.gz: 402c406c37d2c560df3da5a66f0a3c231c89b89edb542e19468fd2e151c3868ce338b223f3dcf364b14022ae34d28c0270b4b6d3c0610b81dd08935943c1b31c
data/README.md CHANGED
@@ -64,6 +64,8 @@ Can be used the following variables:
64
64
  |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
65
65
  |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
66
66
  |daemon_restart_period|86400|time between fc-daemon self restart|
67
+ |daemon_global_delete_limit|1000|limits number of deleted items per query|
68
+ |daemon_global_delete_dela|1|delay in seconds(float) between items delete query|
67
69
 
68
70
  ## Usage
69
71
 
@@ -1,4 +1,5 @@
1
1
  class CheckThread < BaseThread
2
+ require "net/http"
2
3
  def go(storage_name)
3
4
  $log.debug("CheckThread: Run stotage check for #{storage_name}")
4
5
  storage = $storages.detect{|s| s.name == storage_name}
@@ -96,11 +96,15 @@ class GlobalDaemonThread < BaseThread
96
96
  $log.debug("GlobalDaemonThread: delete_deleted_items")
97
97
 
98
98
  r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
99
- ids = r.map{|row| row['id']}
100
- if ids.count > 0
99
+ item_ids = r.map{|row| row['id']}
100
+ limit = FC::Var.get('daemon_global_delete_limit', 1000).to_i
101
+ limit = 1000 if limit < 2
102
+ delay = FC::Var.get('daemon_global_delete_delay', 1).to_f
103
+ item_ids.each_slice(limit) do |ids|
101
104
  ids = ids.join(',')
102
105
  FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
103
106
  $log.info("GlobalDaemonThread: delete items #{ids}")
107
+ sleep delay if delay > 0
104
108
  end
105
109
  end
106
110
 
@@ -4,7 +4,7 @@ require 'psych'
4
4
  module FC
5
5
  module DB
6
6
  class << self
7
- attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting
7
+ attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger, :reconnecting, :reconnect_block
8
8
  end
9
9
 
10
10
  def self.options_yml_path
@@ -20,7 +20,7 @@ module FC
20
20
  @prefix = @options[:prefix].to_s if @options[:prefix]
21
21
  connection = Mysql2::Client.new(@options)
22
22
  @reconnecting = false
23
- @connect_block = nil
23
+ @connect_block = nil unless @options[:keep_lazy_connection]
24
24
  @connects = {} unless @connects
25
25
  @connects[Thread.current.object_id] = connection
26
26
  end
@@ -48,13 +48,22 @@ module FC
48
48
  @connect_block = block
49
49
  end
50
50
 
51
+ def self.lazy_reconnect(&block)
52
+ @reconnect_block = block
53
+ end
54
+
51
55
  def self.connect_by_block(options = {})
52
56
  connection = @connect_block.call
53
57
  @options = connection.query_options.clone.merge(symbolize_keys(options))
54
58
  @prefix = @options[:prefix].to_s if @options[:prefix]
55
59
  @connects = {} unless @connects
56
60
  @connects[Thread.current.object_id] = connection
57
- @connect_block = nil
61
+ @connect_block = nil unless @options[:keep_lazy_connection]
62
+ end
63
+
64
+ def self.reconnect_by_block
65
+ @connects[Thread.current.object_id] = @reconnect_block.call
66
+ @reconnecting = false
58
67
  end
59
68
 
60
69
  def self.connect
@@ -86,7 +95,7 @@ module FC
86
95
  def self.reconnect
87
96
  close if connect
88
97
  @reconnecting = true
89
- connect_by_config(@options)
98
+ @reconnect_block ? reconnect_by_block : connect_by_config(@options)
90
99
  end
91
100
 
92
101
  def self.close
@@ -343,5 +352,10 @@ module FC
343
352
  def self.migrate_6
344
353
  FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
345
354
  end
355
+
356
+ def self.migrate_7
357
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_limit', val='1000', descr='limits number of deleted items per query'")
358
+ FC::DB.query("INSERT IGNORE INTO #{@prefix}vars SET name='daemon_global_delete_delay', val='1', descr='delay in seconds between items delete query'")
359
+ end
346
360
  end
347
361
  end
@@ -137,7 +137,7 @@ module FC
137
137
  raise r if $?.exitstatus != 0
138
138
  else
139
139
  local_path += '/' if File.stat(local_path).directory?
140
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
140
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
141
141
  r = `#{cmd} 2>&1`
142
142
  raise r if $?.exitstatus != 0
143
143
  end
@@ -155,7 +155,7 @@ module FC
155
155
  r = `#{cmd} 2>&1`
156
156
  src_path += '/' if $?.exitstatus == 0
157
157
 
158
- cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
158
+ cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
159
159
  r = `#{cmd} 2>&1`
160
160
  raise r if $?.exitstatus != 0
161
161
  end
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = '0.5.19'.freeze
2
+ VERSION = '0.5.25'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.19
4
+ version: 0.5.25
5
5
  platform: ruby
6
6
  authors:
7
7
  - sh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-03 00:00:00.000000000 Z
11
+ date: 2020-06-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mysql2
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
217
217
  version: '0'
218
218
  requirements: []
219
219
  rubyforge_project:
220
- rubygems_version: 2.7.9
220
+ rubygems_version: 2.6.14
221
221
  signing_key:
222
222
  specification_version: 4
223
223
  summary: Distributed storage