filecluster 0.5.18 → 0.5.19

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 7eaf9a8058c73b2c9a7343ff987ded1edeb94317
4
- data.tar.gz: 94232b51ba9c47cafaa55a69828e3e8666cde45f
2
+ SHA256:
3
+ metadata.gz: cc6dcb8fbcf81a564a087d60e5af17e9e636e262b942452945980b4bb2be8475
4
+ data.tar.gz: d8b18f330e1701ba93c9c05fdee1f31cf769bf319d46a3376e60ab72b3e2aa07
5
5
  SHA512:
6
- metadata.gz: a36265c29dd4707708395899d446c52c225da14dd2aa4cc689e95f1e11d68c3e69c4f2a14d1b253334b9257ac9292f205a312807c93ef20414d8d64998d9dfaa
7
- data.tar.gz: f4c21b8c47115fb68662e65e0301cd11d365e38bc88dd1ebbde44133558d8b9dfe249c08b427d45a9c0cf92f864c721ba0de6a4563e4bf88c914b24966e1840f
6
+ metadata.gz: 566242919a5a563dcdeb80824875b10361b0a3cc4475a90b380f33157b4f7bb5dcdbd454143e8eacd01fcdb77ff720b8fbcdf4ba9e69b995433f7e8ed9da1b3a
7
+ data.tar.gz: 6d9e8bd6c13d062be2131f97e3b51c71f09ddf09a7e4a3852c16b7a0b14ae6977df6b99c32ccf1cf924f4bd8c9658824b6b4443cb8e68e21379d8cd110a1eace
@@ -9,6 +9,22 @@ class CheckThread < BaseThread
9
9
  else
10
10
  error "Storage #{storage.name} with path #{storage.path} not writable"
11
11
  end
12
+ check_http(storage)
12
13
  $log.debug("CheckThread: Finish stotage check for #{storage_name}")
13
14
  end
14
- end
15
+
16
+ def check_http(storage)
17
+ url = "#{storage.url}healthcheck"
18
+ uri = URI.parse(url.sub(%r{^\/\/}, 'http://'))
19
+ request = Net::HTTP.new(uri.host, uri.port)
20
+ request.read_timeout = request.open_timeout = 2
21
+ resp = request.start { |http| http.get(uri.path) } rescue nil
22
+ if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
23
+ storage.update_http_check_time
24
+ else
25
+ error "Storage #{storage.name} with url #{storage.url} not readable"
26
+ end
27
+ rescue => err
28
+ $log.error("CheckThread: check_http error: #{err}")
29
+ end
30
+ end
data/lib/fc/db.rb CHANGED
@@ -340,5 +340,8 @@ module FC
340
340
  FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN autosync_at bigint(11) DEFAULT 0")
341
341
  end
342
342
 
343
+ def self.migrate_6
344
+ FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
345
+ end
343
346
  end
344
347
  end
data/lib/fc/item.rb CHANGED
@@ -167,21 +167,24 @@ module FC
167
167
  def get_available_storages
168
168
  r = FC::DB.query("SELECT st.* FROM #{FC::Storage.table_name} as st, #{FC::ItemStorage.table_name} as ist WHERE
169
169
  ist.item_id = #{id} AND ist.status='ready' AND ist.storage_name = st.name")
170
- r.map{|data| FC::Storage.create_from_fiels(data)}.select {|storage| storage.up? && storage.url_weight.to_i >= 0}
170
+ istorages = r.map { |data| FC::Storage.create_from_fiels(data) }
171
+ .select { |storage| storage.up? && storage.url_weight.to_i >= 0 }
172
+ http_up_storages = istorages.select(&:http_up?)
173
+ istorages = http_up_storages if http_up_storages.any?
174
+ istorages
171
175
  end
172
-
176
+
173
177
  def urls
174
- get_available_storages.map{|storage| File.join(storage.url, name)}
178
+ get_available_storages.map { |storage| File.join(storage.url, name) }
175
179
  end
176
-
180
+
177
181
  def url
178
182
  available_storages = get_available_storages()
179
183
  # sort by random(url_weight)
180
- best_storage = available_storages.map{ |storage|
181
- [storage, Kernel.rand(storage.url_weight.to_i * 100)]
182
- }.sort{ |a, b|
183
- a[1] <=> b[1]
184
- }.map{|el| el[0]}.last
184
+ best_storage = available_storages.map { |storage| [storage, Kernel.rand(storage.url_weight.to_i * 100)] }
185
+ .sort_by { |a| a[1] }
186
+ .map { |el| el[0] }
187
+ .last
185
188
  best_storage = available_storages.sample unless best_storage
186
189
  raise "URL find - no avable storage for item #{id}" unless best_storage
187
190
  File.join(best_storage.url, name)
data/lib/fc/storage.rb CHANGED
@@ -4,7 +4,7 @@ require 'fileutils'
4
4
 
5
5
  module FC
6
6
  class Storage < DbBase
7
- set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at'
7
+ set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at, http_check_time'
8
8
 
9
9
  class << self
10
10
  attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
@@ -93,15 +93,28 @@ module FC
93
93
  self.check_time = Time.new.to_i
94
94
  save
95
95
  end
96
-
96
+
97
+ def update_http_check_time
98
+ self.http_check_time = Time.new.to_i
99
+ save
100
+ end
101
+
97
102
  def check_time_delay
98
103
  Time.new.to_i - check_time.to_i
99
104
  end
105
+
106
+ def http_check_time_delay
107
+ Time.new.to_i - http_check_time.to_i
108
+ end
100
109
 
101
110
  def up?
102
111
  check_time_delay < self.class.check_time_limit
103
112
  end
104
113
 
114
+ def http_up?
115
+ http_check_time_delay < self.class.check_time_limit
116
+ end
117
+
105
118
  def self.speed_limit_to_rsync_opt(speed_limit)
106
119
  return "--bwlimit=#{(speed_limit.to_f * 125.0).ceil} " if speed_limit.to_f > 0
107
120
  ''
@@ -197,7 +210,7 @@ module FC
197
210
  end
198
211
 
199
212
  def dump
200
- super(%i[check_time autosync_at])
213
+ super(%i[check_time autosync_at http_check_time])
201
214
  end
202
215
 
203
216
  def load(data: {})
data/lib/fc/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module FC
2
- VERSION = '0.5.18'.freeze
2
+ VERSION = '0.5.19'.freeze
3
3
  end
data/test/item_test.rb CHANGED
@@ -25,6 +25,14 @@ class ItemTest < Test::Unit::TestCase
25
25
  end
26
26
  end
27
27
 
28
+ setup do
29
+ @@storages.each do |s|
30
+ s.check_time = 0
31
+ s.http_check_time = 0
32
+ s.save
33
+ end
34
+ end
35
+
28
36
  should "create_from_local" do policy = FC::Policy.new
29
37
  assert_raise(ArgumentError) { FC::Item.create_from_local }
30
38
  assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla' }
@@ -47,7 +55,8 @@ class ItemTest < Test::Unit::TestCase
47
55
  @@item.save
48
56
  end
49
57
 
50
- should 'mark_deleted' do @@item.mark_deleted
58
+ should 'mark_deleted' do
59
+ @@item.mark_deleted
51
60
  @@item.reload
52
61
  assert_equal 'deferred_delete', @@item.status
53
62
  @@item_storages.each do |item_storage|
@@ -56,27 +65,51 @@ class ItemTest < Test::Unit::TestCase
56
65
  end
57
66
  end
58
67
 
59
- should "make_item_storage" do storage_size = @@storages[2].size.to_i
68
+ should "make_item_storage" do
69
+ storage_size = @@storages[2].size.to_i
60
70
  assert_kind_of FC::ItemStorage, @@item.make_item_storage(@@storages[2])
61
71
  assert_equal storage_size+@@item.size, @@storages[2].size
62
72
  end
63
73
 
64
- should "get_item_storages" do assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
74
+ should "get_item_storages" do
75
+ assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
65
76
  end
66
77
 
67
- should "item get_available_storages" do @@storages.each{|s| s.check_time = 0; s.save}
78
+ should 'item get_available_storages' do
68
79
  @@storages[0].update_check_time
69
80
  assert_equal 1, @@item.get_available_storages.count
70
81
  assert_equal @@storages[0].name, @@item.get_available_storages.first.name
71
82
  end
72
-
73
- should "item urls" do @@storages.each{|s| s.check_time = 0; s.save}
83
+
84
+ should 'item get_available_storages (http_up? influence)' do
85
+ # all storages have http_up? == false
86
+ storages = @@item_storages.map do |is|
87
+ @@storages.detect { |s| s.name == is.storage_name }
88
+ end.compact
89
+ storages.each do |s|
90
+ assert !s.http_up?
91
+ end
92
+ assert storages.size > 1
93
+ storages.each(&:update_check_time)
94
+ assert_equal storages.size, @@item.get_available_storages.count
95
+
96
+ # set one storage http_up? == true
97
+ storages[0].update_http_check_time
98
+ assert storages[0].http_up?
99
+
100
+ assert_equal 1, @@item.get_available_storages.count
101
+ assert_equal storages[0].name, @@item.get_available_storages.first.name
102
+ assert_equal 'http://rec1/sda/test item', @@item.url
103
+ end
104
+
105
+ should "item urls" do
74
106
  assert_equal 0, @@item.urls.count
75
107
  @@storages.each(&:update_check_time)
76
108
  assert_same_elements ["http://rec1/sda/test item", "http://rec2/sda/test item"], @@item.urls
77
109
  end
78
110
 
79
- should "item url by url_weight" do @@storages.each(&:update_check_time)
111
+ should "item url by url_weight" do
112
+ @@storages.each(&:update_check_time)
80
113
  @@storages.each{|s| s.url_weight = -1; s.save}
81
114
  assert_raise(RuntimeError) { @@item.url }
82
115
 
data/test/schema_test.rb CHANGED
@@ -45,7 +45,7 @@ class SchemaTest < Test::Unit::TestCase
45
45
  dump = create_dump
46
46
  assert_equal 1, dump[:storages].size
47
47
  FC::Storage.table_fields.each do |field|
48
- next if %w[check_time autosync_at].include? field # not dumping attrs
48
+ next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
49
49
  assert_equal storage.send(field), dump[:storages][0][field.to_sym], "field \"#{field}\" not dumped correctly"
50
50
  end
51
51
  assert_equal 1, dump[:copy_rules].size
@@ -68,7 +68,7 @@ class SchemaTest < Test::Unit::TestCase
68
68
  policy = FC::Policy.where.first
69
69
  rule = FC::CopyRule.where.first
70
70
  FC::Storage.table_fields.each do |field|
71
- next if %w[check_time autosync_at].include? field # not dumping attrs
71
+ next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
72
72
  assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
73
73
  end
74
74
  FC::CopyRule.table_fields.each do |field|
@@ -141,7 +141,7 @@ class SchemaTest < Test::Unit::TestCase
141
141
  rule.reload
142
142
 
143
143
  FC::Storage.table_fields.each do |field|
144
- next if %w[check_time autosync_at].include? field # not dumping attrs
144
+ next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
145
145
  assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
146
146
  end
147
147
  FC::CopyRule.table_fields.each do |field|
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: filecluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.18
4
+ version: 0.5.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - sh
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-11 00:00:00.000000000 Z
11
+ date: 2019-12-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mysql2
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
217
217
  version: '0'
218
218
  requirements: []
219
219
  rubyforge_project:
220
- rubygems_version: 2.4.8
220
+ rubygems_version: 2.7.9
221
221
  signing_key:
222
222
  specification_version: 4
223
223
  summary: Distributed storage