filecluster 0.5.18 → 0.5.19
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/daemon/check_thread.rb +17 -1
- data/lib/fc/db.rb +3 -0
- data/lib/fc/item.rb +12 -9
- data/lib/fc/storage.rb +16 -3
- data/lib/fc/version.rb +1 -1
- data/test/item_test.rb +40 -7
- data/test/schema_test.rb +3 -3
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: cc6dcb8fbcf81a564a087d60e5af17e9e636e262b942452945980b4bb2be8475
|
4
|
+
data.tar.gz: d8b18f330e1701ba93c9c05fdee1f31cf769bf319d46a3376e60ab72b3e2aa07
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 566242919a5a563dcdeb80824875b10361b0a3cc4475a90b380f33157b4f7bb5dcdbd454143e8eacd01fcdb77ff720b8fbcdf4ba9e69b995433f7e8ed9da1b3a
|
7
|
+
data.tar.gz: 6d9e8bd6c13d062be2131f97e3b51c71f09ddf09a7e4a3852c16b7a0b14ae6977df6b99c32ccf1cf924f4bd8c9658824b6b4443cb8e68e21379d8cd110a1eace
|
data/lib/daemon/check_thread.rb
CHANGED
@@ -9,6 +9,22 @@ class CheckThread < BaseThread
|
|
9
9
|
else
|
10
10
|
error "Storage #{storage.name} with path #{storage.path} not writable"
|
11
11
|
end
|
12
|
+
check_http(storage)
|
12
13
|
$log.debug("CheckThread: Finish stotage check for #{storage_name}")
|
13
14
|
end
|
14
|
-
|
15
|
+
|
16
|
+
def check_http(storage)
|
17
|
+
url = "#{storage.url}healthcheck"
|
18
|
+
uri = URI.parse(url.sub(%r{^\/\/}, 'http://'))
|
19
|
+
request = Net::HTTP.new(uri.host, uri.port)
|
20
|
+
request.read_timeout = request.open_timeout = 2
|
21
|
+
resp = request.start { |http| http.get(uri.path) } rescue nil
|
22
|
+
if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
|
23
|
+
storage.update_http_check_time
|
24
|
+
else
|
25
|
+
error "Storage #{storage.name} with url #{storage.url} not readable"
|
26
|
+
end
|
27
|
+
rescue => err
|
28
|
+
$log.error("CheckThread: check_http error: #{err}")
|
29
|
+
end
|
30
|
+
end
|
data/lib/fc/db.rb
CHANGED
@@ -340,5 +340,8 @@ module FC
|
|
340
340
|
FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN autosync_at bigint(11) DEFAULT 0")
|
341
341
|
end
|
342
342
|
|
343
|
+
def self.migrate_6
|
344
|
+
FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
|
345
|
+
end
|
343
346
|
end
|
344
347
|
end
|
data/lib/fc/item.rb
CHANGED
@@ -167,21 +167,24 @@ module FC
|
|
167
167
|
def get_available_storages
|
168
168
|
r = FC::DB.query("SELECT st.* FROM #{FC::Storage.table_name} as st, #{FC::ItemStorage.table_name} as ist WHERE
|
169
169
|
ist.item_id = #{id} AND ist.status='ready' AND ist.storage_name = st.name")
|
170
|
-
r.map{|data| FC::Storage.create_from_fiels(data)
|
170
|
+
istorages = r.map { |data| FC::Storage.create_from_fiels(data) }
|
171
|
+
.select { |storage| storage.up? && storage.url_weight.to_i >= 0 }
|
172
|
+
http_up_storages = istorages.select(&:http_up?)
|
173
|
+
istorages = http_up_storages if http_up_storages.any?
|
174
|
+
istorages
|
171
175
|
end
|
172
|
-
|
176
|
+
|
173
177
|
def urls
|
174
|
-
get_available_storages.map{|storage| File.join(storage.url, name)}
|
178
|
+
get_available_storages.map { |storage| File.join(storage.url, name) }
|
175
179
|
end
|
176
|
-
|
180
|
+
|
177
181
|
def url
|
178
182
|
available_storages = get_available_storages()
|
179
183
|
# sort by random(url_weight)
|
180
|
-
best_storage = available_storages.map{ |storage|
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
}.map{|el| el[0]}.last
|
184
|
+
best_storage = available_storages.map { |storage| [storage, Kernel.rand(storage.url_weight.to_i * 100)] }
|
185
|
+
.sort_by { |a| a[1] }
|
186
|
+
.map { |el| el[0] }
|
187
|
+
.last
|
185
188
|
best_storage = available_storages.sample unless best_storage
|
186
189
|
raise "URL find - no avable storage for item #{id}" unless best_storage
|
187
190
|
File.join(best_storage.url, name)
|
data/lib/fc/storage.rb
CHANGED
@@ -4,7 +4,7 @@ require 'fileutils'
|
|
4
4
|
|
5
5
|
module FC
|
6
6
|
class Storage < DbBase
|
7
|
-
set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at'
|
7
|
+
set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at, http_check_time'
|
8
8
|
|
9
9
|
class << self
|
10
10
|
attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
|
@@ -93,15 +93,28 @@ module FC
|
|
93
93
|
self.check_time = Time.new.to_i
|
94
94
|
save
|
95
95
|
end
|
96
|
-
|
96
|
+
|
97
|
+
def update_http_check_time
|
98
|
+
self.http_check_time = Time.new.to_i
|
99
|
+
save
|
100
|
+
end
|
101
|
+
|
97
102
|
def check_time_delay
|
98
103
|
Time.new.to_i - check_time.to_i
|
99
104
|
end
|
105
|
+
|
106
|
+
def http_check_time_delay
|
107
|
+
Time.new.to_i - http_check_time.to_i
|
108
|
+
end
|
100
109
|
|
101
110
|
def up?
|
102
111
|
check_time_delay < self.class.check_time_limit
|
103
112
|
end
|
104
113
|
|
114
|
+
def http_up?
|
115
|
+
http_check_time_delay < self.class.check_time_limit
|
116
|
+
end
|
117
|
+
|
105
118
|
def self.speed_limit_to_rsync_opt(speed_limit)
|
106
119
|
return "--bwlimit=#{(speed_limit.to_f * 125.0).ceil} " if speed_limit.to_f > 0
|
107
120
|
''
|
@@ -197,7 +210,7 @@ module FC
|
|
197
210
|
end
|
198
211
|
|
199
212
|
def dump
|
200
|
-
super(%i[check_time autosync_at])
|
213
|
+
super(%i[check_time autosync_at http_check_time])
|
201
214
|
end
|
202
215
|
|
203
216
|
def load(data: {})
|
data/lib/fc/version.rb
CHANGED
data/test/item_test.rb
CHANGED
@@ -25,6 +25,14 @@ class ItemTest < Test::Unit::TestCase
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
+
setup do
|
29
|
+
@@storages.each do |s|
|
30
|
+
s.check_time = 0
|
31
|
+
s.http_check_time = 0
|
32
|
+
s.save
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
28
36
|
should "create_from_local" do policy = FC::Policy.new
|
29
37
|
assert_raise(ArgumentError) { FC::Item.create_from_local }
|
30
38
|
assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla' }
|
@@ -47,7 +55,8 @@ class ItemTest < Test::Unit::TestCase
|
|
47
55
|
@@item.save
|
48
56
|
end
|
49
57
|
|
50
|
-
should 'mark_deleted' do
|
58
|
+
should 'mark_deleted' do
|
59
|
+
@@item.mark_deleted
|
51
60
|
@@item.reload
|
52
61
|
assert_equal 'deferred_delete', @@item.status
|
53
62
|
@@item_storages.each do |item_storage|
|
@@ -56,27 +65,51 @@ class ItemTest < Test::Unit::TestCase
|
|
56
65
|
end
|
57
66
|
end
|
58
67
|
|
59
|
-
should "make_item_storage" do
|
68
|
+
should "make_item_storage" do
|
69
|
+
storage_size = @@storages[2].size.to_i
|
60
70
|
assert_kind_of FC::ItemStorage, @@item.make_item_storage(@@storages[2])
|
61
71
|
assert_equal storage_size+@@item.size, @@storages[2].size
|
62
72
|
end
|
63
73
|
|
64
|
-
should "get_item_storages" do
|
74
|
+
should "get_item_storages" do
|
75
|
+
assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
|
65
76
|
end
|
66
77
|
|
67
|
-
should
|
78
|
+
should 'item get_available_storages' do
|
68
79
|
@@storages[0].update_check_time
|
69
80
|
assert_equal 1, @@item.get_available_storages.count
|
70
81
|
assert_equal @@storages[0].name, @@item.get_available_storages.first.name
|
71
82
|
end
|
72
|
-
|
73
|
-
should
|
83
|
+
|
84
|
+
should 'item get_available_storages (http_up? influence)' do
|
85
|
+
# all storages have http_up? == false
|
86
|
+
storages = @@item_storages.map do |is|
|
87
|
+
@@storages.detect { |s| s.name == is.storage_name }
|
88
|
+
end.compact
|
89
|
+
storages.each do |s|
|
90
|
+
assert !s.http_up?
|
91
|
+
end
|
92
|
+
assert storages.size > 1
|
93
|
+
storages.each(&:update_check_time)
|
94
|
+
assert_equal storages.size, @@item.get_available_storages.count
|
95
|
+
|
96
|
+
# set one storage http_up? == true
|
97
|
+
storages[0].update_http_check_time
|
98
|
+
assert storages[0].http_up?
|
99
|
+
|
100
|
+
assert_equal 1, @@item.get_available_storages.count
|
101
|
+
assert_equal storages[0].name, @@item.get_available_storages.first.name
|
102
|
+
assert_equal 'http://rec1/sda/test item', @@item.url
|
103
|
+
end
|
104
|
+
|
105
|
+
should "item urls" do
|
74
106
|
assert_equal 0, @@item.urls.count
|
75
107
|
@@storages.each(&:update_check_time)
|
76
108
|
assert_same_elements ["http://rec1/sda/test item", "http://rec2/sda/test item"], @@item.urls
|
77
109
|
end
|
78
110
|
|
79
|
-
should "item url by url_weight" do
|
111
|
+
should "item url by url_weight" do
|
112
|
+
@@storages.each(&:update_check_time)
|
80
113
|
@@storages.each{|s| s.url_weight = -1; s.save}
|
81
114
|
assert_raise(RuntimeError) { @@item.url }
|
82
115
|
|
data/test/schema_test.rb
CHANGED
@@ -45,7 +45,7 @@ class SchemaTest < Test::Unit::TestCase
|
|
45
45
|
dump = create_dump
|
46
46
|
assert_equal 1, dump[:storages].size
|
47
47
|
FC::Storage.table_fields.each do |field|
|
48
|
-
next if %w[check_time autosync_at].include? field # not dumping attrs
|
48
|
+
next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
|
49
49
|
assert_equal storage.send(field), dump[:storages][0][field.to_sym], "field \"#{field}\" not dumped correctly"
|
50
50
|
end
|
51
51
|
assert_equal 1, dump[:copy_rules].size
|
@@ -68,7 +68,7 @@ class SchemaTest < Test::Unit::TestCase
|
|
68
68
|
policy = FC::Policy.where.first
|
69
69
|
rule = FC::CopyRule.where.first
|
70
70
|
FC::Storage.table_fields.each do |field|
|
71
|
-
next if %w[check_time autosync_at].include? field # not dumping attrs
|
71
|
+
next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
|
72
72
|
assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
|
73
73
|
end
|
74
74
|
FC::CopyRule.table_fields.each do |field|
|
@@ -141,7 +141,7 @@ class SchemaTest < Test::Unit::TestCase
|
|
141
141
|
rule.reload
|
142
142
|
|
143
143
|
FC::Storage.table_fields.each do |field|
|
144
|
-
next if %w[check_time autosync_at].include? field # not dumping attrs
|
144
|
+
next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
|
145
145
|
assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
|
146
146
|
end
|
147
147
|
FC::CopyRule.table_fields.each do |field|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
217
217
|
version: '0'
|
218
218
|
requirements: []
|
219
219
|
rubyforge_project:
|
220
|
-
rubygems_version: 2.
|
220
|
+
rubygems_version: 2.7.9
|
221
221
|
signing_key:
|
222
222
|
specification_version: 4
|
223
223
|
summary: Distributed storage
|