filecluster 0.5.1 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fc/db.rb +6 -3
- data/lib/fc/item.rb +7 -2
- data/lib/fc/version.rb +1 -1
- data/test/db_test.rb +10 -0
- data/test/functional_test.rb +23 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 016165196d545cc809e342d8d7a573087f4f5d31
|
4
|
+
data.tar.gz: 90171541e2b6c1f81581bd81c9475bf556d9c859
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 974f810d8c7bc2f37851786f3549021e0d100d887b5c2dae805fd2e7134358b337f579ac054cae6c5a305e57e1a39d6fce54ff3136291bf9f06790ff42e72dd6
|
7
|
+
data.tar.gz: 792e72c575b4a91655283a2a0dc11c760949bc896c31fb0f8cd5d87dabff76059b579b4c9085695d46180b0d5330f5eb30f89f7a58fca98d26648b82948e5587
|
data/lib/fc/db.rb
CHANGED
@@ -95,7 +95,6 @@ module FC
|
|
95
95
|
|
96
96
|
# connect.query with deadlock solution
|
97
97
|
def self.query(sql)
|
98
|
-
raise 'Too many mysql errors' if FC::DB.err_counter && FC::DB.err_counter > 10
|
99
98
|
t1 = Time.new.to_f
|
100
99
|
r = FC::DB.connect.query(sql)
|
101
100
|
t2 = Time.new.to_f
|
@@ -104,13 +103,17 @@ module FC
|
|
104
103
|
r = r.each(:as => :hash) {} if r
|
105
104
|
r
|
106
105
|
rescue Mysql2::Error => e
|
106
|
+
raise e if e.message =~ /You have an error in your SQL syntax/
|
107
107
|
FC::DB.err_counter = FC::DB.err_counter.to_i + 1
|
108
|
-
if
|
108
|
+
if FC::DB.err_counter > 5
|
109
|
+
FC::DB.err_counter = 0
|
110
|
+
raise "Too many mysql errors, #{e.message}"
|
111
|
+
elsif e.message =~ /Deadlock found when trying to get lock/
|
109
112
|
msg = "#{e.message} - retry"
|
110
113
|
@logger ? @logger.error(msg) : puts(msg)
|
111
114
|
sleep 0.1
|
112
115
|
query(sql)
|
113
|
-
elsif e.message
|
116
|
+
elsif e.message =~ /Lost connection to MySQL server during query/
|
114
117
|
msg = "#{e.message} - reconnect"
|
115
118
|
@logger ? @logger.error(msg) : puts(msg)
|
116
119
|
FC::DB.connect.ping
|
data/lib/fc/item.rb
CHANGED
@@ -11,6 +11,7 @@ module FC
|
|
11
11
|
# :replace=true - replace item if it exists
|
12
12
|
# :remove_local=true - delete local_path file/dir after add
|
13
13
|
# :additional_fields - hash of additional FC:Item fields
|
14
|
+
# :no_md5 - don't use md5
|
14
15
|
# If item_name is part of local_path it processed as inplace - local_path is valid path to the item for policy
|
15
16
|
def self.create_from_local(local_path, item_name, policy, options={})
|
16
17
|
raise 'Path not exists' unless File.exists?(local_path)
|
@@ -20,11 +21,15 @@ module FC
|
|
20
21
|
:policy_id => policy.id,
|
21
22
|
:dir => File.directory?(local_path),
|
22
23
|
:size => FC::Storage.new(:host => FC::Storage.curr_host).file_size(local_path),
|
23
|
-
:md5 =>
|
24
|
+
:md5 => nil
|
24
25
|
})
|
26
|
+
item_params[:md5] = FC::Storage.new(
|
27
|
+
:host => FC::Storage.curr_host
|
28
|
+
).md5_sum(local_path) unless item_params[:no_md5]
|
25
29
|
item_params.delete(:replace)
|
26
30
|
item_params.delete(:remove_local)
|
27
31
|
item_params.delete(:not_local)
|
32
|
+
item_params.delete(:no_md5)
|
28
33
|
raise 'Name is empty' if item_params[:name].empty?
|
29
34
|
raise 'Zero size path' if item_params[:size] == 0
|
30
35
|
|
@@ -94,7 +99,7 @@ module FC
|
|
94
99
|
else
|
95
100
|
storage.copy_path(src, name, remove_local, speed_limit)
|
96
101
|
end
|
97
|
-
md5_on_storage = storage.md5_sum(name)
|
102
|
+
md5_on_storage = storage.md5_sum(name) if md5
|
98
103
|
rescue Exception => e
|
99
104
|
item_storage.status = 'error'
|
100
105
|
item_storage.save
|
data/lib/fc/version.rb
CHANGED
data/test/db_test.rb
CHANGED
@@ -106,6 +106,16 @@ class DbTest < Test::Unit::TestCase
|
|
106
106
|
assert_equal FC::DB.instance_variable_get(:@connects).keys.count, 6
|
107
107
|
FC::DB.connect!(:multi_threads => false)
|
108
108
|
end
|
109
|
+
|
110
|
+
should 'mysql errors' do
|
111
|
+
FC::DB.connect!(:reconnect => true)
|
112
|
+
FC::DB.logger = mock
|
113
|
+
FC::DB.logger.expects(:info).at_least_once
|
114
|
+
assert_raise(Mysql2::Error) { FC::DB.query('retertert') }
|
115
|
+
assert_raise(RuntimeError) { FC::DB.query('select ewrwerwerwer()') }
|
116
|
+
FC::DB.connect!(:reconnect => false)
|
117
|
+
FC::DB.logger = nil
|
118
|
+
end
|
109
119
|
|
110
120
|
should "items" do
|
111
121
|
assert @items.count > 0, 'Items not loaded'
|
data/test/functional_test.rb
CHANGED
@@ -74,18 +74,21 @@ class FunctionalTest < Test::Unit::TestCase
|
|
74
74
|
end
|
75
75
|
|
76
76
|
should "item create_from_local replace" do
|
77
|
-
@item
|
77
|
+
@item = FC::Item.new(:name => 'test2', :policy_id => @@policies[3].id)
|
78
78
|
@item.save
|
79
79
|
errors_count = FC::Error.where.count
|
80
|
-
assert_raise(RuntimeError, "replace item") { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[
|
80
|
+
assert_raise(RuntimeError, "replace item") { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:tag => 'test'}) }
|
81
81
|
assert_equal errors_count+1, FC::Error.where.count, "Error not saved after replace item"
|
82
|
-
assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[
|
82
|
+
assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:replace => true, :tag => 'test'}) }
|
83
83
|
assert_equal @item.id, @item2.id, "Item (id1=#{@item.id}, id2=#{@item2.id}) change id after replace"
|
84
84
|
item_storage = @item2.get_item_storages.first
|
85
|
-
item_storage.storage_name = '
|
85
|
+
item_storage.storage_name = 'host1-sdb'
|
86
86
|
item_storage.save
|
87
|
-
assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[
|
88
|
-
|
87
|
+
assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:replace => true, :tag => 'test'}) }
|
88
|
+
item_storages = Hash[*@item2.get_item_storages.map { |el| [el.storage_name, el] }.flatten]
|
89
|
+
assert_same_elements item_storages.keys, ['host1-sdb', 'host2-sda']
|
90
|
+
assert_equal 'delete', item_storages['host1-sdb'].status
|
91
|
+
assert_equal 'ready', item_storages['host2-sda'].status
|
89
92
|
end
|
90
93
|
|
91
94
|
should "item create_from_local available storage" do
|
@@ -109,11 +112,24 @@ class FunctionalTest < Test::Unit::TestCase
|
|
109
112
|
should "item create_from_local check md5" do
|
110
113
|
errors_count = FC::Error.where.count
|
111
114
|
@item = FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[0], {:tag => 'test'})
|
112
|
-
|
115
|
+
assert @item.md5
|
116
|
+
item_storage = @item.make_item_storage(@@storages[0], 'copy')
|
117
|
+
# rewrite item file
|
113
118
|
`dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
|
119
|
+
# md5 check must fail on copy
|
114
120
|
assert_raise(RuntimeError) { @item.copy_item_storage(@@storages[0], @@storages[1], item_storage) }
|
115
121
|
assert_equal errors_count+1, FC::Error.where.count, "Error not saved after check md5"
|
116
122
|
end
|
123
|
+
|
124
|
+
should 'item create_from_local disables md5 check' do
|
125
|
+
@item = FC::Item.create_from_local(@@test_file_path, 'test7', @@policies[0], {:tag => 'test', :no_md5 => true})
|
126
|
+
assert_nil @item.md5
|
127
|
+
item_storage = @item.make_item_storage(@@storages[0], 'copy')
|
128
|
+
# rewrite item file
|
129
|
+
`dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
|
130
|
+
# no md5 check on copy - success
|
131
|
+
assert_nothing_raised { @item.copy_item_storage(@@storages[0], @@storages[1], item_storage) }
|
132
|
+
end
|
117
133
|
|
118
134
|
should "item create_from_local inplace" do
|
119
135
|
tmp_file_path = "/tmp/host2-sda/inplace test"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filecluster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-04-
|
11
|
+
date: 2016-04-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mysql2
|