filecluster 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a47f34292388667e2040dfa0cee212503869229
-  data.tar.gz: d69557e79d7ba8365d9aa48c58e678e0ddf6a23e
+  metadata.gz: 016165196d545cc809e342d8d7a573087f4f5d31
+  data.tar.gz: 90171541e2b6c1f81581bd81c9475bf556d9c859
 SHA512:
-  metadata.gz: 83ea94afcc5c96f8066f71310447744685839dbd5ef657928a5212519f0404b9ea4f1036a96ed922918b77cdfc2c6a5630e64a6c6b04c90bac6ae91941182f9b
-  data.tar.gz: 74aaf8eb6378e07202d6bc10a9d94c66d0df54e7290af322fac9836f0ac30e43a3ffd5b63cfb8831251ae5a0d3070fc00dfb78de4f4db9635e0293091c03197d
+  metadata.gz: 974f810d8c7bc2f37851786f3549021e0d100d887b5c2dae805fd2e7134358b337f579ac054cae6c5a305e57e1a39d6fce54ff3136291bf9f06790ff42e72dd6
+  data.tar.gz: 792e72c575b4a91655283a2a0dc11c760949bc896c31fb0f8cd5d87dabff76059b579b4c9085695d46180b0d5330f5eb30f89f7a58fca98d26648b82948e5587

data/lib/fc/db.rb

@@ -95,7 +95,6 @@ module FC
     
     # connect.query with deadlock solution
     def self.query(sql)
-      raise 'Too many mysql errors' if FC::DB.err_counter && FC::DB.err_counter > 10
       t1 = Time.new.to_f
       r = FC::DB.connect.query(sql)
       t2 = Time.new.to_f
@@ -104,13 +103,17 @@ module FC
       r = r.each(:as => :hash) {} if r
       r
     rescue Mysql2::Error => e
+      raise e if e.message =~ /You have an error in your SQL syntax/
       FC::DB.err_counter = FC::DB.err_counter.to_i + 1
-      if e.message.match('Deadlock found when trying to get lock')
+      if FC::DB.err_counter > 5
+        FC::DB.err_counter = 0
+        raise "Too many mysql errors, #{e.message}"
+      elsif e.message =~ /Deadlock found when trying to get lock/
         msg = "#{e.message} - retry"
         @logger ? @logger.error(msg) : puts(msg)
         sleep 0.1
         query(sql)
-      elsif e.message.match('Lost connection to MySQL server during query')
+      elsif e.message =~ /Lost connection to MySQL server during query/
         msg = "#{e.message} - reconnect"
         @logger ? @logger.error(msg) : puts(msg)
         FC::DB.connect.ping

data/lib/fc/item.rb

@@ -11,6 +11,7 @@ module FC
     #   :replace=true - replace item if it exists
     #   :remove_local=true - delete local_path file/dir after add
     #   :additional_fields - hash of additional FC:Item fields 
+    #   :no_md5 - don't use md5
     # If item_name is part of local_path it processed as inplace - local_path is valid path to the item for policy
     def self.create_from_local(local_path, item_name, policy, options={})
       raise 'Path not exists' unless File.exists?(local_path)
@@ -20,11 +21,15 @@ module FC
         :policy_id => policy.id,
         :dir => File.directory?(local_path),
         :size => FC::Storage.new(:host => FC::Storage.curr_host).file_size(local_path),
-        :md5 => FC::Storage.new(:host => FC::Storage.curr_host).md5_sum(local_path)
+        :md5 => nil
       })
+      item_params[:md5] = FC::Storage.new(
+        :host => FC::Storage.curr_host
+      ).md5_sum(local_path) unless item_params[:no_md5]
       item_params.delete(:replace)
       item_params.delete(:remove_local)
       item_params.delete(:not_local)
+      item_params.delete(:no_md5)
       raise 'Name is empty' if item_params[:name].empty?
       raise 'Zero size path' if item_params[:size] == 0
 
@@ -94,7 +99,7 @@ module FC
         else
           storage.copy_path(src, name, remove_local, speed_limit)
         end
-        md5_on_storage = storage.md5_sum(name)
+        md5_on_storage = storage.md5_sum(name) if md5
       rescue Exception => e
         item_storage.status = 'error'
         item_storage.save

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.1'.freeze
+  VERSION = '0.5.2'.freeze
 end

data/test/db_test.rb

@@ -106,6 +106,16 @@ class DbTest < Test::Unit::TestCase
     assert_equal FC::DB.instance_variable_get(:@connects).keys.count, 6
     FC::DB.connect!(:multi_threads => false)
   end
+
+  should 'mysql errors' do
+    FC::DB.connect!(:reconnect => true)
+    FC::DB.logger = mock
+    FC::DB.logger.expects(:info).at_least_once 
+    assert_raise(Mysql2::Error) { FC::DB.query('retertert') }
+    assert_raise(RuntimeError) { FC::DB.query('select ewrwerwerwer()') }
+    FC::DB.connect!(:reconnect => false)
+    FC::DB.logger = nil
+  end
   
   should "items" do
     assert @items.count > 0, 'Items not loaded'

data/test/functional_test.rb

@@ -74,18 +74,21 @@ class FunctionalTest < Test::Unit::TestCase
   end
   
   should "item create_from_local replace" do
-    @item  = FC::Item.new(:name => 'test2', :policy_id => @@policies[0].id)
+    @item = FC::Item.new(:name => 'test2', :policy_id => @@policies[3].id)
     @item.save
     errors_count = FC::Error.where.count
-    assert_raise(RuntimeError, "replace item") { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[0], {:tag => 'test'}) }
+    assert_raise(RuntimeError, "replace item") { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:tag => 'test'}) }
     assert_equal errors_count+1, FC::Error.where.count, "Error not saved after replace item"
-    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[0], {:replace => true, :tag => 'test'}) }
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:replace => true, :tag => 'test'}) }
     assert_equal @item.id, @item2.id, "Item (id1=#{@item.id}, id2=#{@item2.id}) change id after replace"
     item_storage = @item2.get_item_storages.first
-    item_storage.storage_name = 'host2-sda'
+    item_storage.storage_name = 'host1-sdb'
     item_storage.save
-    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[0], {:replace => true, :tag => 'test'}) }
-    assert_equal 'host2-sda', @item2.get_item_storages.first.storage_name
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[3], {:replace => true, :tag => 'test'}) }
+    item_storages = Hash[*@item2.get_item_storages.map { |el| [el.storage_name, el] }.flatten]
+    assert_same_elements item_storages.keys, ['host1-sdb', 'host2-sda']
+    assert_equal 'delete', item_storages['host1-sdb'].status
+    assert_equal 'ready', item_storages['host2-sda'].status
   end
   
   should "item create_from_local available storage" do
@@ -109,11 +112,24 @@ class FunctionalTest < Test::Unit::TestCase
   should "item create_from_local check md5" do
     errors_count = FC::Error.where.count
     @item = FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[0], {:tag => 'test'})
-    item_storage = @item.make_item_storage(@@storages[0], status = 'copy')
+    assert @item.md5
+    item_storage = @item.make_item_storage(@@storages[0], 'copy')
+    # rewrite item file
     `dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
+    # md5 check must fail on copy
     assert_raise(RuntimeError) { @item.copy_item_storage(@@storages[0], @@storages[1], item_storage) }
     assert_equal errors_count+1, FC::Error.where.count, "Error not saved after check md5"
   end
+
+  should 'item create_from_local disables md5 check' do
+    @item = FC::Item.create_from_local(@@test_file_path, 'test7', @@policies[0], {:tag => 'test', :no_md5 => true})
+    assert_nil @item.md5
+    item_storage = @item.make_item_storage(@@storages[0], 'copy')
+    # rewrite item file
+    `dd if=/dev/urandom of=#{@@storages[0].path}#{@item.name} bs=100K count=1 2>&1`
+    # no md5 check on copy - success
+    assert_nothing_raised { @item.copy_item_storage(@@storages[0], @@storages[1], item_storage) }
+  end
   
   should "item create_from_local inplace" do
     tmp_file_path = "/tmp/host2-sda/inplace test"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-05 00:00:00.000000000 Z
+date: 2016-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2