filecluster 0.4.22 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4874f465a9a024770d5d40cb01e2749c1e45782
-  data.tar.gz: 53e2d183d39749cbaee8dbef14b93a2283b9a514
+  metadata.gz: 0a47f34292388667e2040dfa0cee212503869229
+  data.tar.gz: d69557e79d7ba8365d9aa48c58e678e0ddf6a23e
 SHA512:
-  metadata.gz: 36c3b885ff48bd21933587dae98c6faf865c3b194eaad4908ef624dbed71f359216eae1c2d01528c5b8a38ac490bac9d54296d9e827ff295fc355de2af86a790
-  data.tar.gz: 5619eb4c21e89d16cb0d518fd4f16000c745f30895288f82169e5d60dd40132ab3021f41e5249b494d33f138b237dcd1db6ed9590bd4959d5cfe8e0fd002cf07
+  metadata.gz: 83ea94afcc5c96f8066f71310447744685839dbd5ef657928a5212519f0404b9ea4f1036a96ed922918b77cdfc2c6a5630e64a6c6b04c90bac6ae91941182f9b
+  data.tar.gz: 74aaf8eb6378e07202d6bc10a9d94c66d0df54e7290af322fac9836f0ac30e43a3ffd5b63cfb8831251ae5a0d3070fc00dfb78de4f4db9635e0293091c03197d

data/bin/fc-daemon

@@ -25,7 +25,7 @@ $update_tasks_thread  = nil
 $run_tasks_thread     = nil
 
 args = ARGV.clone
-default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
+default_db_config = FC::DB.options_yml_path
 descriptions = {
   :config      => {:short => 'c',  :full => 'config',    :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
   :log_level   => {:short => 'l',  :full => 'log_level', :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},

data/bin/fc-manage

@@ -8,7 +8,7 @@ require 'filecluster'
 require 'utils'
 require 'manage'
 
-default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
+default_db_config = FC::DB.options_yml_path
 descriptions = {
   :config      => {:short => 'c',  :full => 'config',     :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
   :curr_host   => {:short => 'h', :full => 'host',       :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}

data/bin/fc-setup-db

@@ -27,7 +27,7 @@ options = option_parser_init(descriptions, desc)
 trap("INT", proc {exit})
 
 if !options[:__keys][:host] && !options[:__keys][:database] && !options[:__keys][:username] && !options[:__keys][:password] && !options[:__keys][:port] && !options[:__keys][:prefix]
-  default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
+  default_db_config = FC::DB.options_yml_path
   if File.exists?(default_db_config)
     db_options = Psych.load(File.read(default_db_config))
     options.merge!(db_options)

data/lib/fc/base.rb

@@ -16,7 +16,7 @@ module FC
     end
     
     def self.table_name
-      FC::DB.connect unless FC::DB.options
+      FC::DB.connect! unless FC::DB.options
       "#{FC::DB.prefix}#{@table_name}"
     end
         

data/lib/fc/db.rb

@@ -1,8 +1,15 @@
 require 'mysql2'
+require 'psych'
 
 module FC
   module DB
-    class << self; attr_accessor :options, :prefix, :err_counter end
+    class << self
+      attr_accessor :options, :prefix, :err_counter, :no_active_record, :connect_block, :logger 
+    end
+
+    def self.options_yml_path
+      File.expand_path(File.dirname(__FILE__) + '../../../bin/db.yml')
+    end
     
     def self.connect_by_config(options)
       @options = options.clone
@@ -11,45 +18,67 @@ module FC
       @connects = {} unless @connects
       @connects[Thread.current.object_id] = Mysql2::Client.new(@options)
     end
-    
-    def self.connect(options = {})
-      if !@options
-        if options[:host] || options[:database] || options[:username] || options[:password] || 
-           !defined?(ActiveRecord::Base) || !ActiveRecord::Base.connection
-          self.connect_by_config(options)
-        else
-          if defined?(Octopus::Proxy) && ActiveRecord::Base.connection.is_a?(Octopus::Proxy)
-            connection = ActiveRecord::Base.connection.select_connection.instance_variable_get(:@connection)
-          else
-            connection = ActiveRecord::Base.connection.instance_variable_get(:@connection)
-          end
-          @options = connection.query_options.clone
-          @options.merge!(options)
-          @prefix = @options[:prefix].to_s if @options[:prefix]
-          @connects = {} unless @connects
-          @connects[Thread.current.object_id] = connection
-        end
+
+    def self.connect_by_yml(options = {})
+      db_options = Psych.load(File.read(options_yml_path))
+      connect_by_config(db_options.merge(options))
+    end
+
+    def self.connect_by_active_record(options = {})
+      if defined?(Octopus::Proxy) && ActiveRecord::Base.connection.is_a?(Octopus::Proxy)
+        connection = ActiveRecord::Base.connection.select_connection.instance_variable_get(:@connection)
       else
-        @options.merge!(options)
+        connection = ActiveRecord::Base.connection.instance_variable_get(:@connection)
       end
-      if @options[:multi_threads]
-        @connects[Thread.current.object_id] ||= Mysql2::Client.new(@options)
+      @options = connection.query_options.clone
+      @options.merge!(options)
+      @prefix = @options[:prefix].to_s if @options[:prefix]
+      @connects = {} unless @connects
+      @connects[Thread.current.object_id] = connection
+    end
+
+    def self.lazy_connect(&block)
+      @connect_block = block
+    end
+
+    def self.connect_by_block(options = {})
+      connection = @connect_block.call
+      @options = connection.query_options.clone.merge(options)
+      @prefix = @options[:prefix].to_s if @options[:prefix]
+      @connects = {} unless @connects
+      @connects[Thread.current.object_id] = connection
+      @connect_block = nil
+    end
+
+    def self.connect
+      connect_by_block if @connect_block
+      return nil unless @options
+      connect_by_config(@options) if @options[:multi_threads] && !@connects[Thread.current.object_id]
+      if @options[:multi_threads] 
+        @connects[Thread.current.object_id]
       else
-        @connects.first[1]
+        @connects.first && @connects.first[1]
       end
     end
     
     def self.connect!(options = {})
-      self.connect(options)
-    end
-    
-    # deprecated!
-    def self.connect=(connect, options = {})
-      self.connect_by_config connect.query_options.merge(options).merge(:as => :hash)
+      close if @connects && @connects[Thread.current.object_id]
+      if @connect_block
+        connect_by_block(options)
+      elsif options[:host] || options[:database] || options[:username] || options[:password]
+        connect_by_config(options)
+      elsif @options
+        connect_by_config(@options.merge(options))
+      elsif !@no_active_record && defined?(ActiveRecord::Base) && ActiveRecord::Base.connection
+        connect_by_active_record(options)
+      else
+        connect_by_yml(options)
+      end
     end
-    class << self
-      extend Gem::Deprecate
-      deprecate :connect=, :connect!, 2016, 01
+
+    def self.reconnect
+      close if connect
+      connect_by_config(@options)
     end
     
     def self.close
@@ -60,35 +89,42 @@ module FC
         end
       else
         @connects.first[1].close
+        @connects.clear
       end
     end
     
     # connect.query with deadlock solution
     def self.query(sql)
       raise 'Too many mysql errors' if FC::DB.err_counter && FC::DB.err_counter > 10
+      t1 = Time.new.to_f
       r = FC::DB.connect.query(sql)
+      t2 = Time.new.to_f
+      @logger.debug(format('FC SQL (%.1fms) %s', (t2 - t1) * 1000, sql)) if @logger
       FC::DB.err_counter = 0
-      r = r.each(:as => :hash){} if r
+      r = r.each(:as => :hash) {} if r
       r
     rescue Mysql2::Error => e
       FC::DB.err_counter = FC::DB.err_counter.to_i + 1
       if e.message.match('Deadlock found when trying to get lock')
-        puts "#{e.message} - retry"
+        msg = "#{e.message} - retry"
+        @logger ? @logger.error(msg) : puts(msg)
         sleep 0.1
-        self.query(sql)
+        query(sql)
       elsif e.message.match('Lost connection to MySQL server during query')
-        puts "#{e.message} - reconnect"
+        msg = "#{e.message} - reconnect"
+        @logger ? @logger.error(msg) : puts(msg)
         FC::DB.connect.ping
         sleep 0.1
-        self.query(sql)
+        query(sql)
       elsif @options[:reconnect]
-        puts "#{e.message} - reconnect"
-        self.connect_by_config(@options)
-        self.query(sql)
+        msg = "#{e.message} - reconnect"
+        @logger ? @logger.info(msg) : puts(msg)
+        reconnect
+        query(sql)
       else
         raise e
       end
-    end    
+    end
     
     def self.server_time
       FC::DB.query("SELECT UNIX_TIMESTAMP() as curr_time").first['curr_time'].to_i

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.4.22"
+  VERSION = '0.5.1'.freeze
 end

data/lib/manage/storages.rb

@@ -109,7 +109,7 @@ def storages_update_size
     print "Calc current size.. "
     size = storage.file_size('', true)
     storage.size = size
-    FC::DB.connect
+    FC::DB.reconnect
     begin
       storage.save
     rescue Exception => e
@@ -244,7 +244,7 @@ def make_storages_sync(storage, make_delete, silent = false, no_reconnect = fals
   process_storage_dir_sync.call
     
   # rm delete_files
-  FC::DB.connect unless no_reconnect
+  FC::DB.reconnect unless no_reconnect
   if make_delete
     puts "Deleting files" unless silent
     delete_files.each do |f|

data/test/daemon_test.rb

@@ -77,6 +77,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   should "daemon_all" do
+    puts 'Start' if @debug
     @@storages.each {|storage| storage.reload}
     assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 
     assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
@@ -90,9 +91,10 @@ class DaemonTest < Test::Unit::TestCase
     
     @@policy.copies = 3
     @@policy.save
-    sleep 2
-    
+
     # wait for copy
+    sleep 2
+    puts 'Check copy' if @debug
     [1, 2, 3].each do |i|
       ['b', 'c'].each do |j|
         assert_equal `du -sb /tmp/host1-sda/bla/bla/test$i 2>&1`.to_i, `du -sb /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
@@ -105,7 +107,9 @@ class DaemonTest < Test::Unit::TestCase
     item_storage = FC::ItemStorage.where('item_id = ? AND storage_name = ?', @item1.id, 'host1-sdc').first
     item_storage.status = 'delete'
     item_storage.save
+    
     sleep 2
+    puts 'Check delete' if @debug
     assert_equal 0, `du -sb /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
     assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
     
@@ -118,6 +122,7 @@ class DaemonTest < Test::Unit::TestCase
     @item3.save
     
     sleep 6
+    puts 'Check mark_deleted' if @debug
     assert_raise(RuntimeError, "Item not deleted after mark_deleted") {@item1.reload}
     assert_equal 0, FC::ItemStorage.where('item_id = ?', @item2.id).count, "ItemStorages not deleted after status='error'"
     @item3.reload

data/test/db_test.rb

@@ -51,6 +51,61 @@ class DbTest < Test::Unit::TestCase
     @item_storage = @item_storages.first
     @item_storage2 = @item_storages[1]
   end
+
+  should 'sql logger' do
+    FC::DB.logger = mock
+    FC::DB.logger.expects(:debug).at_least_once 
+    FC::DB.query 'SELECT 1'
+    FC::DB.logger = nil
+  end
+
+  should 'close, reconnect and connect_by_yml' do
+    db_config_file = File.expand_path(File.dirname(__FILE__)) + '/db_test.yml'
+    File.open(db_config_file, 'w') do |f|
+      f.write(FC::DB.options.to_yaml)
+    end
+    
+    connect = FC::DB.connect
+    FC::DB.close
+    assert_false connect.ping, 'Mysql2 connect not closed after close call'
+    assert_nil FC::DB.connect, 'FC::DB.connect is not empty after close call'
+
+    FC::DB.stubs(:options_yml_path).returns(db_config_file)
+    FC::DB.connect_by_yml
+    assert_true FC::DB.connect.ping, 'Not connected after connect_by_yml'
+
+    FC::DB.close
+    FC::DB.reconnect
+    assert_true FC::DB.connect.ping, 'Not connected after reconnect'
+  end
+
+  should 'lazy_connect' do
+    FC::DB.close
+    FC::DB.lazy_connect do
+      Mysql2::Client.new(FC::DB.options)
+    end
+    FC::DB.connect!(:multi_threads => true)
+    assert_true FC::DB.connect.ping, 'Not connected after lazy_connect with Mysql2'
+    assert_true FC::DB.options[:multi_threads], 'Options from connect!(options) was not setted'
+    
+    FC::DB.close
+    FC::DB.lazy_connect do
+      FC::DB.connect_by_config(FC::DB.options)
+    end
+    assert_true FC::DB.connect.ping, 'Not connected after lazy_connect with FC::DB.connect_by_config'
+  end
+
+  should 'multi threads' do
+    FC::DB.connect!(:multi_threads => true)
+    threads = Array.new(5) do
+      Thread.new do
+        assert_nothing_raised { FC::DB.query('select sleep(0.05)') }
+      end
+    end
+    threads.each(&:join)
+    assert_equal FC::DB.instance_variable_get(:@connects).keys.count, 6
+    FC::DB.connect!(:multi_threads => false)
+  end
   
   should "items" do
     assert @items.count > 0, 'Items not loaded'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.4.22
+  version: 0.5.1
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-23 00:00:00.000000000 Z
+date: 2016-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2