filecluster 0.5.18 → 0.5.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7eaf9a8058c73b2c9a7343ff987ded1edeb94317
-  data.tar.gz: 94232b51ba9c47cafaa55a69828e3e8666cde45f
+SHA256:
+  metadata.gz: cc6dcb8fbcf81a564a087d60e5af17e9e636e262b942452945980b4bb2be8475
+  data.tar.gz: d8b18f330e1701ba93c9c05fdee1f31cf769bf319d46a3376e60ab72b3e2aa07
 SHA512:
-  metadata.gz: a36265c29dd4707708395899d446c52c225da14dd2aa4cc689e95f1e11d68c3e69c4f2a14d1b253334b9257ac9292f205a312807c93ef20414d8d64998d9dfaa
-  data.tar.gz: f4c21b8c47115fb68662e65e0301cd11d365e38bc88dd1ebbde44133558d8b9dfe249c08b427d45a9c0cf92f864c721ba0de6a4563e4bf88c914b24966e1840f
+  metadata.gz: 566242919a5a563dcdeb80824875b10361b0a3cc4475a90b380f33157b4f7bb5dcdbd454143e8eacd01fcdb77ff720b8fbcdf4ba9e69b995433f7e8ed9da1b3a
+  data.tar.gz: 6d9e8bd6c13d062be2131f97e3b51c71f09ddf09a7e4a3852c16b7a0b14ae6977df6b99c32ccf1cf924f4bd8c9658824b6b4443cb8e68e21379d8cd110a1eace

data/lib/daemon/check_thread.rb

@@ -9,6 +9,22 @@ class CheckThread < BaseThread
     else 
       error "Storage #{storage.name} with path #{storage.path} not writable"
     end
+    check_http(storage)
     $log.debug("CheckThread: Finish stotage check for #{storage_name}")
   end
-end
+
+  def check_http(storage)
+    url = "#{storage.url}healthcheck"
+    uri = URI.parse(url.sub(%r{^\/\/}, 'http://'))
+    request = Net::HTTP.new(uri.host, uri.port)
+    request.read_timeout = request.open_timeout = 2
+    resp = request.start { |http| http.get(uri.path) } rescue nil
+    if resp && resp.code.to_i == 200 && resp.body.to_s.chomp == 'OK'
+      storage.update_http_check_time
+    else
+      error "Storage #{storage.name} with url #{storage.url} not readable"
+    end
+  rescue => err
+    $log.error("CheckThread: check_http error: #{err}")
+  end
+end

data/lib/fc/db.rb

@@ -340,5 +340,8 @@ module FC
       FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN autosync_at bigint(11) DEFAULT 0")
     end
 
+    def self.migrate_6
+      FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN http_check_time int(11) DEFAULT 0")
+    end
   end
 end

data/lib/fc/item.rb

@@ -167,21 +167,24 @@ module FC
     def get_available_storages
       r = FC::DB.query("SELECT st.* FROM #{FC::Storage.table_name} as st, #{FC::ItemStorage.table_name} as ist WHERE 
         ist.item_id = #{id} AND ist.status='ready' AND ist.storage_name = st.name")
-      r.map{|data| FC::Storage.create_from_fiels(data)}.select {|storage| storage.up? && storage.url_weight.to_i >= 0}
+      istorages = r.map { |data| FC::Storage.create_from_fiels(data) }
+                   .select { |storage| storage.up? && storage.url_weight.to_i >= 0 }
+      http_up_storages = istorages.select(&:http_up?)
+      istorages = http_up_storages if http_up_storages.any?
+      istorages
     end
-    
+
     def urls
-      get_available_storages.map{|storage| File.join(storage.url, name)}
+      get_available_storages.map { |storage| File.join(storage.url, name) }
     end
-    
+
     def url
       available_storages = get_available_storages()
       # sort by random(url_weight) 
-      best_storage = available_storages.map{ |storage| 
-        [storage, Kernel.rand(storage.url_weight.to_i * 100)] 
-      }.sort{ |a, b|
-        a[1] <=> b[1]
-      }.map{|el| el[0]}.last
+      best_storage = available_storages.map { |storage| [storage, Kernel.rand(storage.url_weight.to_i * 100)] }
+                                       .sort_by { |a| a[1] }
+                                       .map { |el| el[0] }
+                                       .last
       best_storage = available_storages.sample unless best_storage
       raise "URL find - no avable storage for item #{id}" unless best_storage
       File.join(best_storage.url, name)

data/lib/fc/storage.rb

@@ -4,7 +4,7 @@ require 'fileutils'
 
 module FC
   class Storage < DbBase
-    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at'
+    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at, http_check_time'
     
     class << self
       attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
@@ -93,15 +93,28 @@ module FC
       self.check_time = Time.new.to_i
       save
     end
-    
+
+    def update_http_check_time
+      self.http_check_time = Time.new.to_i
+      save
+    end
+
     def check_time_delay
       Time.new.to_i - check_time.to_i
     end
+
+    def http_check_time_delay
+      Time.new.to_i - http_check_time.to_i
+    end
     
     def up?
       check_time_delay < self.class.check_time_limit
     end
 
+    def http_up?
+      http_check_time_delay < self.class.check_time_limit
+    end
+
     def self.speed_limit_to_rsync_opt(speed_limit)
       return "--bwlimit=#{(speed_limit.to_f * 125.0).ceil} " if speed_limit.to_f > 0
       ''
@@ -197,7 +210,7 @@ module FC
     end
 
     def dump
-      super(%i[check_time autosync_at])
+      super(%i[check_time autosync_at http_check_time])
     end
 
     def load(data: {})

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.18'.freeze
+  VERSION = '0.5.19'.freeze
 end

data/test/item_test.rb

@@ -25,6 +25,14 @@ class ItemTest < Test::Unit::TestCase
     end
   end
   
+  setup do
+    @@storages.each do |s|
+      s.check_time = 0
+      s.http_check_time = 0
+      s.save
+    end
+  end
+
   should "create_from_local" do    policy = FC::Policy.new
     assert_raise(ArgumentError) { FC::Item.create_from_local }
     assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla' }
@@ -47,7 +55,8 @@ class ItemTest < Test::Unit::TestCase
     @@item.save
   end
 
-  should 'mark_deleted' do    @@item.mark_deleted
+  should 'mark_deleted' do
+    @@item.mark_deleted
     @@item.reload
     assert_equal 'deferred_delete', @@item.status
     @@item_storages.each do |item_storage| 
@@ -56,27 +65,51 @@ class ItemTest < Test::Unit::TestCase
     end
   end
   
-  should "make_item_storage" do    storage_size = @@storages[2].size.to_i
+  should "make_item_storage" do
+    storage_size = @@storages[2].size.to_i
     assert_kind_of FC::ItemStorage, @@item.make_item_storage(@@storages[2])
     assert_equal storage_size+@@item.size, @@storages[2].size
   end
   
-  should "get_item_storages" do    assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
+  should "get_item_storages" do
+    assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
   end
   
-  should "item get_available_storages" do    @@storages.each{|s| s.check_time = 0; s.save}
+  should 'item get_available_storages' do
     @@storages[0].update_check_time
     assert_equal 1, @@item.get_available_storages.count
     assert_equal @@storages[0].name, @@item.get_available_storages.first.name
   end
-  
-  should "item urls" do    @@storages.each{|s| s.check_time = 0; s.save}
+
+  should 'item get_available_storages (http_up? influence)' do
+    # all storages have http_up? == false
+    storages = @@item_storages.map do |is|
+      @@storages.detect { |s| s.name == is.storage_name }
+    end.compact
+    storages.each do |s|
+      assert !s.http_up?
+    end
+    assert storages.size > 1
+    storages.each(&:update_check_time)
+    assert_equal storages.size, @@item.get_available_storages.count
+
+    # set one storage http_up? == true
+    storages[0].update_http_check_time
+    assert storages[0].http_up?
+
+    assert_equal 1, @@item.get_available_storages.count
+    assert_equal storages[0].name, @@item.get_available_storages.first.name
+    assert_equal 'http://rec1/sda/test item', @@item.url
+  end
+
+  should "item urls" do
     assert_equal 0, @@item.urls.count
     @@storages.each(&:update_check_time)
     assert_same_elements ["http://rec1/sda/test item", "http://rec2/sda/test item"], @@item.urls
   end
   
-  should "item url by url_weight" do    @@storages.each(&:update_check_time)
+  should "item url by url_weight" do
+    @@storages.each(&:update_check_time)
     @@storages.each{|s| s.url_weight = -1; s.save}
     assert_raise(RuntimeError) { @@item.url }
     

data/test/schema_test.rb

@@ -45,7 +45,7 @@ class SchemaTest < Test::Unit::TestCase
     dump = create_dump
     assert_equal 1, dump[:storages].size
     FC::Storage.table_fields.each do |field|
-      next if %w[check_time autosync_at].include? field # not dumping attrs
+      next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
       assert_equal storage.send(field), dump[:storages][0][field.to_sym], "field \"#{field}\" not dumped correctly"
     end
     assert_equal 1, dump[:copy_rules].size
@@ -68,7 +68,7 @@ class SchemaTest < Test::Unit::TestCase
     policy = FC::Policy.where.first
     rule = FC::CopyRule.where.first
     FC::Storage.table_fields.each do |field|
-      next if %w[check_time autosync_at].include? field # not dumping attrs
+      next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
       assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
     end
     FC::CopyRule.table_fields.each do |field|
@@ -141,7 +141,7 @@ class SchemaTest < Test::Unit::TestCase
     rule.reload
 
     FC::Storage.table_fields.each do |field|
-      next if %w[check_time autosync_at].include? field # not dumping attrs
+      next if %w[check_time autosync_at http_check_time].include? field # not dumping attrs
       assert_equal storage.send(field), schema[:storages][0][field.to_sym], "field \"#{field}\" not loaded correctly"
     end
     FC::CopyRule.table_fields.each do |field|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.5.18
+  version: 0.5.19
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-11 00:00:00.000000000 Z
+date: 2019-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -217,7 +217,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: Distributed storage