filecluster 0.4.7 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff920ef51fc5d2c823900026fedc150173f6f52e
-  data.tar.gz: a81702454f3bb7b5f754c62c7a3d4633122036c3
+  metadata.gz: 60a7cea9fe6d2870268e45b69048e1e6baefaf58
+  data.tar.gz: df929b92efd64aabdf1f15293864481985d7c556
 SHA512:
-  metadata.gz: d9c31c9663b17e861653282877d70eeab6c108327cdb4acacc07752569a773bb438464e6c97bbd1a28319dd2d4a5c77ba998294c05d75f53aa62d76d2526df8b
-  data.tar.gz: 15c10f14d399b173f5ffa748b623500a261632c856005143373e777215f109a9ef7430883218d4c46e87fbc8cbd1eb871500c979f03c88b29c152b2ad6f25ca2
+  metadata.gz: c0f1686a52969cbd70d7ebc2c0fd79ca00200b0929e9a0b7b5ac51d92fc59f4a0a82305c1801c87dc4182e4bd004cab5e0e2130a0bde7c7bb11df77f2984b35f
+  data.tar.gz: b22af5bff0622d403d12f3c6fe3767534e502c04d7ec5d32b9f55faf04db876316ad99783b7959d6c9f44132efef5b19bb6aa7fda7399fe834c5d0035dca1b7e

data/bin/fc-daemon

@@ -17,6 +17,7 @@ $tasks_copy_threads   = {}          # copy threads by storage name
 $tasks_delete_threads = {}          # delete threads by storage name
 $check_threads        = {}          # check threads by storage name
 $copy_count           = 0           # copy tasks count for current host
+$copy_cont_avg        = 0           # copy tasks avg count for current host
 $copy_speed           = 0           # copy tasks speed sum for current host
 $exit_signal          = false 
 $global_daemon_thread = nil

data/lib/daemon/copy_task_thread.rb

@@ -4,7 +4,7 @@ class CopyTaskThread < BaseThread
     Thread.current[:tasks_processed] = 0 unless Thread.current[:tasks_processed]
     while task = $tasks_copy[storage_name].shift do
       $curr_tasks << task
-      $log.debug("CopyTaskThread(#{storage_name}): run task for item_storage ##{task.id}, copy_count=#{$copy_count}, copy_speed=#{$copy_speed}")
+      $log.debug("CopyTaskThread(#{storage_name}): run task for item_storage ##{task.id}, copy_count=#{$copy_count}/#{$copy_cont_avg}, copy_speed=#{$copy_speed}")
       make_copy(task)
       $curr_tasks.delete(task)
       $log.debug("CopyTaskThread(#{storage_name}): finish task for item_storage ##{task.id}")
@@ -19,13 +19,18 @@ class CopyTaskThread < BaseThread
     speed_limit = nil
     if limit
       if $copy_count == 0
-        speed_limit = (limit - $copy_speed) / 0.75
+        $copy_speed = 0
+        speed_limit = limit * 0.75
+      elsif $copy_count <= $copy_cont_avg && (speed_limit = limit / $copy_cont_avg) < limit * 1.1 - $copy_speed
       else
-        sleep 0.1 while (speed_limit = (limit - $copy_speed) / ($copy_count + 0.4)) < limit*0.1
+        while (speed_limit = (limit - $copy_speed) * 0.75) < limit*0.1 do
+          sleep 0.1
+        end
       end
     end
     $copy_count += 1
     $copy_speed += speed_limit if speed_limit
+    calc_avg_count()
     
     storage = $storages.detect{|s| s.name == task.storage_name}
     begin
@@ -46,7 +51,7 @@ class CopyTaskThread < BaseThread
     end
     src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
     $log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
-    item.copy_item_storage(src_storage, storage, task, speed_limit)
+    item.copy_item_storage(src_storage, storage, task, false, speed_limit)
   rescue Exception => e
     error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => task.item_id, :item_storage_id => task.id
     $curr_tasks.delete(task)
@@ -54,4 +59,8 @@ class CopyTaskThread < BaseThread
     $copy_count -= 1 if $copy_count > 0
     $copy_speed -= speed_limit if speed_limit
   end
-end
+
+  def calc_avg_count
+    $copy_cont_avg = ($copy_count + $copy_cont_avg) / 2.0
+  end
+end

data/lib/fc/db.rb

@@ -85,7 +85,7 @@ module FC
     def self.init_db(silent = false)
       FC::DB.query(%{
         CREATE TABLE #{@prefix}items (
-          id int NOT NULL AUTO_INCREMENT,
+          id bigint NOT NULL AUTO_INCREMENT,
           name varchar(1024) NOT NULL DEFAULT '',
           tag varchar(255) DEFAULT NULL,
           outer_id int DEFAULT NULL,
@@ -155,8 +155,8 @@ module FC
       
       FC::DB.query(%{
         CREATE TABLE #{@prefix}items_storages (
-          id int NOT NULL AUTO_INCREMENT,
-          item_id int DEFAULT NULL,
+          id bigint NOT NULL AUTO_INCREMENT,
+          item_id bigint DEFAULT NULL,
           storage_name varchar(255) DEFAULT NULL,
           status ENUM('new', 'copy', 'error', 'ready', 'delete') NOT NULL DEFAULT 'new',
           time int DEFAULT NULL,
@@ -199,8 +199,8 @@ module FC
       FC::DB.query(%{
         CREATE TABLE #{@prefix}errors (
           id int NOT NULL AUTO_INCREMENT,
-          item_id int DEFAULT NULL,
-          item_storage_id int DEFAULT NULL,
+          item_id bigint DEFAULT NULL,
+          item_storage_id bigint DEFAULT NULL,
           host varchar(255) DEFAULT NULL,
           message text DEFAULT NULL,
           time int DEFAULT NULL,

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.4.7"
+  VERSION = "0.4.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.4.8
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-24 00:00:00.000000000 Z
+date: 2015-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2