filecluster 0.1.7 → 0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b456dec7949f50fd5082b13bd5aad2c6ecf852c6
+  data.tar.gz: 0c05bff392e9a9fddb9843104cef69335327fd1e
+SHA512:
+  metadata.gz: cdd07e299a989e265c94045198104600cf6a87234cea244ca8d32bc4774a2c5820583d21670d2a396da8792f678e573d772a14607e5d19b600e1f516ce3d180a
+  data.tar.gz: cf967fd389881cf282da7a93516710c5998ef4df4fa4790f915781d2134c57ebd2dcdfd41b2e5d52f884748be118ff998baca0960bca1b84e9dffb5617b67188

data/README.md

@@ -33,6 +33,19 @@ Selecting available storage to store item by policy.create_storages (from left t
 
 Selecting available storage to copy item by policy.copy_storages (from left to tight) with the nearest copy_id.
 
+## Variables
+
+|name|default value|description|
+|:---|:-----------:|:---------|
+|global_daemon_host||set fc_daemon when run global daemon task (only in one instance)|
+|daemon_cycle_time|30|time between global daemon checks and storages available checks|
+|daemon_global_wait_time|120|time between runs global daemon if it does not running|
+|daemon_global_tasks_group_limit|1000|limit for select for tasks|
+|daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
+|daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
+|daemon_global_tasks_per_thread|10|tasks count for one task thread|
+|daemon_global_tasks_threads_limit|3|tasks threads count limit for one storage|
+
 ## Usage
 
 TODO: Write usage instructions here

data/TODO

@@ -1,4 +1,4 @@
-! что делать с error is?
+пернести захардкоженные настройки 
 периодически удалять пустые папки
 при создании item-а по папке проверять что папка не пустая
 добавить в check storage проверку на дорступность урла

data/bin/fc-daemon

@@ -18,10 +18,8 @@ $global_daemon_thread = nil
 
 default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
 descriptions = {
-  :config      => {:short => 'c',  :full => 'config',     :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
-  :log_level   => {:short => 'l',  :full => 'log_level',  :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},
-  :cycle_time  => {:short => 't',  :full => 'time',       :default => 30, :text => 'Time between checks database and storages available, default 30'},
-  :global_wait => {:short => 'g', :full => 'wait',       :default => 120, :text => 'Time between runs global daemon if it does not running, default 120'},
+  :config      => {:short => 'c',  :full => 'config',    :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
+  :log_level   => {:short => 'l',  :full => 'log_level', :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},
   :curr_host   => {:short => 'h', :full => 'host',       :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
 }
 desc = %q{Run FileCluster daemon.
@@ -42,7 +40,7 @@ FC::DB.connect_by_config(db_options.merge(:reconnect => true, :multi_threads =>
 $log.info('Connected to database')
 
 def quit_on_quit
-  $log.info('Exit signal')
+  Thread.new { $log.info('Exit signal') }.join
   $exit_signal = true
 end
 trap("TERM") {quit_on_quit}
@@ -59,12 +57,12 @@ while true do
     $log.info('Exit')
     exit
   else
-    run_global_daemon options[:global_wait].to_i
+    run_global_daemon
     update_storages
     storages_check
     update_tasks
     run_tasks
   end
   $log.debug('sleep')
-  sleep options[:cycle_time].to_i
+  sleep FC::Var.get('daemon_cycle_time', 30).to_i
 end

data/bin/fc-manage

@@ -24,6 +24,8 @@ Command:
    rm <name>           delete storage
    change <name>       change storage attributes
    update_size <name>  update storage size
+   sync_info           show information about synchronization storage and file system
+   sync                synchronize storage and file system
 }],
   'policies' => [
     'show list and manage plicies', 
@@ -37,13 +39,21 @@ Command:
 }],
   'show' => [
     'show variable', 
-    %q{Usage: fc-manage [options] show <variable>
-Variable:
+    %q{Usage: fc-manage [options] show <command>
+Command:
    current_host        show current host
    global_daemon       show host and uptime where run global daemon
    errors [<count>]    show last count (default 10) errors
    host_info [<host>]  show info for host (default current host)
    items_info          show items statistics       
+}],
+  'var' => [
+    'show and change FC::Var', 
+    %q{Usage: fc-manage [options] var <command>
+Command:
+   list                show all FC::Var-s
+   show <variable>     show current value for variable   
+   change <variable>   change variable       
 }]
 }
 desc = %q{Get info and manage for storages, policies and items.
@@ -66,8 +76,7 @@ if ARGV.empty?
   exit
 end
 
-case command
-when 'help'
+if command == 'help'
   if !ARGV[1]
     puts $options['optparse']
   elsif commands_help[ARGV[1]]
@@ -75,31 +84,14 @@ when 'help'
   else
     puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
   end
-when 'show'
-  if !ARGV[1]
-    puts "Need variable name. See 'fc-manage help show'."
-  elsif self.private_methods.member?("show_#{ARGV[1]}".to_sym)
-    send "show_#{ARGV[1]}"
-  else
-    puts "Unknown variable. See 'fc-manage help show'."
-  end
-when 'storages'
+elsif commands_help[command]
   if !ARGV[1]
     puts "Need command. See 'fc-manage help storages'."
-  elsif self.private_methods.member?("storages_#{ARGV[1]}".to_sym)
-    send "storages_#{ARGV[1]}"
+  elsif self.private_methods.member?("#{command}_#{ARGV[1]}".to_sym)
+    send "#{command}_#{ARGV[1]}"
   else
-    puts "Unknown command. See 'fc-manage help storages'."
-  end
-when 'policies'
-  if !ARGV[1]
-    puts "Need command. See 'fc-manage help policies'."
-  elsif self.private_methods.member?("policies_#{ARGV[1]}".to_sym)
-    send "policies_#{ARGV[1]}"
-  else
-    puts "Unknown command. See 'fc-manage help policies'."
+    puts "Unknown command. See 'fc-manage help #{command}'."
   end
 else
   puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
-end
-
+end

data/lib/daemon.rb

@@ -15,12 +15,13 @@ class << FC::Error
   end
 end
 
-def run_global_daemon(timeout)
+def run_global_daemon
   $log.debug('Run global daemon check')
+  timeout = FC::Var.get('daemon_global_wait_time', 120).to_i
   r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   if !r || r['curr_time'].to_i - r['time'].to_i > timeout
     $log.debug('Set global daemon host to current')
-    FC::DB.query("REPLACE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}', name='global_daemon_host'")
+    FC::Var.set('global_daemon_host', FC::Storage.curr_host)
     sleep 1
     r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   end
@@ -66,14 +67,17 @@ def update_tasks
     cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
     ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
       flatten.map{|task| task[:item_storage].id}
-    ids += $curr_tasks.select{|task| task[:action] == type}
-      
+    ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
+    
+    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
     cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
-    cond << "LIMIT 1000"
+    cond << " LIMIT #{limit}"
     FC::ItemStorage.where(cond).each do |item_storage|
-      $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
-      $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
-      $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
+      unless ids.include?(item_storage.id)
+        $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
+        $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage} 
+        $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
+      end
     end
   end
   
@@ -88,9 +92,13 @@ def run_tasks
     $tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
     tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
     threads_count = $tasks_threads[storage.name].count
-    # 10 tasks per thread, maximum 3 tasks
-    run_threads_count = (tasks_count/10.0).ceil - threads_count
-    run_threads_count = 3 if run_threads_count > 3
+    
+    # <max_threads> tasks per thread, maximum <tasks_per_thread> threads
+    max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 3).to_i
+    tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
+    
+    run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil - threads_count
+    run_threads_count = max_threads if run_threads_count > max_threads
     $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
     run_threads_count.times do
       $log.debug("spawn TaskThread for #{storage.name}") 

data/lib/daemon/global_daemon_thread.rb

@@ -6,18 +6,18 @@ class GlobalDaemonThread < BaseThread
       sleep timeout.to_f/2
       exit if $exit_signal
       
-      r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
-      if r['val'] == FC::Storage.curr_host
-        FC::DB.query("UPDATE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}' WHERE name='global_daemon_host'")
+      if FC::Var.get('global_daemon_host') == FC::Storage.curr_host
+        FC::Var.set('global_daemon_host', FC::Storage.curr_host)
       else
-        $log.info("Exit from GlobalDaemonThread: global daemon already running on #{r['val']}")
+        $log.info("Exit from GlobalDaemonThread: global daemon already running on #{FC::Var.get('global_daemon_host')}")
         FC::DB.close
         exit
       end
       
       make_item_copies
+      make_deleted_error_items_storages
+      make_deleted_error_items
       delete_deleted_items
-      #периодическая проверка на item со статусом delete, последним обновлением дольше суток (NOW - time > 86400) и без связанных is - удаление таких из базы
       #TODO: периодически удалять (проставлять статус delete) для лиших is (число копий больше необходимого)
     end
   end
@@ -37,9 +37,10 @@ class GlobalDaemonThread < BaseThread
       end
     end
     
+    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
     sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages, p.id as policy_id, p.copies as policy_copies "+
       "FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p, #{FC::ItemStorage.table_name} as ist WHERE "+
-      "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT 1000"
+      "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
     r = FC::DB.query(sql)
     r.each do |row|
       $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
@@ -52,8 +53,11 @@ class GlobalDaemonThread < BaseThread
         src_storage = all_storages.detect{|s| item_storages.first == s.name}
         policy = all_policies.detect{|p| row['policy_id'] == p.id}
         storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy 
-        error 'No available storage', :item_id => row['item_id'] unless storage
-        FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
+        if storage
+          FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
+        else
+          error 'No available storage', :item_id => row['item_id']
+        end
       end
     end
   end
@@ -69,4 +73,16 @@ class GlobalDaemonThread < BaseThread
       $log.info("GlobalDaemonThread: delete items #{ids}")
     end
   end
+  
+  def make_deleted_error_items_storages
+    $log.debug("GlobalDaemonThread: make_deleted_error_items_storages")
+    ttl = FC::Var.get('daemon_global_error_items_storages_ttl', 86400).to_i
+    FC::DB.query("UPDATE #{FC::ItemStorage.table_name} SET status = 'delete' WHERE status = 'error' AND time < #{Time.new.to_i - ttl}")
+  end
+  
+  def make_deleted_error_items
+    $log.debug("GlobalDaemonThread: make_deleted_error_items")
+    ttl = FC::Var.get('daemon_global_error_items_ttl', 86400).to_i
+    FC::DB.query("UPDATE #{FC::Item.table_name} SET status = 'delete' WHERE status = 'error' AND time < #{Time.new.to_i - ttl}")
+  end
 end

data/lib/daemon/task_thread.rb

@@ -18,7 +18,6 @@ class TaskThread < BaseThread
   
   def make_delete(task)
     item_storage = task[:item_storage]
-    # TODO: не лазить в базу за item
     storage = $storages.detect{|s| s.name == item_storage.storage_name}
     item = FC::Item.find(item_storage.item_id)
     storage.delete_file(item.name)
@@ -30,9 +29,9 @@ class TaskThread < BaseThread
   
   def make_copy(task)
     item_storage = task[:item_storage]
-    # TODO: не лазить в базу за item, item_storages - перенести на стадию подготовки task-а
     storage = $storages.detect{|s| s.name == item_storage.storage_name}
     item = FC::Item.find(item_storage.item_id)
+    return nil unless item && item.status == 'ready'
     src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
     unless src_item_storage
       $log.info("Item ##{item.id} #{item.name} has no ready item_storage")

data/lib/fc/db.rb

@@ -48,7 +48,11 @@ module FC
       end
     end
     
-    def DB.init_db
+    def self.server_time
+      FC::DB.query("SELECT UNIX_TIMESTAMP() as curr_time").first['curr_time'].to_i
+    end
+    
+    def self.init_db
       FC::DB.connect.query(%{
         CREATE TABLE #{@prefix}items (
           id int NOT NULL AUTO_INCREMENT,
@@ -185,12 +189,20 @@ module FC
         CREATE TABLE #{@prefix}vars (
           name varchar(255) DEFAULT NULL,
           val varchar(255) DEFAULT NULL,
+          descr text DEFAULT NULL,
           time int DEFAULT NULL,
           PRIMARY KEY (name)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
       })
       FC::DB.connect.query("CREATE TRIGGER fc_vars_before_insert BEFORE INSERT on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
       FC::DB.connect.query("CREATE TRIGGER fc_vars_before_update BEFORE UPDATE on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_cycle_time', val='30', descr='time between global daemon checks and storages available checks'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_wait_time', val='120', descr='time between runs global daemon if it does not running'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_group_limit', val='1000', descr='limit for select for tasks'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='3', descr='tasks threads count limit for one storage'")
     end
   end
 end

data/lib/fc/item.rb

@@ -7,7 +7,7 @@ module FC
     # Create item by local path.
     # Additional options: 
     #   :replace=true - replace item if it exists
-    # If item_name is part of local_path it prcessed as inplace - local_path is valid path to the item for policy
+    # If item_name is part of local_path it processed as inplace - local_path is valid path to the item for policy
     def self.create_from_local(local_path, item_name, policy, options={})
       raise 'Path not exists' unless File.exists?(local_path)
       raise 'Policy is not FC::Policy' unless policy.instance_of?(FC::Policy)

data/lib/fc/var.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+module FC
+  class Var
+    class << self
+      attr_accessor :cache_time
+    end
+    @cache_time = 120 # ttl for get_all
+    @all_vars = {}
+    
+    def self.set(name, val)
+      FC::DB.query("UPDATE #{FC::DB.prefix}vars SET val='#{Mysql2::Client.escape(val.to_s)}' WHERE name='#{Mysql2::Client.escape(name.to_s)}'")
+      FC::DB.query("INSERT IGNORE INTO #{FC::DB.prefix}vars SET val='#{Mysql2::Client.escape(val.to_s)}', name='#{Mysql2::Client.escape(name.to_s)}'")
+      @all_vars[name.to_s] = val.to_s
+      @all_vars[name.to_sym] = val.to_s
+    end
+    
+    def self.get(name, default_value = nil)
+      get_all[name] || default_value
+    end
+    
+    def self.get_all
+      if !@get_all_read_time || Time.new.to_i - @get_all_read_time > cache_time
+        @all_vars = {}
+        FC::DB.query("SELECT * FROM #{FC::DB.prefix}vars").each do |row|
+          @all_vars[row['name']] = row['val']
+          @all_vars[row['name'].to_sym] = row['val']
+        end
+        @get_all_read_time = Time.new.to_i
+      end
+      @all_vars 
+    end
+  end
+end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.1.7"
+  VERSION = "0.2"
 end

data/lib/filecluster.rb

@@ -7,6 +7,7 @@ require "fc/item"
 require "fc/policy"
 require "fc/storage"
 require "fc/error"
+require "fc/var"
 
 module FC
 

data/lib/manage.rb

@@ -2,3 +2,4 @@ require 'rb-readline'
 require "manage/policies"
 require "manage/storages"
 require "manage/show"
+require "manage/var"

data/lib/manage/policies.rb

@@ -85,8 +85,8 @@ def policies_change
     copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
         
     policy.name = name unless name.empty?
-    policy.create_storages = name unless create_storages.empty?
-    policy.copy_storages = name unless copy_storages.empty?
+    policy.create_storages = create_storages unless create_storages.empty?
+    policy.copy_storages = copy_storages unless copy_storages.empty?
     policy.copies = copies.to_i unless copies.empty?
     
     puts %Q{\nStorage

data/lib/manage/show.rb

@@ -4,7 +4,7 @@ end
 
 def show_global_daemon
   r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
-  if r 
+  if r['val']
     puts "Global daemon run on #{r['val']}\nLast run #{r['curr_time']-r['time']} seconds ago."
   else
     puts "Global daemon is not runnning."

data/lib/manage/storages.rb

@@ -146,6 +146,28 @@ def storages_change
   end
 end
 
+def sync_info
+  if storage = find_storage
+    print "Synchronization info for (#{storage.name}) storage and file system (#{storage.path})."
+    make_storages_sync(storage, false)
+  end
+end
+
+def storages_sync
+  if storage = find_storage
+    print "Synchronize (#{storage.name}) storage and file system (#{storage.path})."
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      make_storages_sync(storage, true)
+      puts "Synchronize done."
+      storages_update_size
+    else
+      puts "Canceled."
+    end
+  end
+end
+
 private
 
 def find_storage
@@ -154,3 +176,58 @@ def find_storage
   puts "Storage #{name} not found." if !storage
   storage
 end
+
+def make_storages_sync(storage, make_delete, silent = false)
+  # get all items for storage
+  db_items = {}
+  FC::DB.query("SELECT i.name, ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}'").each do |row|
+    name = row['name'].sub(/\/$/, '').sub(/^\//, '').strip
+    path = ''
+    name.split('/').each do |dir|
+      path << dir
+      db_items[path] = [false, path == name ? row['id'].to_i : nil]
+      path << '/'
+    end
+  end
+  
+  # walk on all storage folders and files
+  delete_files = []
+  process_storage_dir_sync = lambda do |dir = ''|
+    Dir.glob(storage.path+dir+'*').each do |f|
+      path = f.sub(storage.path, '')
+      if db_items[path]
+        db_items[path][0] = true
+        next if db_items[path][1]
+      end
+      delete_files << path if File.file?(f)
+      process_storage_dir_sync.call(path+'/') if File.directory?(f)
+    end
+  end  
+  process_storage_dir_sync.call
+    
+  # rm delete_files
+  if make_delete
+    delete_files.each do |f|
+      # check in DB again
+      next if FC::DB.query("SELECT ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}' AND i.name='#{f}'").first      
+      path = storage.path+f
+      File.delete(path) rescue nil
+    end
+  end
+  puts "Deleted #{delete_files.count} files" unless silent
+  
+  # delete non synchronize items_storages
+  count = 0  
+  db_items.values.each do |item|
+    if !item[0] && item[1] 
+      count += 1
+      FC::DB.query("DELETE FROM #{FC::ItemStorage.table_name} WHERE id=#{item[1]}") if make_delete
+    end
+  end
+  puts "Deleted #{count} items_storages" unless silent
+  
+  # delete empty folders
+  count = `find #{storage.path} -empty -type d`.split("\n").count
+  `find #{storage.path} -empty -type d -delete` if make_delete
+  puts "Deleted #{count} empty folders" unless silent
+end

data/lib/manage/var.rb

@@ -0,0 +1,51 @@
+def var_list
+  vars = FC::DB.query("SELECT * FROM #{FC::DB.prefix}vars WHERE descr IS NOT NULL")
+  if vars.size == 0
+    puts "No vars."
+  else
+    vars.each do |var|
+      puts var['name']
+    end
+  end
+end
+
+def var_show
+  if var = find_var
+    puts %Q{Var
+  Name:         #{var['name']}
+  Value:        #{var['val']}
+  Description:  #{var['descr']}}
+  end
+end
+
+def var_change
+  if var = find_var
+    puts "Change var #{var['name']}"
+    val = stdin_read_val("Value (now #{var['val']})")
+        
+    puts %Q{\nVar
+    Name:         #{var['name']}
+    Value:        #{val}}
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      begin
+        FC::Var.set(var['name'], val)
+      rescue Exception => e
+        puts "Error: #{e.message}"
+        exit
+      end
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+private
+
+def find_var
+  var = FC::DB.query("SELECT * FROM #{FC::DB.prefix}vars WHERE descr IS NOT NULL AND name='#{Mysql2::Client.escape(ARGV[2].to_s)}'").first
+  puts "Var #{ARGV[2]} not found." unless var
+  var
+end

data/test/daemon_test.rb

@@ -17,9 +17,13 @@ class DaemonTest < Test::Unit::TestCase
       
       @@errors_count = FC::Error.where.count
       
+      FC::Var.set('daemon_cycle_time', 1)
+      FC::Var.set('daemon_global_wait_time', 1)
+      FC::Var.set('daemon_global_error_items_storages_ttl', 0)
+      FC::Var.set('daemon_global_error_items_ttl', 0)
       @stotage_checks = 0
       Thread.new do
-        Open3.popen2e("#{daemon_bin} -c #{db_config_file} -l debug -t 1 -g 1 -h host1") do |stdin, stdout, t|
+        Open3.popen2e("#{daemon_bin} -c #{db_config_file} -l debug -h host1") do |stdin, stdout, t|
           @@pid = t.pid
           while line = stdout.readline
             @stotage_checks += 1 if line =~ /Finish stotage check/i
@@ -90,7 +94,8 @@ class DaemonTest < Test::Unit::TestCase
         assert_equal `du -sb /tmp/host1-sda/bla/bla/test$i 2>&1`.to_i, `du -sb /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
       end
     end
-        
+    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
+    
     @@policy.copies = 2
     @@policy.save
     item_storage = FC::ItemStorage.where('item_id = ? AND storage_name = ?', @item1.id, 'host1-sdc').first
@@ -101,7 +106,18 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
     
     @item1.mark_deleted
+    FC::ItemStorage.where('item_id = ?', @item2.id).each do |item_storage|
+      item_storage.status = 'error'
+      item_storage.save
+    end
+    @item3.status = 'error'
+    @item3.save
+    
     sleep 2
-    assert_raise(RuntimeError, 'Item not deleted after mark_deleted') {@item1.reload}
+    assert_raise(RuntimeError, "Item not deleted after mark_deleted") {@item1.reload}
+    assert_equal 0, FC::ItemStorage.where('item_id = ?', @item2.id).count, "ItemStorages not deleted after status='error'"
+    @item3.reload
+    assert_equal 'delete', @item3.status, "ItemStorages not deleted after status='error'"    
+    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
   end
 end

data/test/storage_sync.rb

@@ -0,0 +1,65 @@
+require 'helper'
+require 'manage'
+
+class DaemonTest < Test::Unit::TestCase
+  class << self
+    def startup
+      # tmp fake storage dir
+      `rm -rf /tmp/host*-sd*`
+      `mkdir -p /tmp/host1-sda/`
+      
+      # test files to copy
+      @@test_file_path = '/tmp/fc_test_file'
+      `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
+      
+      @@storage = FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000, :check_time => Time.new.to_i)
+      @@storage.save
+      @@policy = FC::Policy.new(:create_storages => 'host1-sda', :copy_storages => 'host1-sda', :copies => 1, :name => 'policy 1')
+      @@policy.save
+    end
+    
+    def shutdown
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
+      `rm -rf /tmp/host*-sd*`
+      `rm -rf #{@@test_file_path}`
+    end
+  end
+  
+  def setup
+    FC::Storage.any_instance.stubs(:host).returns('host1')
+    FC::Storage.stubs(:curr_host).returns('host1')
+  end
+  
+  should "sync_all" do
+    assert_nothing_raised { @item1 = FC::Item.create_from_local(@@test_file_path, 'a/test1', @@policy, {:tag => 'test'}) }
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'a/b/test2', @@policy, {:tag => 'test'}) }
+    assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/test3', @@policy, {:tag => 'test'}) }
+    assert_nothing_raised { @item4 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/d/test4', @@policy, {:tag => 'test'}) }
+    `mv /tmp/host1-sda/a/test1 /tmp/host1-sda/test1`
+    `mv /tmp/host1-sda/a/b/c/d/test4 /tmp/host1-sda/a/b/c/d/test5`
+    `mkdir /tmp/host1-sda/test_dir`
+    `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t1`
+    `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t2`
+    
+    make_storages_sync(@@storage, true, true)
+    
+    @item1.reload
+    @item2.reload
+    @item3.reload
+    @item4.reload
+    assert_equal 'error', @item1.status
+    assert_equal 'ready', @item2.status
+    assert_equal 'ready', @item3.status
+    assert_equal 'error', @item4.status
+    size = `du -sb #{@@test_file_path} 2>&1`.to_i
+    assert_equal 0, `du -sb /tmp/host1-sda/test1 2>&1`.to_i
+    assert_equal size, `du -sb /tmp/host1-sda/a/b/test2 2>&1`.to_i
+    assert_equal size, `du -sb /tmp/host1-sda/a/b/c/test3 2>&1`.to_i
+    assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d/test5 2>&1`.to_i
+    assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d 2>&1`.to_i
+    assert_equal 0, `du -sb tmp/host1-sda/test_dir 2>&1`.to_i
+  end
+end

data/test/var_test.rb

@@ -0,0 +1,33 @@
+require 'helper'
+
+class VarTest < Test::Unit::TestCase
+  should "set and get" do
+    assert_nothing_raised {FC::Var.set('test1', 111)}
+    assert_nothing_raised {FC::Var.set(:test2, '222')}
+    assert_equal '111', FC::Var.get(:test1)
+    assert_equal '222', FC::Var.get('test2')
+  end
+  should "change" do
+    assert_nothing_raised {FC::Var.set('test3', '333')}
+    assert_equal '333', FC::Var.get('test3')
+    FC::Var.set('test3', '3332')
+    assert_equal '3332', FC::Var.get('test3')
+  end
+  should "get all" do
+    assert_nothing_raised {FC::Var.set('test4', '444')}
+    assert_nothing_raised {FC::Var.set('test5', '555')}
+    assert_nothing_raised {FC::Var.set('test6', '666')}
+    vars = FC::Var.get_all
+    assert_equal '444', vars['test4']
+    assert_equal '555', vars[:test5]
+    assert_equal '666', vars[:test6]
+  end
+  should "change and get all" do
+    assert_nothing_raised {FC::Var.set('test7', '777')}
+    vars = FC::Var.get_all
+    assert_equal '777', vars['test7']
+    FC::Var.set('test7', '7772')
+    vars = FC::Var.get_all
+    assert_equal '7772', vars['test7']
+  end
+end

metadata

@@ -1,126 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.1.7
-  prerelease: 
+  version: '0.2'
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-22 00:00:00.000000000 Z
+date: 2013-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: shoulda-context
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.13.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.13.3
 description: Distributed storage
@@ -155,12 +140,14 @@ files:
 - lib/fc/item_storage.rb
 - lib/fc/policy.rb
 - lib/fc/storage.rb
+- lib/fc/var.rb
 - lib/fc/version.rb
 - lib/filecluster.rb
 - lib/manage.rb
 - lib/manage/policies.rb
 - lib/manage/show.rb
 - lib/manage/storages.rb
+- lib/manage/var.rb
 - lib/utils.rb
 - test/base_test.rb
 - test/daemon_test.rb
@@ -170,37 +157,32 @@ files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
+- test/storage_sync.rb
 - test/storage_test.rb
+- test/var_test.rb
 - test/version_test.rb
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -778580397
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -778580397
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Distributed storage
 test_files:
 - test/base_test.rb
@@ -211,5 +193,7 @@ test_files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
+- test/storage_sync.rb
 - test/storage_test.rb
+- test/var_test.rb
 - test/version_test.rb