filecluster 0.5.14 → 0.5.15

data/lib/daemon/autosync_thread.rb

@@ -0,0 +1,16 @@
+require 'autosync'
+
+class AutosyncThread < BaseThread
+  attr_accessor :start_time, :files_to_delete, :items_to_delete
+  def go(storages)
+    storages.each do |storage|
+      $log.debug("AutosyncThread: Run storage synchronization for #{storage.name}")
+      Autosync.new(storage).run
+      storage.reload
+      storage.autosync_at = Time.now.to_i
+      storage.save
+      $log.debug("AutosyncThread: Finish storage synchronization for #{storage.name}")
+      break if $exit_signal
+    end
+  end
+end

data/lib/daemon/global_daemon_thread.rb

@@ -39,8 +39,8 @@ class GlobalDaemonThread < BaseThread
         "ist.item_id = i.id AND i.copies IN (#{copies}) AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
       r = FC::DB.query(sql)
       r.each do |row|
-        $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
         item_storages = row['storages'].split(',')
+        $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}, exclude #{item_storages}")
         if row['item_copies'] != item_storages.count
           $log.warn("GlobalDaemonThread: ItemStorage count <> item.copies for item #{row['item_id']}")
         elsif item_storages.count >= policy.copies.to_i

data/lib/daemon/update_tasks_thread.rb

@@ -9,7 +9,7 @@ class UpdateTasksThread < BaseThread
     count = 0
     limit = FC::Var.get("daemon_tasks_#{type}_group_limit", 1000).to_i
     tasks = (type == :copy ? $tasks_copy : $tasks_delete) 
-    $storages.each do |storage|
+    $storages.select { |storage| storage.write_weight.to_i >= 0 }.each do |storage|
       tasks[storage.name] = [] unless tasks[storage.name]
       ids = tasks[storage.name].map(&:id) + $curr_tasks.compact.map(&:id)
       if ids.length > limit*2

data/lib/fc/db.rb

@@ -147,7 +147,7 @@ module FC
           id bigint NOT NULL AUTO_INCREMENT,
           name varchar(1024) NOT NULL DEFAULT '',
           tag varchar(255) DEFAULT NULL,
-          outer_id int DEFAULT NULL,
+          outer_id bigint DEFAULT NULL,
           policy_id int NOT NULL,
           dir tinyint(1) NOT NULL DEFAULT 0,
           size bigint NOT NULL DEFAULT 0,
@@ -336,5 +336,9 @@ module FC
       FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN auto_size bigint(20) DEFAULT 0")
     end
 
+    def self.migrate_5
+      FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN autosync_at bigint(11) DEFAULT 0")
+    end
+
   end
 end

data/lib/fc/storage.rb

@@ -4,7 +4,7 @@ require 'fileutils'
 
 module FC
   class Storage < DbBase
-    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size'
+    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size, autosync_at'
     
     class << self
       attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
@@ -124,7 +124,7 @@ module FC
         raise r if $?.exitstatus != 0
       else
         local_path += '/' if File.stat(local_path).directory?
-        cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
+        cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"#{recreate_dirs_cmd} && ionice -c 2 -n 7 rsync\" #{local_path.shellescape} #{self.host}:\"#{dst_path.shellescape}\""
         r = `#{cmd} 2>&1`
         raise r if $?.exitstatus != 0
       end
@@ -142,7 +142,7 @@ module FC
       r = `#{cmd} 2>&1`
       src_path += '/' if $?.exitstatus == 0
 
-      cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
+      cmd = "ionice -c 2 -n 7 rsync -e \"ssh -o StrictHostKeyChecking=no\" -a --no-t #{FC::Storage.speed_limit_to_rsync_opt(speed_limit)}--rsync-path=\"ionice -c 2 -n 7 rsync\" #{self.host}:\"#{src_path.shellescape}\" #{local_path.shellescape}"
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
     end

data/lib/fc/var.rb

@@ -35,7 +35,26 @@ module FC
       end
       @all_vars 
     end
-    
+
+    def self.get_autosync
+      sync_interval = {
+        'all' => 604_800 # default 7 days in seconds
+      }
+      sync_interval.merge! Hash[get('autosync_intervals').to_s.split(';;').map { |v| v.split('::') }]
+      sync_interval.each { |host, val| sync_interval[host] = val.to_i > 0 ? val.to_i : 0 }
+    end
+
+    def self.set_autosync(host, val)
+      current = get_autosync
+      if val.nil? || val == ''
+        current.delete(host)
+      else
+        current[host] = val
+      end
+      list = current.map { |h, v| "#{h}::#{v}" }.join(';;')
+      set('autosync_intervals', list)
+    end
+
     def self.get_speed_limits
       limits = {
         'all' => nil

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.14'.freeze
+  VERSION = '0.5.15'.freeze
 end

data/lib/iostat.rb

@@ -0,0 +1,48 @@
+require 'open3'
+require 'shellwords'
+
+class Iostat
+  attr_reader :w_await, :r_await, :util, :disk_name
+  def initialize(path)
+    @path = path
+    @w_await = 0
+    @r_await = 0
+    @util = 0
+    @disk_name = 'unknown'
+    run
+  end
+
+  def run
+    @run = true
+    Thread.new do
+      disk_await_monitor
+    end
+  end
+
+  def stop
+    @run = false
+  end
+
+  private
+
+  def disk_await_monitor
+    drive = `df #{@path.shellescape}`.split("\n")[1].split(' ')[0].split('/').last
+    Open3.popen3('iostat -x 1 -p') do |_, stderr, _, thread|
+      while line = stderr.gets
+        update_stats(line) if line.split(' ')[0] == drive
+        unless @run
+          Process.kill('KILL', thread.pid) rescue nil
+          break
+        end
+      end
+    end
+  end
+
+  def update_stats(line)
+    parts = line.gsub(/\s+/, ' ').split(' ')
+    @disk_name = parts.first
+    @util = parts.last.to_f
+    @w_await = parts[parts.size - 2].to_f
+    @r_await = parts[parts.size - 3].to_f
+  end
+end

data/lib/manage.rb

@@ -5,4 +5,6 @@ require "manage/show"
 require "manage/copy_rules"
 require "manage/var"
 require "manage/item"
-require "manage/copy_speed"
+require "manage/copy_speed"
+require "manage/autosync"
+require 'autosync'

data/lib/manage/autosync.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require 'shellwords'
+
+def autosync_list
+  FC::Var.get_autosync.each do |name, val|
+    puts name.to_s+(val.to_i > 0 ? " - every: #{val} seconds" : " - never")
+  end
+end
+
+def autosync_add
+  hosts = ['all'] + all_hosts
+  puts 'Set autosync interval'
+  begin
+    host = stdin_read_val("Host (default #{FC::Storage.curr_host})", true).strip
+    host = FC::Storage.curr_host if host.empty?
+    puts "Host can be one of: #{hosts.join(', ')}" unless hosts.index(host)
+  end until hosts.index(host)
+  interval = stdin_read_val('Autosync interval, seconds (0 - never, empty = all)', true)
+  confirm_autosync_set(host, interval)
+end
+
+def autosync_change
+  return if (host = find_host).to_s.empty?
+  puts "Change autosync interval for host #{host}"
+  interval = FC::Var.get_autosync[host]
+  txt = interval.to_s.empty? ? 'default (=all)' : nil
+  txt = interval.to_i.zero? ? 'never' : "#{interval}" unless txt
+  interval = stdin_read_val("Autosync interval, seconds (now #{txt}, 0 - never, empty = all)", true)
+  confirm_autosync_set(host, interval)
+end
+
+private
+
+def confirm_autosync_set(host, interval)
+  txt = interval.to_s.empty? ? 'default (=all)' : nil
+  txt = interval.to_i.zero? ? 'never' : interval unless txt
+  puts %(\nAutosync interval
+  Host:     #{host}
+  Interval: #{txt})
+  s = Readline.readline('Continue? (y/n) ', false).strip.downcase
+  puts ''
+  if %w[y yes].include?(s)
+    begin
+      FC::Var.set_autosync(host, interval.to_s.empty? ? interval : interval.to_i)
+    rescue Exception => e
+      puts "Error: #{e.message}"
+      exit
+    end
+    puts 'ok'
+  else
+    puts 'Canceled.'
+  end
+end

data/lib/manage/copy_speed.rb

@@ -69,6 +69,6 @@ end
 
 def find_host
   host = ARGV[2].to_s.strip
-  puts "Storage with host #{host} not found." unless (['all'] + all_hosts).index(host)
+  puts "Storage with host \"#{host}\" not found." unless (['all'] + all_hosts).index(host)
   host
 end

data/lib/manage/storages.rb

@@ -203,6 +203,53 @@ def storages_change
 end
 
 def storages_sync_info
+  if storage = find_storage
+    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
+    puts "Get synchronization info for (#{storage.name}) storage and file system (#{storage.path}).."
+    init_console_logger
+    manual_sync(storage, true)
+    puts 'Done.'
+  end
+end
+
+def storages_sync
+  if storage = find_storage
+    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
+    puts "Synchronize (#{storage.name}) storage and file system (#{storage.path}).."
+    s = Readline.readline('Continue? (y/n) ', false).strip.downcase
+    puts ''
+    if s == 'y' || s == 'yes'
+      init_console_logger
+      manual_sync(storage, false)
+      s = Readline.readline('Update storage size? (y/n) ', false).strip.downcase
+      storages_update_size if s == 'y' || s == 'yes'
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+def manual_sync(storage, dry_run)
+  syncer = Autosync.new(storage, dry_run)
+  syncer.run
+  puts "Deleted #{syncer.files_to_delete.size} files"
+  puts "Deleted #{syncer.items_to_delete.size} items_storages"
+  if (ARGV[3])
+    File.open(ARGV[3], 'w') do |file|
+      syncer.files_to_delete.each { |f| file.puts f }
+    end
+    puts "Save deleted files to #{ARGV[3]}"
+  end
+
+  if (ARGV[4])
+    File.open(ARGV[4], 'w') do |file|
+      syncer.items_to_delete.each { |item_storage_id| file.puts item_storage_id }
+    end
+    puts "Save deleted items_storages to #{ARGV[4]}"
+  end
+end
+
+def storages_sync_info_old
   if storage = find_storage
     return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
     puts "Get synchronization info for (#{storage.name}) storage and file system (#{storage.path}).."
@@ -211,7 +258,7 @@ def storages_sync_info
   end
 end
 
-def storages_sync
+def storages_sync_old
   if storage = find_storage
     return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
     puts "Synchronize (#{storage.name}) storage and file system (#{storage.path}).."
@@ -230,6 +277,15 @@ end
 
 private
 
+def init_console_logger
+  require 'logger'
+  $log = Logger.new(STDOUT)
+  $log.level = Logger::DEBUG
+  $log.formatter = proc { |severity, datetime, progname, msg|
+    "[#{severity}]: #{msg}\n"
+  }
+end
+
 def find_storage
   name = ARGV[2]
   storage = FC::Storage.where('name = ?', name).first

data/test/autosync_test.rb

@@ -0,0 +1,186 @@
+require 'helper'
+require 'autosync'
+
+class AutosyncTest < Test::Unit::TestCase
+  class << self
+    def startup
+      @@storage = FC::Storage.new(
+        name: 'rec1-sda',
+        host: 'rec1',
+        size: 0,
+        copy_storages: '',
+        size_limit: 10,
+        path: '/tmp/rec-1-sda/mediaroot/'
+      )
+      `mkdir -p #{@@storage.path}`
+      `touch #{@@storage.path}healthcheck`
+      `touch -t 9901010000 #{@@storage.path}healthcheck`
+      @@storage.save
+    end
+
+    def shutdown
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM errors")
+      `rm -rf #{@@storage.path}` if @@storage.path
+    end
+  end
+
+  def setup
+    @item_storages = {}
+    [
+      'live_hls/otr_1/01/a.ts',
+      'live_hls/otr_1/01/b.ts',
+      'live_hls/otr_1/02/c.ts'
+    ].each { |i| @item_storages[i] = create_item_storage(i) }
+  end
+
+  def teardown
+    FC::DB.query("DELETE FROM items_storages")
+    FC::DB.query("DELETE FROM items")
+    FC::DB.query("DELETE FROM errors")
+    `rm -rf #{@@storage.path}` if @@storage.path
+  end
+
+  should "fill_db with items on storage" do
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    assert db_struct['live_hls']
+    assert db_struct['live_hls']['otr_1']
+    assert db_struct['live_hls']['otr_1']['01']
+    assert db_struct['live_hls']['otr_1']['01']['a.ts']
+    assert db_struct['live_hls']['otr_1']['01']['b.ts']
+    assert db_struct['live_hls']['otr_1']['02']
+    assert db_struct['live_hls']['otr_1']['02']['c.ts']
+  end
+
+  should "content synchonized, nothing to delete from DISK and DB" do
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.empty?
+    as.scan_db(db_struct, '')
+    assert as.items_to_delete.empty?
+  end
+
+  should 'select to remove disk file entry unless found in DB' do
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    unstored_file = "#{@@storage.path}live_hls/otr_1/01/z.ts"
+    `touch -t 9901010000 #{unstored_file}`
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.size == 1
+    assert as.files_to_delete.first == unstored_file
+  end
+
+  should 'physically remove disk entries older than 1 hour unless found in DB' do
+    `mkdir -p #{@@storage.path}live_hls/empty_subfolders/sub1/sub2`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders/sub1`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders/sub1/sub2`
+    assert File.exist?("#{@@storage.path}live_hls/empty_subfolders")
+
+    `mkdir -p #{@@storage.path}live_hls/not_empty_subfolders/sub1/sub2`
+    `touch -t 9901010000 #{@@storage.path}live_hls/not_empty_subfolders`
+    `touch -t 9901010000 #{@@storage.path}live_hls/not_empty_subfolders/sub1`
+    assert File.exist?("#{@@storage.path}live_hls/not_empty_subfolders/sub1/sub2")
+
+    `touch -t 9901010000 #{@@storage.path}live_hls/not_empty_subfolders/sub1/sub2/some_file`
+    assert File.exist?("#{@@storage.path}live_hls/not_empty_subfolders/sub1/sub2/some_file")
+    `touch -t 9901010000 #{@@storage.path}live_hls/not_empty_subfolders/sub1/sub2`
+
+    `mkdir -p #{@@storage.path}live_hls/new_empty_folders/sub1/sub2`
+
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.size == 2
+    as.delete_diffs
+    assert !File.exist?("#{@@storage.path}live_hls/empty_subfolders")
+    assert !File.exist?("#{@@storage.path}live_hls/not_empty_subfolders")
+    assert File.exist?("#{@@storage.path}live_hls/new_empty_folders/sub1/sub2")
+  end
+
+  should 'not remove folder from disk if some file appeared between disk scan and delete process' do
+    # make old empty folder
+    `mkdir -p #{@@storage.path}live_hls/empty_subfolders/sub1/sub2`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders/sub1`
+    `touch -t 9901010000 #{@@storage.path}live_hls/empty_subfolders/sub1/sub2`
+    assert File.exist?("#{@@storage.path}live_hls/empty_subfolders")
+
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    # scan disk
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.size == 1
+    assert as.files_to_delete[0] == "#{@@storage.path}live_hls/empty_subfolders"
+    # new file
+    `touch #{@@storage.path}live_hls/empty_subfolders/sub1/sub2/new_appeared_file`
+    as.delete_diffs
+    assert File.exist?("#{@@storage.path}live_hls/empty_subfolders/sub1/sub2/new_appeared_file")
+  end
+
+  should 'select for delete file with mtime less than 1 hour' do
+    old_file = "#{@@storage.path}live_hls/otr_1/01/old.ts"
+    new_file = "#{@@storage.path}live_hls/otr_1/01/new.ts"
+    `touch -t #{(Time.now - 3601).strftime('%Y%m%d%H%M.%S')} #{old_file}`
+    `touch -t #{(Time.now - 3509).strftime('%Y%m%d%H%M.%S')} #{new_file}`
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.size == 1
+    assert as.files_to_delete.first == old_file
+  end
+
+  should 'not select for delete disk folder with entries which is in db' do
+    `mkdir -p #{@@storage.path}track/01`
+    create_item_storage('track/01')
+    # make it all old
+    `touch -t 9901010000 #{@@storage.path}track`
+    `touch -t 9901010000 #{@@storage.path}track/01`
+    # make some old files
+    `touch -t 9901010000 #{@@storage.path}track/01/s-0001.ts`
+    `touch -t 9901010000 #{@@storage.path}track/01/s-0002.ts`
+    `touch -t 9901010000 #{@@storage.path}track/01/s-0003.ts`
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.size.zero?
+  end
+
+  should 'select and remove DB item if not exist on disk' do
+    item_name = @item_storages.keys.first
+    `rm -f #{@@storage.path}#{item_name}`
+    removed_item_storage_id = @item_storages[item_name].id
+    as = Autosync.new(@@storage)
+    db_struct = as.fill_db
+    as.scan_disk(db_struct, '')
+    assert as.files_to_delete.empty?
+    as.scan_db(db_struct, '')
+    assert as.items_to_delete.size == 1
+    assert as.items_to_delete[0] == removed_item_storage_id
+    as.delete_diffs
+    @item_storages[item_name].reload
+    assert @item_storages[item_name].status == 'error'
+    assert FC::Error.where('1').to_a.size == 1
+  end
+
+  def create_item_storage(item_name)
+    item = FC::Item.new
+    item.name = item_name
+    item.size = 0
+    item.save
+    item_storage = FC::ItemStorage.new
+    item_storage.item_id = item.id
+    item_storage.storage_name = @@storage.name
+    item_storage.status = 'ready'
+    item_storage.save
+    `mkdir -p #{@@storage.path}#{File.dirname(item_name)}`
+    `touch -t 9901010000 #{@@storage.path}#{item_name}`
+    item_storage
+  end
+
+end