filecluster 0.0.3

data/lib/fc/item_storage.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+module FC
+  class ItemStorage < DbBase
+    set_table :items_storages, 'item_id, storage_name, status, time'
+    
+  end
+end

data/lib/fc/policy.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module FC
+  class Policy < DbBase
+    set_table :policies, 'storages, copies'
+    
+    def get_storages
+      FC::Storage.where("name IN (#{storages.split(',').map{|s| "'#{s}'"}.join(',')})")
+    end
+    
+    # get available storage for object by size
+    def get_proper_storage(size, exclude = [])
+      get_storages.detect do |storage|
+        !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
+      end
+    end
+  end
+end

data/lib/fc/storage.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+
+module FC
+  class Storage < DbBase
+    set_table :storages, 'name, host, path, url, size, size_limit, check_time'
+    
+    class << self
+      attr_accessor :check_time_limit
+    end
+    @check_time_limit = 120 # ttl for up status check
+    
+    def self.curr_host
+      @uname || @uname = `uname -n`.chomp
+    end
+    
+    def initialize(params = {})
+      path = (params['path'] || params[:path])
+      if path && !path.to_s.empty?
+        raise "Storage path must be like '/bla/bla../'" unless path.match(/^\/.*\/$/)
+      end
+      super params
+    end
+    
+    def update_check_time
+      self.check_time = Time.new.to_i
+      save
+    end
+    
+    def check_time_delay
+      Time.new.to_i - check_time.to_i
+    end
+    
+    def up?
+      Time.new.to_i - check_time.to_i <= self.class.check_time_limit
+    end
+    
+    # copy local_path to storage
+    def copy_path(local_path, file_name)
+      cmd = self.class.curr_host == host ? 
+        "cp -r #{local_path} #{self.path}#{file_name}" : 
+        "scp -rB #{local_path} #{self.host}:#{self.path}#{file_name}"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+    end
+    
+    # copy object to local_path
+    def copy_to_local(file_name, local_path)
+      cmd = self.class.curr_host == host ? 
+        "cp -r #{self.path}#{file_name} #{local_path}" : 
+        "scp -rB #{self.host}:#{self.path}#{file_name} #{local_path}"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+    end
+    
+    # delete object from storage
+    def delete_file(file_name)
+      cmd = self.class.curr_host == host ? 
+        "rm -rf #{self.path}#{file_name}" : 
+        "ssh -oBatchMode=yes #{self.host} 'rm -rf #{self.path}#{file_name}'"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      
+      cmd = self.class.curr_host == host ? 
+        "ls -la #{self.path}#{file_name}" : 
+        "ssh -oBatchMode=yes #{self.host} 'ls -la #{self.path}#{file_name}'"
+      r = `#{cmd} 2>/dev/null`
+      raise "Path #{self.path}#{file_name} not deleted" unless r.empty?
+    end
+    
+    # return object size on storage
+    def file_size(file_name)
+      cmd = self.class.curr_host == host ? 
+        "du -sb #{self.path}#{file_name}" : 
+        "ssh -oBatchMode=yes #{self.host} 'du -sb #{self.path}#{file_name}'"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      r.to_i
+    end
+  end
+end

data/lib/fc/version.rb

@@ -0,0 +1,3 @@
+module FC
+  VERSION = "0.0.3"
+end

data/lib/filecluster.rb

@@ -0,0 +1,13 @@
+require "date"
+require "fc/version"
+require "fc/db"
+require "fc/base"
+require "fc/item_storage"
+require "fc/item"
+require "fc/policy"
+require "fc/storage"
+require "fc/error"
+
+module FC
+
+end

data/lib/manage.rb

@@ -0,0 +1,4 @@
+require 'readline'
+require "manage/policies"
+require "manage/storages"
+require "manage/show"

data/lib/manage/policies.rb

@@ -0,0 +1,70 @@
+def policies_list
+  policies = FC::Policy.where
+  if policies.size == 0
+    puts "No storages."
+  else
+    policies.each do |policy|
+      puts "##{policy.id} storages: #{policy.storages}, copies: #{policy.copies}"
+    end
+  end
+end
+
+def policies_show
+  id = ARGV[2]
+  policy = FC::Policy.where('id = ?', id).first
+  if !policy
+    puts "Policy ##{id} not found."
+  else
+    count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
+    puts %Q{Policy
+  ID:         #{policy.id}
+  Storages:   #{policy.storages}
+  Copies:     #{policy.copies}
+  Items:      #{count}}
+  end
+end
+
+def policies_add
+  puts "Add Policy"
+  storages = stdin_read_val('Storages')
+  copies = stdin_read_val('Copies').to_i
+  begin
+    policy = FC::Policy.new(:storages => storages, :copies => copies)
+  rescue Exception => e
+    puts "Error: #{e.message}"
+    exit
+  end
+  puts %Q{\nPolicy
+  Storages:   #{storages} 
+  Copies:     #{copies}}
+  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+  puts ""
+  if s == "y" || s == "yes"
+    begin
+      policy.save
+    rescue Exception => e
+      puts "Error: #{e.message}"
+      exit
+    end
+    puts "ok"
+  else
+    puts "Canceled."
+  end
+end
+
+def policies_rm
+  id = ARGV[2]
+  policy = FC::Policy.where('id = ?', id).first
+  if !policy
+    puts "Policy ##{id} not found."
+  else
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      policy.delete
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end

data/lib/manage/show.rb

@@ -0,0 +1,57 @@
+def show_current_host
+ puts "Current host: #{FC::Storage.curr_host}"
+end
+
+def show_global_daemon
+  r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+  if r 
+    puts "Global daemon run on #{r['val']}\nLast run #{r['curr_time']-r['time']} seconds ago."
+  else
+    puts "Global daemon is not runnning."
+  end
+end
+
+def show_errors
+  count = ARGV[2] || 10
+  errors = FC::Error.where("1 ORDER BY id desc LIMIT #{count.to_i}")
+  if errors.size == 0
+    puts "No errors."
+  else
+    errors.each do |error|
+      puts "#{Time.at(error.time)} item_id: #{error.item_id}, item_storage_id: #{error.item_storage_id}, host: #{error.host}, message: #{error.message}"
+    end
+  end
+end
+
+def show_host_info
+  host = ARGV[2] || FC::Storage.curr_host
+  storages = FC::Storage.where("host = ?", host)
+  if storages.size == 0
+    puts "No storages."
+  else
+    puts "Info for host #{host}"
+    storages.each do |storage|
+      counts = FC::DB.connect.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}' GROUP BY status")
+      str = "#{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
+      str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
+      str += " #{storage.check_time_delay} seconds ago" if storage.check_time
+      str += "\n"
+      counts.each do |r|
+        str += "   Items storages #{r['status']}: #{r['cnt']}\n"
+      end
+      puts str
+    end
+  end
+end
+
+def show_items_info
+  puts "Items by status:"
+  counts = FC::DB.connect.query("SELECT status, count(*) as cnt FROM #{FC::Item.table_name} WHERE 1 GROUP BY status")
+  counts.each do |r|
+    puts "   #{r['status']}: #{r['cnt']}"
+  end
+  count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready'").first['cnt']
+  puts "Items to copy: #{count}"
+  count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > p.copies AND i.status = 'ready'").first['cnt']
+  puts "Items to delete: #{count}"
+end

data/lib/manage/storages.rb

@@ -0,0 +1,87 @@
+def storages_list
+  storages = FC::Storage.where("1 ORDER BY host")
+  if storages.size == 0
+    puts "No storages."
+  else
+    storages.each do |storage|
+      str = "#{colorize_string(storage.host, :yellow)} #{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
+      str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
+      str += " #{storage.check_time_delay} seconds ago" if storage.check_time
+      puts str
+    end
+  end
+end
+
+def storages_show
+  name = ARGV[2]
+  storage = FC::Storage.where('name = ?', name).first
+  if !storage
+    puts "Storage #{name} not found."
+  else
+    count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
+    puts %Q{Storage
+  Name:           #{storage.name}
+  Host:           #{storage.host} 
+  Path:           #{storage.path} 
+  Url:            #{storage.url}
+  Size:           #{size_to_human storage.size}
+  Size limit:     #{size_to_human storage.size_limit}
+  Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
+  Status:         #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
+  Items storages: #{count}}
+  end
+end
+
+def storages_add
+  host = FC::Storage.curr_host
+  puts "Add storage to host #{host}"
+  name = stdin_read_val('Name')
+  path = stdin_read_val('Path')
+  url = stdin_read_val('Url')
+  size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
+  begin
+    storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit)
+    size = storage.file_size('')
+  rescue Exception => e
+    puts "Error: #{e.message}"
+    exit
+  end
+  puts %Q{\nStorage
+  Name:       #{name}
+  Host:       #{host} 
+  Path:       #{path} 
+  Url:        #{url}
+  Size:       #{size_to_human size}
+  Size limit: #{size_to_human size_limit}}
+  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+  puts ""
+  if s == "y" || s == "yes"
+    storage.size = size
+    begin
+      storage.save
+    rescue Exception => e
+      puts "Error: #{e.message}"
+      exit
+    end
+    puts "ok"
+  else
+    puts "Canceled."
+  end
+end
+
+def storages_rm
+  name = ARGV[2]
+  storage = FC::Storage.where('name = ?', name).first
+  if !storage
+    puts "Storage #{name} not found."
+  else
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      storage.delete
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end

data/lib/utils.rb

@@ -0,0 +1,76 @@
+def option_parser_init(descriptions, text)
+  options = {}
+  optparse = OptionParser.new do |opts|
+    opts.banner = text
+    opts.separator "Options:"
+    
+    descriptions.each_entry do |key, desc|
+      options[key] = desc[:default]
+      opts.on("-#{desc[:short]}", "--#{desc[:full]}#{desc[:no_val] ? '' : '='+desc[:full].upcase}", desc[:text]) {|s| options[key] = s }
+    end
+    opts.on_tail("-?", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+    opts.on_tail("-v", "--version", "Show version") do
+      puts FC::VERSION
+      exit
+    end
+  end
+  optparse.parse!
+  options['optparse'] = optparse
+  options
+end
+
+def size_to_human(size)
+  return "0" if size == 0
+  units = %w{B KB MB GB TB}
+  e = (Math.log(size)/Math.log(1024)).floor
+  s = "%.2f" % (size.to_f / 1024**e)
+  s.sub(/\.?0*$/, units[e])
+end
+
+def human_to_size(size)
+  r = /^(\d+(\.\d+)?)\s*(.*)/
+  units = {'k' => 1024, 'm' => 1024*1024, 'g' => 1024*1024*1024, 't' => 1024*1024*1024*1024}
+  return nil unless matches = size.to_s.match(r)
+  unit = units[matches[3].to_s.strip.downcase[0]]
+  result = matches[1].to_f
+  result *= unit if unit
+  result.to_i
+end
+
+def stdin_read_val(name)
+  while val = Readline.readline("#{name}: ", false).strip.downcase
+    if val.empty?
+      puts "Input non empty #{name}."
+    else 
+      if block_given?
+        if err = yield(val) 
+          puts err
+        else 
+          return val
+        end
+      else 
+        return val
+      end
+    end
+  end
+end
+
+def colorize_string(str, color)
+  return str unless color
+  case color.to_s
+  when 'red'
+    color_code = 31
+  when 'green'
+    color_code = 32
+  when 'yellow'
+    color_code = 33
+  when 'pink'
+    color_code = 35
+  else
+    color_code = color.to_i
+  end 
+  "\e[#{color_code}m#{str}\e[0m"
+end

data/test/base_test.rb

@@ -0,0 +1,74 @@
+require 'helper'
+
+class BaseTest < Test::Unit::TestCase
+  class << self
+    def shutdown
+      FC::DB.connect.query("DELETE FROM items")
+    end
+  end
+  
+  def setup
+    @@policy_id ||= 1
+    @item = FC::Item.new(:name => 'test1', :tag => 'test tag', :dir => 0, :size => 100, :blabla => 'blabla', :policy_id => @@policy_id)
+    @@policy_id += 1
+  end
+  
+  should "correct init" do
+    assert_raise(NoMethodError, 'Set not table field') { @item.blabla }
+    assert @item, 'Item not created'
+    assert_nil @item.id, 'Not nil id for new item'
+  end
+  
+  should "correct add and save item" do
+    @item.save
+    assert id=@item.id, 'Nil id after save'
+    # double save check
+    @item.save
+    assert_equal id, @item.id, 'Changed id after double save'
+    @item.copies = 2
+    @item.save
+    assert_equal id, @item.id, 'Changed id after save with changes'
+  end
+  
+  should "correct where" do
+    @item.save
+    ids = [@item.id]
+    item2 = FC::Item.new(:name => 'test2', :tag => 'test tag', :dir => 0, :size => 100, :blabla => 'blabla', :policy_id => 100)
+    item2.save
+    ids << item2.id
+    items = FC::Item.where("id IN (#{ids.join(',')})")
+    assert_same_elements items.map(&:id), ids, "Items by where load <> items by find"
+  end
+  
+  should "correct reload item" do
+    @item.save
+    @item.name = 'new test'
+    @item.tag = 'new test tag'
+    @item.dir = '1'
+    @item.size = '777'
+    @item.reload
+    assert_same_elements ['test1', 'test tag', 0, 100], [@item.name, @item.tag, @item.dir, @item.size], 'Fields not restoted after reload'
+  end
+  
+  should "correct update and load item" do
+    assert_raise(RuntimeError) { FC::Item.find(12454845) }
+    @item.save
+    @item.copies = 1
+    @item.save
+    @item.outer_id = 111
+    @item.save
+    loaded_item = FC::Item.find(@item.id)
+    assert_kind_of FC::Item, loaded_item, 'Load not FC::Item'
+    assert_equal @item.name, loaded_item.name, 'Saved item name <> loaded item name'
+    assert_equal @item.tag, loaded_item.tag, 'Saved item tag <> loaded item tag'
+    assert_equal @item.dir, loaded_item.dir, 'Saved item dir <> loaded item dir'
+    assert_equal @item.size, loaded_item.size, 'Saved item size <> loaded item size'
+    assert_equal @item.copies, loaded_item.copies, 'Saved item copies <> loaded item copies'
+    assert_equal @item.outer_id, loaded_item.outer_id, 'Saved item outer_id <> loaded item outer_id'
+  end
+  
+  should "correct delete item" do
+    assert_nothing_raised { @item.delete }
+    assert_raise(RuntimeError, 'Item not deleted') { FC::Item.find(@item.id) }
+  end
+end