filecluster 0.0.3

data/lib/daemon/check_thread.rb

@@ -0,0 +1,12 @@
+class CheckThread < BaseThread
+  def go(storage_name)
+    $log.debug("CheckThread: Run stotage check for #{storage_name}")
+    storage = $storages.detect{|s| s.name == storage_name}
+    if File.writable?(storage.path)
+      storage.update_check_time
+    else 
+      error "Storage #{storage.name} with path #{storage.path} not writable"
+    end
+    $log.debug("CheckThread: Finish stotage check for #{storage_name}")
+  end
+end

data/lib/daemon/global_daemon_thread.rb

@@ -0,0 +1,60 @@
+class GlobalDaemonThread < BaseThread
+  def go(timeout)
+    $log.info("Start global daemon thread with timeout=#{timeout}")
+    while true do
+      exit if $exit_signal
+      sleep timeout.to_f/2
+      exit if $exit_signal
+      
+      r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+      if r['val'] == FC::Storage.curr_host
+        FC::DB.connect.query("UPDATE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}' WHERE name='global_daemon_host'")
+      else
+        $log.info("Exit from GlobalDaemonThread: global daemon already running on #{r['val']}")
+        FC::DB.close
+        exit
+      end
+      
+      make_item_copies
+      
+      #периодическая проверка на item со статусом delete, последним обновлением дольше суток (NOW - time > 86400) и без связанных is - удаление таких из базы
+      #периодически удалять (проставлять статус delete) для лиших is (число копий больше необходимого)
+    end
+  end
+  
+  # make item copies by policy
+  def make_item_copies
+    $log.debug("GlobalDaemonThread: make_item_copies")
+    
+    all_storages = FC::Storage.where
+    all_policies = FC::Policy.where
+    
+    # policies.get_storages => all_policies.select
+    all_policies.each do |policy|
+      metaclass = class << policy; self; end
+      metaclass.send(:define_method, :get_storages) do
+        policy_storages = self.storages.split(',')
+        all_storages.select{|storage| policy_storages.member?(storage.name)}
+      end
+    end
+    
+    sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name) as storages, p.id as policy_id, p.copies as policy_copies "+
+      "FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p, #{FC::ItemStorage.table_name} as ist WHERE "+
+      "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT 1000"
+    r = FC::DB.connect.query(sql)
+    r.each do |row|
+      $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
+      item_storages = row['storages'].split(',')
+      if row['item_copies'] != item_storages.size
+        $log.warn("GlobalDaemonThread: ItemStorage count <> item.copies for item #{row['item_id']}")
+      elsif item_storages.size >= row['policy_copies']
+        $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
+      else
+        policy = all_policies.detect{|p| row['policy_id'] == p.id}
+        storage = policy.get_proper_storage(row['size'], item_storages) if policy
+        error 'No available storage', :item_id => row['item_id'] unless storage
+        FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
+      end
+    end
+  end
+end

data/lib/daemon/task_thread.rb

@@ -0,0 +1,37 @@
+class TaskThread < BaseThread
+  def go(storage_name)
+    while task = $tasks[storage_name].shift do
+      $curr_task[storage_name] = task
+      $log.debug("TaskThread(#{storage_name}): run task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
+      if task[:action] == :delete
+        make_delete(task[:item_storage])
+      elsif task[:action] == :copy
+        make_copy(task[:item_storage])
+      else
+        error "Unknown task action: #{task[:action]}"
+      end
+      $curr_task[storage_name] = nil
+      $log.debug("TaskThread(#{storage_name}): Finish task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
+    end
+  end
+  
+  def make_delete(item_storage)
+    storage = $storages.detect{|s| s.name == item_storage.storage_name}
+    item = FC::Item.find(item_storage.item_id)
+    storage.delete_file(item.name)
+    item_storage.delete
+  rescue Exception => e
+    error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
+  end
+  
+  def make_copy(item_storage)
+    # TODO: не лазить в базу за item, item_storages - перенести на стадию подготовки task-а
+    storage = $storages.detect{|s| s.name == item_storage.storage_name}
+    item = FC::Item.find(item_storage.item_id)
+    src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
+    src_storage = $storages.detect{|s| s.name == src_item_storage.storage_name}
+    item.copy_item_storage(src_storage, storage, item_storage)
+  rescue Exception => e
+    error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
+  end
+end

data/lib/fc/base.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+module FC
+  class DbBase
+    attr_accessor :id, :database_fields
+    
+    class << self
+      attr_accessor :table_name, :table_fields
+    end
+    
+    def initialize(params = {})
+      self.class.table_fields.each {|key| self.send("#{key}=", params[key] || params[key.to_sym]) }
+      @id = (params["id"] || params[:id]).to_i if params["id"] || params[:id]
+      @database_fields = params[:database_fields] || {}
+    end
+    
+    # define table name and fields
+    def self.set_table(name, fields)
+      self.table_name = "#{FC::DB.prefix}#{name}"
+      self.table_fields = fields.split(',').map{|e| e.gsub(' ','')}
+      self.table_fields.each{|e| attr_accessor e.to_sym}
+    end
+    
+    # make instance on fields hash
+    def self.create_from_fiels(data)
+      # use only defined in set_table fields
+      database_fields = data.select{|key, val| self.table_fields.include?(key.to_s)}
+      self.new(database_fields.merge({:id => data["id"].to_s, :database_fields => database_fields}))
+    end
+    
+    # get element by id
+    def self.find(id)
+      r = FC::DB.connect.query("SELECT * FROM #{self.table_name} WHERE id=#{id.to_i}")
+      raise "Record not found (#{self.table_name}.id=#{id})" if r.count == 0
+      self.create_from_fiels(r.first)
+    end
+    
+    # get elements array by sql condition (possibles '?' placeholders)
+    def self.where(cond = "1", *params)
+      i = -1
+      sql = "SELECT * FROM #{self.table_name} WHERE #{cond.gsub('?'){i+=1; "'#{Mysql2::Client.escape(params[i].to_s)}'"}}"
+      r = FC::DB.connect.query(sql)
+      r.map{|data| self.create_from_fiels(data)}
+    end
+    
+    # save changed fields
+    def save
+      sql = @id.to_i != 0 ? "UPDATE #{self.class.table_name} SET " : "INSERT IGNORE INTO #{self.class.table_name} SET "
+      fields = []
+      self.class.table_fields.each do |key|
+        db_val = @database_fields[key]
+        val = self.send(key)
+        if val.to_s != db_val.to_s || val.nil? && !db_val.nil? || !val.nil? && db_val.nil?
+          fields << "#{key}=#{val ? (val.class == String ? "'#{FC::DB.connect.escape(val)}'" : val.to_i) : 'NULL'}"
+        end
+      end
+      if fields.length > 0
+        sql << fields.join(',')
+        sql << " WHERE id=#{@id.to_i}" if @id
+        FC::DB.connect.query(sql)
+        @id = FC::DB.connect.last_id unless @id
+        self.class.table_fields.each do |key|
+          @database_fields[key] = self.send(key)
+        end
+      end
+    end
+    
+    # reload object from DB
+    def reload
+      raise "Can't reload object without id" if !@id || @id.to_i == 0
+      new_obj = self.class.find(@id)
+      self.database_fields = new_obj.database_fields
+      self.class.table_fields.each {|key| self.send("#{key}=", new_obj.send(key)) }
+    end
+
+    # delete object from DB
+    def delete
+      FC::DB.connect.query("DELETE FROM #{self.class.table_name} WHERE id=#{@id.to_i}") if @id
+    end
+    
+  end
+end

data/lib/fc/db.rb

@@ -0,0 +1,168 @@
+require 'mysql2'
+
+module FC
+  module DB
+    class << self; attr_accessor :options, :prefix end
+    
+    def DB.connect_by_config(options)
+      @options = options.clone
+      @options[:port] = options[:port].to_i if options[:port]
+      @connects = {}
+      @connects[Thread.current.object_id] = Mysql2::Client.new(@options)
+      @prefix = options[:prefix].to_s
+    end
+    
+    def self.connect
+      if @options[:multi_threads]
+        @connects[Thread.current.object_id] ||= Mysql2::Client.new(@options)
+      else
+        @connects.first[1]
+      end
+    end
+    
+    def self.close
+      if @options[:multi_threads]
+        if @connects[Thread.current.object_id]
+          @connects[Thread.current.object_id].close
+          @connects.delete(Thread.current.object_id)
+        end
+      else
+        @connects.first[1].close
+      end
+    end
+    
+    def DB.init_db
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}items (
+          id int NOT NULL AUTO_INCREMENT,
+          name varchar(1024) NOT NULL DEFAULT '',
+          tag varchar(255) DEFAULT NULL,
+          outer_id int DEFAULT NULL,
+          policy_id int NOT NULL,
+          dir tinyint(1) NOT NULL DEFAULT 0,
+          size bigint NOT NULL DEFAULT 0,
+          status ENUM('new', 'ready', 'error', 'delete') NOT NULL DEFAULT 'new',
+          time int DEFAULT NULL,
+          copies int NOT NULL DEFAULT 0,
+          PRIMARY KEY (id), UNIQUE KEY (name(255), policy_id), 
+          KEY (outer_id), KEY (time, status), KEY (status),  KEY (copies, status, policy_id)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      proc_time = %{
+        SET NEW.time = UNIX_TIMESTAMP();
+      }
+      FC::DB.connect.query("CREATE TRIGGER fc_items_before_insert BEFORE INSERT on #{@prefix}items FOR EACH ROW BEGIN #{proc_time} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_items_before_update BEFORE UPDATE on #{@prefix}items FOR EACH ROW BEGIN #{proc_time} END")
+      
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}storages (
+          id int NOT NULL AUTO_INCREMENT,
+          name varchar(255) NOT NULL DEFAULT '',
+          host varchar(255) NOT NULL DEFAULT '',
+          path text NOT NULL DEFAULT '',
+          url text NOT NULL DEFAULT '',
+          size bigint NOT NULL DEFAULT 0,
+          size_limit bigint NOT NULL DEFAULT 0,
+          check_time int DEFAULT NULL,
+          PRIMARY KEY (id), UNIQUE KEY (name), KEY (host)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      proc = %{
+        # update policy.storages on storage delete and update
+        UPDATE #{@prefix}policies, 
+          (SELECT #{@prefix}policies.id, GROUP_CONCAT(name) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON FIND_IN_SET(name, storages) GROUP BY #{@prefix}policies.id) as new_policy
+        SET #{@prefix}policies.storages = new_policy.storages WHERE #{@prefix}policies.id = new_policy.id;
+      }
+      proc_update = %{
+        IF OLD.name <> NEW.name THEN 
+          #{proc}
+        END IF;
+      }
+      FC::DB.connect.query("CREATE TRIGGER fc_storages_after_delete AFTER DELETE on #{@prefix}storages FOR EACH ROW BEGIN #{proc} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_storages_after_update AFTER UPDATE on #{@prefix}storages FOR EACH ROW BEGIN #{proc_update} END")
+      
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}policies (
+          id int NOT NULL AUTO_INCREMENT,
+          storages text NOT NULL DEFAULT '',
+          copies int NOT NULL DEFAULT 0,
+          PRIMARY KEY (id)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      proc = %{
+        # update policy.storages on policy change - guarantee valid policy.storages
+        SELECT GROUP_CONCAT(name) INTO @storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.storages);
+        SET NEW.storages = @storages_list;
+      }
+      FC::DB.connect.query("CREATE TRIGGER fc_policies_before_insert BEFORE INSERT on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_policies_before_update BEFORE UPDATE on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
+      
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}items_storages (
+          id int NOT NULL AUTO_INCREMENT,
+          item_id int DEFAULT NULL,
+          storage_name varchar(255) DEFAULT NULL,
+          status ENUM('new', 'copy', 'error', 'ready', 'delete') NOT NULL DEFAULT 'new',
+          time int DEFAULT NULL,
+          PRIMARY KEY (id), UNIQUE KEY (item_id, storage_name), KEY (storage_name), KEY (time, status), KEY (status, storage_name),          
+          FOREIGN KEY (item_id) REFERENCES #{@prefix}items(id) ON UPDATE RESTRICT ON DELETE RESTRICT,
+          FOREIGN KEY (storage_name) REFERENCES #{@prefix}storages(name) ON UPDATE RESTRICT ON DELETE RESTRICT
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      proc = %{
+        SELECT status, copies, size INTO @item_status, @item_copies, @item_size FROM #{@prefix}items WHERE id = NEW.item_id;
+        SET @curr_copies = (SELECT count(*) FROM #{@prefix}items_storages WHERE item_id = NEW.item_id AND status <> 'delete');
+        SET @curr_copies_ready = (SELECT count(*) FROM #{@prefix}items_storages WHERE item_id = NEW.item_id AND status = 'ready');
+        # calc item.copies
+        IF @curr_copies <> @item_copies THEN 
+          UPDATE #{@prefix}items SET copies=@curr_copies WHERE id = NEW.item_id;
+        END IF;
+        # check error status
+        IF @item_status <> 'new' AND @item_status <> 'delete' AND @curr_copies_ready = 0 THEN 
+          UPDATE #{@prefix}items SET status='error' WHERE id = NEW.item_id;
+        END IF;
+        # check ready status
+        IF @curr_copies_ready > 0 THEN 
+          UPDATE #{@prefix}items SET status='ready' WHERE id = NEW.item_id;
+        END IF;
+      }
+      proc_add = %{
+        #{proc}
+        UPDATE #{@prefix}storages SET size=size+@item_size WHERE name = NEW.storage_name;
+      }
+      proc_del = %{
+        #{proc.gsub('NEW', 'OLD')}
+        UPDATE #{@prefix}storages SET size=size-@item_size WHERE name = OLD.storage_name;
+      }
+      FC::DB.connect.query("CREATE TRIGGER fc_items_storages_before_insert BEFORE INSERT on #{@prefix}items_storages FOR EACH ROW BEGIN #{proc_time} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_items_storages_before_update BEFORE UPDATE on #{@prefix}items_storages FOR EACH ROW BEGIN #{proc_time} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_items_storages_after_update AFTER UPDATE on #{@prefix}items_storages FOR EACH ROW BEGIN #{proc} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_items_storages_after_insert AFTER INSERT on #{@prefix}items_storages FOR EACH ROW BEGIN #{proc_add} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_items_storages_after_delete AFTER DELETE on #{@prefix}items_storages FOR EACH ROW BEGIN #{proc_del} END")
+      
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}errors (
+          id int NOT NULL AUTO_INCREMENT,
+          item_id int DEFAULT NULL,
+          item_storage_id int DEFAULT NULL,
+          host varchar(255) DEFAULT NULL,
+          message varchar(255) DEFAULT NULL,
+          time int DEFAULT NULL,
+          PRIMARY KEY (id), KEY (item_id), KEY (item_storage_id), KEY (host), KEY (time)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      FC::DB.connect.query("CREATE TRIGGER fc_errors_before_insert BEFORE INSERT on #{@prefix}errors FOR EACH ROW BEGIN #{proc_time} END")
+      
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}vars (
+          name varchar(255) DEFAULT NULL,
+          val varchar(255) DEFAULT NULL,
+          time int DEFAULT NULL,
+          PRIMARY KEY (name)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      FC::DB.connect.query("CREATE TRIGGER fc_vars_before_insert BEFORE INSERT on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
+      FC::DB.connect.query("CREATE TRIGGER fc_vars_before_update BEFORE UPDATE on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
+    end
+  end
+end

data/lib/fc/error.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+module FC
+  class Error < DbBase
+    set_table :errors, 'item_id, item_storage_id, host, message, time'
+    
+    def self.raise(error, options = {})
+      self.new(options.merge(:host => FC::Storage.curr_host, :message => error)).save
+      Kernel.raise error
+    end 
+  end
+end

data/lib/fc/item.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+
+module FC
+  class Item < DbBase
+    set_table :items, 'name, tag, outer_id, policy_id, dir, size, status, time, copies'
+    
+    # create item by local path 
+    # TODO проверка curr_host и local_path одному из доступных стораджей -> создание без копирования (для кусочков)
+    def self.create_from_local(local_path, item_name, policy, options={})
+      raise 'Path not exists' unless File.exists?(local_path)
+      raise 'Policy is not FC::Policy' unless policy.instance_of?(FC::Policy)
+      item_params = options.merge({
+        :name => item_name.to_s.gsub('//', '/').sub(/\/$/, '').sub(/^\//, '').strip,
+        :policy_id => policy.id,
+        :dir => File.directory?(local_path),
+        :size => `du -sb #{local_path}`.to_i
+      })
+      item_params.delete(:replace)
+      raise 'Name is empty' if item_params[:name].empty?
+      raise 'Zero size path' if item_params[:size] == 0
+      
+      # new item?
+      item = FC::Item.where('name=? AND policy_id=?', item_params[:name], policy.id).first
+      if item
+        if options[:replace]
+          # mark delete item_storages on replace
+          FC::DB.connect.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{item.id}")
+          # replace all fields
+          item_params.each{|key, val| item.send("#{key}=", val)}
+        else
+          FC::Error.raise 'Item already exists', :item_id => item.id
+        end
+      else
+        item = FC::Item.new(item_params)
+      end
+      item.save
+      
+      storage = policy.get_proper_storage(item.size)
+      FC::Error.raise 'No available storage', :item_id => item.id unless storage
+      
+      item_storage = item.make_item_storage(storage)
+      item.copy_item_storage(local_path, storage, item_storage)
+      return item
+    end
+    
+    def make_item_storage(storage, status = 'new')
+      # new storage_item?
+      item_storage = FC::ItemStorage.where('item_id=? AND storage_name=?', id, storage.name).first
+      item_storage.delete if item_storage
+      
+      item_storage = FC::ItemStorage.new({:item_id => id, :storage_name => storage.name, :status => status})
+      item_storage.save
+      item_storage
+    end
+    
+    def copy_item_storage(src, storage, item_storage)
+      begin
+        if src.instance_of?(FC::Storage)
+          src.copy_to_local(name, "#{storage.path}#{name}")
+        else
+          storage.copy_path(src, name)
+        end
+        size_on_storage = storage.file_size(name)
+      rescue Exception => e
+        item_storage.status = 'error'
+        item_storage.save
+        FC::Error.raise "Copy error: #{e.message}", :item_id => id, :item_storage_id => item_storage.id
+      else
+        begin
+          item_storage.reload
+        rescue Exception => e
+          FC::Error.raise "After copy error: #{e.message}", :item_id => id, :item_storage_id => item_storage.id
+        else
+          if size_on_storage != size
+            item_storage.status = 'error'
+            item_storage.save
+            FC::Error.raise "Check size after copy error", :item_id => id, :item_storage_id => item_storage.id
+          else
+            item_storage.status = 'ready'
+            item_storage.save
+            reload
+          end
+        end
+      end
+    end
+    
+    # mark items_storages for delete
+    def mark_deleted
+      FC::DB.connect.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{id}")
+      self.status = 'delete'
+      save
+    end
+    
+    def dir?
+      dir.to_i == 1
+    end
+    
+    def get_item_storages
+      FC::ItemStorage.where("item_id = #{id}")
+    end
+  end
+end