filecluster 0.0.3

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in filecluster.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 sh
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Dstorage
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'dstorage'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dstorage
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList["test/*_test.rb"]
+end
+
+desc "Run tests"
+task :default => :test

data/TODO

@@ -0,0 +1,9 @@
+добавить в check storage проверку на дорступность урла
+
+алерт на доступность в каждоый политике стораджа на запись
+алерт на доступность стораджей
+алерт: проверка раз в сутки на количество is задержавшихся в статусе delete и copy дольше суток (NOW - time > 86400)
+
+bin:
+управление из командной строки: добавление, изменение и получение статуса объектов
+в ручную запускаемая задача синхронизации фс и бд

data/bin/fc-daemon

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+require 'psych'
+require 'logger'
+require 'optparse'
+require 'filecluster'
+require 'utils'
+require 'daemon'
+
+$storages = []
+$tasks = {}           # tasks by storage name
+$curr_task = {}       # task by storage name
+$tasks_threads = {}   # threads by storage name
+$check_threads = {}   # threads by storage name
+$exit_signal = false 
+$global_daemon_thread = nil
+
+default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
+descriptions = {
+  :config      => {:short => 'c',  :full => 'config',     :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
+  :log_level   => {:short => 'l',  :full => 'log_level',  :default => 'info', :text => 'log level (fatal, error, warn, info or debug), default info'},
+  :cycle_time  => {:short => 't',  :full => 'time',       :default => 30, :text => 'Time between checks database and storages available, default 30'},
+  :global_wait => {:short => 'g', :full => 'wait',       :default => 120, :text => 'Time between runs global daemon if it does not running, default 120'},
+  :curr_host   => {:short => 'h', :full => 'host',       :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
+}
+desc = %q{Run FileCluster daemon.
+Usage: fc-daemon [options]}
+options = option_parser_init(descriptions, desc)
+FC::Storage.instance_variable_set(:@uname, options[:curr_host]) if options[:curr_host] && options[:curr_host] != FC::Storage.curr_host
+
+STDOUT.sync = true
+$log = Logger.new(STDOUT)
+$log.formatter = proc do |severity, datetime, progname, msg|
+  "[#{datetime}] [#{severity}] [#{Thread.current.object_id}] #{msg}\n"
+end
+$log.level = Logger.const_get(options[:log_level].upcase)
+$log.info('Started')
+
+db_options = Psych.load(File.read(options[:config]))
+FC::DB.connect_by_config(db_options.merge(:reconnect => true, :multi_threads => true))
+$log.info('Connected to database')
+
+def quit_on_quit
+  $log.info('Exit signal')
+  $exit_signal = true
+end
+trap("TERM") {quit_on_quit}
+trap("INT") {quit_on_quit}
+
+while true do
+  if $exit_signal
+    $log.debug('wait tasks_threads')
+    $tasks_threads.each {|t| t.join}
+    if $global_daemon_thread
+      $log.debug('wait global_daemon_thread')
+      $global_daemon_thread.join
+    end 
+    $log.info('Exit')
+    exit
+  else
+    run_global_daemon options[:global_wait].to_i
+    update_storages
+    storages_check
+    update_tasks
+    run_tasks
+  end
+  $log.debug('sleep')
+  sleep options[:cycle_time].to_i
+end

data/bin/fc-manage

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+require 'psych'
+require 'logger'
+require 'optparse'
+require 'filecluster'
+require 'utils'
+require 'manage'
+
+default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
+descriptions = {
+  :config      => {:short => 'c',  :full => 'config',     :default => default_db_config, :text => "path to db.yml file, default #{default_db_config}"},
+  :curr_host   => {:short => 'h', :full => 'host',       :default => FC::Storage.curr_host, :text => "Host for storages, default #{FC::Storage.curr_host}"}
+}
+commands_help = {
+  'storages' => [
+    'show list and manage storages', 
+    %q{Usage: fc-manage [options] storages <command>
+Command:
+   list        show all stotages for all hosts
+   show <name> show full info for storage   
+   add         add new storage
+   rm <name>   delete storage
+}],
+  'policies' => [
+    'show list and manage plicies', 
+    %q{Usage: fc-manage [options] plicies <command>
+Command:
+   list        show all plicies
+   show <id>   show full info for policy   
+   add         add new policy
+   rm <id>     delete policy
+}],
+  'show' => [
+    'show variable', 
+    %q{Usage: fc-manage [options] show <variable>
+Variable:
+   current_host        show current host
+   global_daemon       show host and uptime where run global daemon
+   errors [<count>]    show last count (default 10) errors
+   host_info [<host>]  show info for host (default current host)
+   items_info          show items statistics       
+}]
+}
+desc = %q{Get info and manage for storages, policies and items.
+Usage: fc-manage [options] <command> [<args>]
+Commands:
+}
+commands_help.each{|key, val| desc << "   #{key}#{" "*(10-key.size)}#{val[0]}\n"}
+desc << "   help      show help for commands ('fc-manage help <command>')\n"
+$options = option_parser_init(descriptions, desc)
+FC::Storage.instance_variable_set(:@uname, $options[:curr_host]) if $options[:curr_host] && $options[:curr_host] != FC::Storage.curr_host
+trap("INT", proc {exit})
+
+STDOUT.sync = true
+db_options = Psych.load(File.read($options[:config]))
+FC::DB.connect_by_config(db_options.merge(:reconnect => true, :multi_threads => true))
+
+command = ARGV[0]
+if ARGV.empty?
+  puts $options['optparse']
+  exit
+end
+
+case command
+when 'help'
+  if !ARGV[1]
+    puts $options['optparse']
+  elsif commands_help[ARGV[1]]
+    puts commands_help[ARGV[1]][1]
+  else
+    puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
+  end
+when 'show'
+  if !ARGV[1]
+    puts "Need variable name. See 'fc-manage help show'."
+  elsif self.private_methods.member?("show_#{ARGV[1]}".to_sym)
+    send "show_#{ARGV[1]}"
+  else
+    puts "Unknown variable. See 'fc-manage help show'."
+  end
+when 'storages'
+  if !ARGV[1]
+    puts "Need command. See 'fc-manage help storages'."
+  elsif self.private_methods.member?("storages_#{ARGV[1]}".to_sym)
+    send "storages_#{ARGV[1]}"
+  else
+    puts "Unknown command. See 'fc-manage help storages'."
+  end
+when 'policies'
+  if !ARGV[1]
+    puts "Need command. See 'fc-manage help policies'."
+  elsif self.private_methods.member?("policies_#{ARGV[1]}".to_sym)
+    send "policies_#{ARGV[1]}"
+  else
+    puts "Unknown command. See 'fc-manage help policies'."
+  end
+else
+  puts "'#{command}' is not a fc-manage command. See 'fc-manage --help'."
+end
+

data/bin/fc-setup-db

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+require 'optparse'
+require 'psych'
+require 'filecluster'
+require 'utils'
+require 'readline'
+
+descriptions = {
+  :host =>       {:short => 'h', :full => 'host',     :default => 'localhost', :text => 'mysql host name, default "localhost"', :save => true},
+  :database =>   {:short => 'd', :full => 'db',       :default => 'fc',        :text => 'mysql database, default "fc"', :save => true},
+  :username =>   {:short => 'u', :full => 'user',     :default => 'root',      :text => 'mysql user, default "root"', :save => true},
+  :password =>   {:short => 'p', :full => 'password', :default => '',          :text => 'mysql password, default ""', :save => true},
+  :port =>       {:short => 'P', :full => 'port',     :default => '3306',      :text => 'mysql port, default "3306"', :save => true},
+  :prefix =>     {:short => 't', :full => 'prefix',   :default => '',          :text => 'tables prefix, default ""', :save => true},
+  :init_tables =>{:short => 'i', :full => 'init',     :default => false,       :text => 'init tables, default no', :no_val => true}
+}
+desc = %q{Setup FileCluster database connection options.
+Create tables if nessary.
+Usage: fc-init-db [options]}
+options = option_parser_init(descriptions, desc)
+options.delete('optparse')
+trap("INT", proc {exit})
+
+puts options.inspect.gsub(/[\{\}\:]/, "").gsub(", ", "\n").gsub(/(.{7,})=>/, "\\1:\t").gsub("=>", ":\t\t")
+
+s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+puts ""
+if s == "y" || s == "yes"
+  print "Test connection..  "
+  FC::DB.connect_by_config(options)
+  puts "ok"
+  if options[:init_tables]
+    print "Make tables..      "
+    FC::DB.init_db
+    puts "ok"
+  end
+  print "Save to config..   "
+  options.select!{|key, val| descriptions[key][:save]}
+  File.open(File.expand_path(File.dirname(__FILE__))+'/db.yml', 'w') do |f|
+    f.write(options.to_yaml)
+  end
+  puts "ok"
+else
+  puts "Canceled."
+end

data/filecluster.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/fc/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["sh"]
+  gem.email         = ["cntyrf@gmail.com"]
+  gem.description   = %q{Distributed storage}
+  gem.summary       = %q{Distributed storage}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "filecluster"
+  gem.require_paths = ["lib"]
+  gem.version       = FC::VERSION
+  
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "shoulda-context"
+  gem.add_development_dependency "mocha", ">= 0.13.3"
+end

data/lib/daemon.rb

@@ -0,0 +1,91 @@
+require "date"
+require "daemon/base_thread"
+require "daemon/global_daemon_thread"
+require "daemon/check_thread"
+require "daemon/task_thread"
+
+def error(msg, options = {})
+  $log.error(msg)
+  FC::Error.new(options.merge(:host => FC::Storage.curr_host, :message => msg)).save
+end
+
+class << FC::Error
+  def raise(msg, options = {})
+    error(msg, options)
+  end
+end
+
+def run_global_daemon(timeout)
+  $log.debug('Run global daemon check')
+  r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+  if !r || r['curr_time'].to_i - r['time'].to_i > timeout
+    $log.debug('Set global daemon host to current')
+    FC::DB.connect.query("REPLACE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}', name='global_daemon_host'")
+    sleep 1
+    r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+  end
+  if r['val'] == FC::Storage.curr_host
+    if !$global_daemon_thread || !$global_daemon_thread.alive?
+      $log.debug("spawn GlobalDaemonThread")
+      $global_daemon_thread = GlobalDaemonThread.new(timeout)
+    end 
+  else
+    if $global_daemon_thread
+      $log.warn("Kill global daemon thread (new host = #{r['host']})")
+      $global_daemon_thread.exit
+    end 
+  end
+end
+
+def update_storages
+  $log.debug('Update storages')
+  $storages = FC::Storage.where('host = ?', FC::Storage.curr_host)
+end
+
+def storages_check
+  $log.debug('Run storages check')
+  $check_threads.each do |storage_name, thread|
+    if thread.alive?
+      error "Storage #{storage_name} check timeout"
+      thread.exit
+    end
+  end
+  $storages.each do|storage| 
+    $log.debug("spawn CheckThread for #{storage.name}")
+    $check_threads[storage.name] = CheckThread.new(storage.name)
+  end
+end
+
+def update_tasks
+  $log.debug('Update tasks')
+  return if $storages.length == 0
+  
+  def check_tasks(type)
+    storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
+    cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
+    ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
+      flatten.map{|task| task[:item_storage].id}
+    $curr_task.map{|storage_name, task| ids << task[:item_storage].id if task && task[:action] == type}
+      
+    cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
+    FC::ItemStorage.where(cond).each do |item_storage|
+      $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
+      $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
+      $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
+    end
+  end
+  
+  check_tasks(:delete)
+  check_tasks(:copy)
+end
+
+def run_tasks
+  $log.debug('Run tasks')
+  $storages.each do |storage|
+    thread = $tasks_threads[storage.name]
+    if (!thread || !thread.alive?) && $tasks[storage.name] && $tasks[storage.name].size > 0
+      $log.debug("spawn TaskThread for #{storage.name}") 
+      $tasks_threads[storage.name] = TaskThread.new(storage.name)
+    end
+  end
+end

data/lib/daemon/base_thread.rb

@@ -0,0 +1,13 @@
+class BaseThread < Thread
+  def initialize(*args)
+    super(*args) do |*p|
+      begin
+        self.go(*p)
+      rescue Exception => e
+        error "#{self.class}: #{e.message}; #{e.backtrace.join(', ')}"
+      end
+      FC::DB.close
+      $log.debug("close #{self.class}")
+    end
+  end
+end