filecluster 0.5.14 → 0.5.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df71ee0cd0c4068b96655ad85da5a84de2aa2044
-  data.tar.gz: f7787796c064db4b2e7f373d94422cb0af708eca
+  metadata.gz: 6dc0301108025f48875d13f484a6dda7d1d9be97
+  data.tar.gz: 35987690b4cd3f40cf439f1f63e6c78bfa6b98fb
 SHA512:
-  metadata.gz: 053ef243b2f1c07f955207af615764909245cdcf5d7b35d76c34f267e3f80a9572f1465208cd2d2bfd34e2095103988609cbba66fb3f8c9d1ad29b27443b5f96
-  data.tar.gz: 52e8e8ee2cf22ede6b1f31448d3b7233ea3a883938c8a4ec4d27922b08dc124a76b2f74668e146c4cc6d51e45060dd0b563071375e5afc6fc32b0077374855da
+  metadata.gz: 8953e38a31a07cddf4912fa30abc6f3897703ca09ff318986ba6158901c434dfca5331dd5ff2a47b6a0f94f5f46008d000e9be15ee0536000bf3bdbcac20f78b
+  data.tar.gz: bde827cae587aaeda3b7d8d1259dd1033c2d55549afb7e3faa78ab89d4ce500bb17136a4bf01570320b8b3f3796d8bd095d9b7575359a9154c3cc91f0e8c38c3

data/.gitignore

@@ -17,3 +17,6 @@ test/tmp
 test/version_tmp
 tmp
 *.yml
+.byebug_history
+/.ssh/*
+!/.ssh/.keep

data/Dockerfile

@@ -0,0 +1,40 @@
+FROM ubuntu:16.04
+
+RUN  apt-get update \
+  && apt-get install -y wget \
+  build-essential \
+  zlib1g-dev \
+  openssl \
+  libssl-dev \
+  git \
+  libreadline-dev \
+  libmysqlclient-dev \
+  tzdata \
+  super \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN wget http://ftp.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz \
+  && tar -xzvf ruby-2.3.3.tar.gz \
+  && rm ruby-2.3.3.tar.gz \
+  && cd ruby-2.3.3/ \
+  && ./configure --with-openssl-dir=/usr/bin \
+  && make \
+  && make install \
+  && cd .. \
+  && rm -rf ruby-2.3.3 \
+  && gem install bundler
+
+RUN mkdir -p /app
+WORKDIR /app
+
+COPY ./ /app
+
+RUN groupadd --gid 1000 filecluster \
+    && useradd --uid 1000 --gid 1000 filecluster --shell /bin/bash
+
+
+COPY ./entrypoint.sh /usr/local/bin/
+RUN bundle install --jobs 20 --retry 6
+
+ENTRYPOINT  ["/usr/local/bin/entrypoint.sh"]
+CMD ["/app/bin/fc-daemon", "-l", "debug"]

data/Gemfile

@@ -2,3 +2,4 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in filecluster.gemspec
 gemspec
+

data/README_RU.md

@@ -0,0 +1,32 @@
+FileCluster
+
+Набор скриптов для управления контентом по верх файловой системы
+Уровень регистрации в базе - файл или папка
+
+
+Принцип работы
+
+Информация о файлах/папках записывается в базу данных (mysql)
+Демон на основе записей в базе данных приводит соответствие файлов списку зарегистрированных файлов/папок
+
+
+Компоненты
+
+filecluster - deamon/manager (ruby)
+mysql -  мета информация
+sshd + rsync  - транспорт
+
+
+
+Быстрый запуск тестов
+
+первый запуск mysql -
+
+docker-compose run --rm filecluster-db
+
+Окончание инициализации можно будет увидеть по сообщению о готовности к работе mysql
+CTRL+C
+
+Запуск тестов
+
+docker-compose  run --rm filecluser1 rake

data/bin/fc-daemon

@@ -90,9 +90,12 @@ while true do
     storages_check
     update_tasks
     run_tasks
+    autosync
   end
   $log.debug('sleep')
-  sleep FC::Var.get('daemon_cycle_time', 30).to_i
+  sleep_time = FC::Var.get('daemon_cycle_time', 30).to_f
+  sleep_time = 0.3 if sleep_time < 0.3
+  sleep sleep_time
   alive_time = Time.new.to_i - start_time
   if !$exit_signal && alive_time > FC::Var.get('daemon_restart_period', 86400).to_i
     $log.info("Self restart, #{alive_time} seconds up")

data/bin/fc-manage

@@ -72,6 +72,14 @@ Command:
    list                show all limits
    change <host>       change current copy speed limit for host   
    add                 add copy speed limit for host
+}],
+  'autosync' => [
+    'show and change autosync intervals for FC hosts',
+    %q{Usage: fc-manage autosync <command>
+Command:
+  list                show all intervals
+  change <host>       change sync interval for host
+  add                 add non default sync interval for host
 }],
   'item' => [
     'show and manage items', 

data/bin/fc-setup-db

@@ -52,9 +52,9 @@ if s == "y" || s == "yes"
   end
   unless default_db_config
     print "Save to config..   "
-    options.select!{|key, val| descriptions[key][:save]}
+    config = options.select { |key, _| descriptions[key][:save] }
     File.open(File.expand_path(File.dirname(__FILE__))+'/db.yml', 'w') do |f|
-      f.write(options.to_yaml)
+      f.write(config.to_yaml)
     end
     puts "ok"
   end

data/docker-compose-mysql56.yaml

@@ -0,0 +1,35 @@
+version: '3.5'
+
+services:
+  filecluster-db:
+    image: mysql:5.6
+    volumes:
+      - filecluster_db56:/var/lib/mysql
+    env_file:
+      - ./docker/development.env
+
+  filecluster1:
+    build: ./
+    depends_on:
+      - filecluster-db
+      - filecluster1-ssh
+    env_file:
+      - ./docker/development.env
+
+    volumes:
+      - filetest_1:/tmp/
+      - ./:/app/
+      - ./.ssh:/home/filecluster/.ssh
+
+  filecluster1-ssh:
+    image: asigatchov/ubuntu16-sshd
+    volumes:
+      - filetest_1:/tmp/
+      - ./.ssh:/home/filecluster/.ssh
+
+      
+volumes:
+  filecluster_db56:
+    driver: local
+  filetest_1:
+    driver: local

data/docker-compose.yaml

@@ -0,0 +1,35 @@
+version: '3.5'
+
+services:
+  filecluster-db:
+    image: mysql:5.7.21
+    volumes:
+      - filecluster_db:/var/lib/mysql
+    env_file:
+      - ./docker/development.env
+
+  filecluster1:
+    build: ./
+    depends_on:
+      - filecluster-db
+      - filecluster1-ssh
+    volumes:
+      - filetest_1:/tmp/
+      - ./:/app/
+      - ./.ssh:/home/filecluster/.ssh
+    hostname: filecluster1
+    env_file:
+      - ./docker/development.env
+
+
+  filecluster1-ssh:
+    image: asigatchov/ubuntu16-sshd
+    volumes:
+      - filetest_1:/tmp/
+      - ./.ssh:/home/filecluster/.ssh
+
+volumes:
+  filecluster_db:
+    driver: local
+  filetest_1:
+    driver: local

data/docker/Dockerfile.sshd

@@ -0,0 +1,17 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y openssh-server rsync
+RUN mkdir /var/run/sshd
+
+#RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+ENV NOTVISIBLE "in users profile"
+RUN echo "export VISIBLE=now" >> /etc/profile
+RUN groupadd --gid 1000 filecluster \
+    && useradd --uid 1000 --gid 1000 filecluster --shell /bin/bash
+
+EXPOSE 22
+CMD ["/usr/sbin/sshd", "-D"]

data/docker/development.env

@@ -0,0 +1,7 @@
+MYSQL_ALLOW_EMPTY_PASSWORD=1
+MYSQL_USER=filecluster
+MYSQL_DATABASE =filecluser
+MYSQL_PASSWORD=clusterfile
+MYSQL_HOST=filecluster-db
+SSH_HOST=filecluster1-ssh
+FILECLUSTE_ENV=development

data/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+if [ ! -f /app/bin/db.yml ] ; then
+  ./bin/fc-setup-db -h $MYSQL_HOST -u $MYSQL_USER -p $MYSQL_PASSWORD -d $MYSQL_DATABASE -f -i -m
+fi
+
+if [ ! -f /home/filecluster/.ssh/id_rsa ]; then
+  mkdir /home/filecluster/.ssh -p
+  chown filecluster:filecluster /home/filecluster/.ssh
+  chmod 0700 /home/filecluster/.ssh
+  ssh-keygen -t rsa -b 2048 -N "" -C "filecluster key" -f /home/filecluster/.ssh/id_rsa
+  cp /home/filecluster/.ssh/{id_rsa.pub,authorized_keys}
+  chown -R filecluster:filecluster /home/filecluster
+  echo "ID_RSA - generate"
+fi
+
+export HOME=/home/filecluster/
+exec  setuid 1000 "$@"

data/filecluster.gemspec

@@ -22,4 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake"
   gem.add_development_dependency "shoulda-context"
   gem.add_development_dependency "mocha", ">= 0.13.3"
+  gem.add_development_dependency "byebug"
+  gem.add_development_dependency "rubocop"
 end

data/lib/autosync.rb

@@ -0,0 +1,149 @@
+require 'iostat'
+
+class Autosync
+  attr_accessor :files_to_delete, :items_to_delete
+  def initialize(storage, dry_run = false)
+    @dry_run = dry_run
+    @start_time = Time.now.to_i - 3600
+    @storage = storage
+    @files_to_delete = []
+    @items_to_delete = []
+    @removed_files_size = 0
+    @io_stat = Iostat.new(@storage.path)
+  end
+
+  def self.error(msg, options = {})
+    $log.error(msg) if $log
+    FC::Error.new(options.merge(:host => FC::Storage.curr_host, :message => msg)).save
+  end
+
+  def run
+    @db_struct = fill_db
+    $log.debug("Autosync: Scanning disk #{@storage.name} (#{@storage.path})") if $log
+    scan_disk(@db_struct, '')
+    return if $exit_signal
+    $log.debug("Autosync: Scanning DB items #{@storage.name}") if $log
+    scan_db(@db_struct, '')
+    return if $exit_signal
+    delete_diffs unless @dry_run
+  ensure
+    @io_stat.stop
+  end
+
+  def relax_drive
+    sleep 1 while @io_stat.util > 50
+  end
+
+  def fill_db
+    $log.debug("Autosync: Reading DB items for #{@storage.name} ...") if $log
+
+    db_struct = {}
+    last_item_storage_id = 0
+    items_count = 0
+    loop do
+      items = FC::DB.connect.query(%(
+        SELECT its.id, itm.name
+        FROM #{FC::Item.table_name} itm
+        JOIN #{FC::ItemStorage.table_name} its ON its.item_id = itm.id
+        WHERE its.storage_name = '#{@storage.name}'
+        AND its.status = 'ready'
+        AND its.id > #{last_item_storage_id}
+        ORDER BY its.id
+        LIMIT 10000
+      ), cache_rows: false, symbolize_keys: true)
+      break if $exit_signal
+
+      # make tree structure with array of values (items) on leafs
+      items.each do |i|
+        items_count += 1
+        last_item_storage_id = i[:id]
+        ref = db_struct
+        path = i[:name].split('/')
+        last_idx = path.size - 1
+        path.each_with_index do |part, idx|
+          if idx == last_idx
+            ref[part] = [false, i[:id]]
+          else
+            ref[part] ||= {}
+            ref = ref[part]
+          end
+        end
+      end
+      break unless items.size == 10_000
+    end
+    $log.debug("Autosync: Reading DB items for #{@storage.name} done. Items: #{items_count}") if $log
+    db_struct
+  end
+
+  def scan_disk(db_path, relative_path)
+    return if $exit_signal
+    sleep 0.001
+    relax_drive
+    Dir.glob("#{@storage.path}#{relative_path}*").each do |disk_entry|
+      next if disk_entry == "#{@storage.path}healthcheck"
+      db_item = db_path[disk_entry.split('/').last]
+      case
+      when db_item.is_a?(Array) # tree leaf
+        db_item[0] = true # mark db_item as exists on disk
+      when db_item.is_a?(Hash) && File.directory?(disk_entry) # tree node
+        scan_disk(db_item, "#{disk_entry[@storage.path.size..-1]}/")
+      else # not found in db
+        mtime = File.stat(disk_entry).mtime.to_i rescue Time.now.to_i
+        @files_to_delete << disk_entry if @start_time > mtime # older than 1 hour
+      end
+    end
+  end
+
+  def scan_db(db_item, node_path)
+    return if $exit_signal
+    db_item.each do |item_name, item_data|
+      if item_data.is_a?(Array) # tree leaf
+        @items_to_delete << item_data[1] unless item_data[0]
+      else # tree node
+        scan_db(item_data, "#{node_path}#{item_name}/")
+      end
+    end
+  end
+
+  def delete_disk_entry(entry)
+    return false if $exit_signal
+    return true unless File.exist?(entry)
+    remove = true
+    stat = File.stat(entry) rescue nil
+    if File.directory?(entry)
+      Dir.glob("#{entry}/*").each do |sub_entry|
+        relax_drive
+        remove = false unless delete_disk_entry(sub_entry)
+      end
+    else
+      mtime = stat ? stat.mtime.to_i : Time.now.to_i
+      remove = @start_time > mtime
+    end
+    if remove
+      @removed_files_size += stat.size if stat
+      $log.debug("deleting disk entry #{entry}") if $log
+      FileUtils.rm_rf(entry)
+    end
+    remove
+  end
+
+  def delete_diffs
+    $log.debug("Removing #{@files_to_delete.size} disk entries") if $log
+    @files_to_delete.each do |f|
+      break if $exit_signal
+      delete_disk_entry(f)
+    end
+    return if $exit_signal
+    self.class.error("Autosync removed #{@files_to_delete.size} files/dirs from #{@storage.name}. Size: #{@removed_files_size} bytes") if @removed_files_size > 0
+    $log.debug("Removing items #{@items_to_delete.size} from DB for #{@storage.name}") if $log
+    counter = 0
+    @items_to_delete.each do |item_storage_id|
+      its = FC::ItemStorage.where('id = ?', item_storage_id).first
+      next unless its
+      its.status = 'error'
+      its.save
+      self.class.error("item does not exist on storage #{@storage.name}", item_storage_id: item_storage_id.to_i) rescue nil
+      sleep 10 if (counter += 1) % 1000 == 0
+    end
+  end
+end

data/lib/daemon.rb

@@ -6,6 +6,7 @@ require "daemon/run_tasks_thread"
 require "daemon/update_tasks_thread"
 require "daemon/copy_task_thread"
 require "daemon/delete_task_thread"
+require "daemon/autosync_thread"
 
 def error(msg, options = {})
   $log.error(msg)
@@ -20,7 +21,8 @@ end
 
 def run_global_daemon
   $log.debug('Run global daemon check')
-  timeout = FC::Var.get('daemon_global_wait_time', 120).to_i
+  timeout = FC::Var.get('daemon_global_wait_time', 120).to_f
+  timeout = 0.3 if timeout < 0.3
   r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   if !r || r['curr_time'].to_i - r['time'].to_i > timeout
     $log.debug('Set global daemon host to current')
@@ -72,3 +74,17 @@ def run_tasks
     $run_tasks_thread = RunTasksThread.new
   end
 end
+
+def autosync
+  if !$autosync_thread || !$autosync_thread.alive?
+    intervals = FC::Var.get_autosync
+    storage_interval = intervals[FC::Storage.curr_host] || intervals['all']
+    return if storage_interval.zero? # do not run aytosync
+    storages = $storages.select do |s|
+      s.autosync_at.to_i + storage_interval < Time.now.to_i
+    end
+    return unless storages.any?
+    $log.debug("spawn AutosyncThread for storages #{storages.map(&:name).join(', ')}")
+    $autosync_thread = AutosyncThread.new(storages)
+  end
+end