content_server 0.0.10 → 1.0.0

data/bin/backup_server

@@ -7,21 +7,33 @@
 
 # NOTE this file mainly is a copy of content_server
 
-begin
-  require 'yaml'
-  require 'params'
-  require 'run_in_background'
-  require 'content_server'
-rescue LoadError
-  require 'rubygems'
-  require 'yaml'
-  require 'params'
-  require 'run_in_background'
-  require 'content_server'
-end
-include BBFS
+require 'log'
+require 'params'
+require 'run_in_background'
+require 'content_server'
 
 Params.init ARGV
 Log.init
 
-RunInBackground.run { ContentServer.run_backup_server }
+Thread.abort_on_exception = true  # TODO (genadyp) should be treated globally? by param for example.
+
+retries = 0  # number of retries to run a server
+
+begin
+  RunInBackground.run { ContentServer.run_backup_server }
+rescue SystemExit, SignalException => exc
+  # TODO (genadyp) do we need to trap signals by types?
+  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.flush
+  exit
+rescue Exception => exc
+  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}")
+  if retries > 0
+    Log.info("Restarting (retries:#{retries}).\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  else
+    Log.info("Exiting...\nBacktrace:\n#{exc.backtrace.join("\n")}")
+    Log.flush
+  end
+  retries -= 1
+end while retries >= 0

data/bin/content_server

@@ -4,31 +4,33 @@
 # The files in those directories are indexed (calculating their SHA1).
 # Each unique content is backed up to the remote (backup) server.
 
-begin
-  print "1"
-  require 'yaml'
-  print "2"
-  require 'params'
-  print "3"
-  require 'run_in_background'
-  print "4"
-  require 'content_server'
-  print "5"
-rescue LoadError
-  print "6"
-  require 'rubygems'
-  print "7"
-  require 'yaml'
-  print "8"
-  require 'params'
-  print "9"
-  require 'run_in_background'
-  print "0"
-  require 'content_server'
-end
-include BBFS
+require 'log'
+require 'params'
+require 'run_in_background'
+require 'content_server'
 
 Params.init ARGV
 Log.init
 
-RunInBackground.run { ContentServer.run }
+Thread.abort_on_exception = true  # TODO (genadyp) should be treated globally? by param for example.
+
+retries = 0  # number of retries to run a server
+
+begin
+  RunInBackground.run { ContentServer.run_content_server }
+rescue SystemExit, SignalException => exc
+  # TODO (genadyp) do we need to trap signals by types?
+  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.flush
+  exit
+rescue Exception => exc
+  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}")
+  if retries > 0
+    Log.info("Restarting (retries:#{retries}).\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  else
+    Log.info("Exiting...\nBacktrace:\n#{exc.backtrace.join("\n")}")
+    Log.flush
+  end
+  retries -= 1
+end while retries >= 0

data/lib/content_server.rb

@@ -1,189 +1,13 @@
-require 'fileutils'
-require 'set'
-require 'thread'
+require 'content_server/content_server'
+require 'content_server/backup_server'
 
-require 'content_data'
-require 'content_server/content_receiver'
-require 'content_server/queue_indexer'
-require 'content_server/queue_copy'
-require 'content_server/remote_content'
-require 'file_indexing'
-require 'file_monitoring'
-require 'log'
-require 'networking/tcp'
-require 'params'
+module ContentServer
+  # Flags combined for content and backup server.
+  Params.path('local_content_data_path', '', 'ContentData file path.')
 
+  # Monitoring
+  Params.boolean('enable_monitoring', false, 'Whether to enable process monitoring or not.')
 
-
-# Content server. Monitors files, index local files, listen to backup server content,
-# copy changes and new files to backup server.
-module BBFS
-  module ContentServer
-    Params.string('remote_server', 'localhost', 'IP or DNS of backup server.')
-    Params.string('backup_username', nil, 'Backup server username.')
-    Params.string('backup_password', nil, 'Backup server password.')
-    Params.integer('backup_file_listening_port', 4444, 'Listening port in backup server for files')
-    Params.string('content_data_path', File.expand_path('~/.bbfs/var/content.data'),
-                     'ContentData file path.')
-    Params.string('monitoring_config_path', File.expand_path('~/.bbfs/etc/file_monitoring.yml'),
-                     'Configuration file for monitoring.')
-    Params.integer('remote_content_port', 3333, 'Default port for remote content copy.')
-    Params.integer('backup_check_delay', 5, 'Time between two content vs backup checks.')
-
-    def run
-      all_threads = []
-
-      # # # # # # # # # # # #
-      # Initialize/Start monitoring
-      monitoring_events = Queue.new
-      fm = FileMonitoring::FileMonitoring.new
-      fm.set_config_path(Params['monitoring_config_path'])
-      fm.set_event_queue(monitoring_events)
-      # Start monitoring and writing changes to queue
-      all_threads << Thread.new do
-        fm.monitor_files
-      end
-
-      # # # # # # # # # # # # # # # # # # # # # # # # #
-      # Initialize/Start backup server content data listener
-      #backup_server_content_data = nil
-      #backup_server_content_data_queue = Queue.new
-      #content_data_receiver = ContentDataReceiver.new(
-      #    backup_server_content_data_queue,
-      #    Params['remote_listening_port'])
-      # Start listening to backup server
-      #all_threads << Thread.new do
-      #  content_data_receiver.run
-      #end
-
-      # # # # # # # # # # # # # #
-      # Initialize/Start local indexer
-      local_server_content_data_queue = Queue.new
-      queue_indexer = QueueIndexer.new(monitoring_events,
-                                       local_server_content_data_queue,
-                                       Params['content_data_path'])
-      # Start indexing on demand and write changes to queue
-      all_threads << queue_indexer.run
-
-      # # # # # # # # # # # # # # # # # # # # # #
-      # Initialize/Start content data comparator
-      copy_files_events = Queue.new
-      local_dynamic_content_data = ContentData::DynamicContentData.new
-      all_threads << Thread.new do
-      #  backup_server_content_data = ContentData::ContentData.new
-      #  local_server_content_data = nil
-        while true do
-
-          # Note: This thread should be the only consumer of local_server_content_data_queue
-          Log.info 'Waiting on local server content data.'
-          local_server_content_data = local_server_content_data_queue.pop
-          local_dynamic_content_data.update(local_server_content_data)
-      #
-      #    # Note: This thread should be the only consumer of backup_server_content_data_queue
-      #    # Note: The server will wait in the first time on pop until backup sends it's content data
-      #    while backup_server_content_data_queue.size > 0
-      #      Log.info 'Waiting on backup server content data.'
-      #      backup_server_content_data = backup_server_content_data_queue.pop
-      #    end
-
-      #    Log.info 'Updating file copy queue.'
-      #    Log.debug1 "local_server_content_data #{local_server_content_data}."
-      #    Log.debug1 "backup_server_content_data #{backup_server_content_data}."
-      #    # Remove backup content data from local server
-      #    content_to_copy = ContentData::ContentData.remove(backup_server_content_data, local_server_content_data)
-      #    content_to_copy = local_server_content_data
-      #    # Add copy instruction in case content is not empty
-      #    Log.debug1 "Content to copy: #{content_to_copy}"
-      #    copy_files_events.push([:COPY_MESSAGE, content_to_copy]) unless content_to_copy.empty?
-        end
-      end
-
-      remote_content_client = RemoteContentClient.new(local_dynamic_content_data,
-                                                      Params['remote_content_port'])
-      all_threads << remote_content_client.tcp_thread
-
-      # # # # # # # # # # # # # # # #
-      # Start copying files on demand
-      copy_server = FileCopyServer.new(copy_files_events, Params['backup_file_listening_port'])
-      all_threads.concat(copy_server.run())
-
-      # Finalize server threads.
-      all_threads.each { |t| t.abort_on_exception = true }
-      all_threads.each { |t| t.join }
-      # Should never reach this line.
-    end
-    module_function :run
-
-    def run_backup_server
-      all_threads = []
-
-      # # # # # # # # # # # #
-      # Initialize/Start monitoring
-      monitoring_events = Queue.new
-      fm = FileMonitoring::FileMonitoring.new
-      fm.set_config_path(Params['monitoring_config_path'])
-      fm.set_event_queue(monitoring_events)
-      # Start monitoring and writing changes to queue
-      all_threads << Thread.new do
-        fm.monitor_files
-      end
-
-      # # # # # # # # # # # # # #
-      # Initialize/Start local indexer
-      local_server_content_data_queue = Queue.new
-      queue_indexer = QueueIndexer.new(monitoring_events,
-                                       local_server_content_data_queue,
-                                       Params['content_data_path'])
-      # Start indexing on demand and write changes to queue
-      all_threads << queue_indexer.run
-
-      # # # # # # # # # # # # # # # # # # # # # # # # # # #
-      # Initialize/Start backup server content data sender
-      dynamic_content_data = ContentData::DynamicContentData.new
-      #content_data_sender = ContentDataSender.new(
-      #    Params['remote_server'],
-      #    Params['remote_listening_port'])
-      # Start sending to backup server
-      all_threads << Thread.new do
-        while true do
-          Log.info 'Waiting on local server content data queue.'
-          cd = local_server_content_data_queue.pop
-      #    content_data_sender.send_content_data(cd)
-          dynamic_content_data.update(cd)
-        end
-      end
-
-      content_server_dynamic_content_data = ContentData::DynamicContentData.new
-      remote_content = ContentServer::RemoteContent.new(content_server_dynamic_content_data,
-                                                        Params['remote_server'],
-                                                        Params['remote_content_port'],
-                                                        Params['backup_destination_folder'])
-      all_threads.concat(remote_content.run())
-
-      file_copy_client = FileCopyClient.new(Params['remote_server'],
-                                               Params['backup_file_listening_port'],
-                                               dynamic_content_data)
-      all_threads.concat(file_copy_client.threads)
-
-      # Each
-      all_threads << Thread.new do
-        loop do
-          sleep(Params['backup_check_delay'])
-          local_cd = dynamic_content_data.last_content_data()
-          remote_cd = content_server_dynamic_content_data.last_content_data()
-          diff = ContentData::ContentData.remove(local_cd, remote_cd)
-          Log.debug2("Files to send? #{!diff.empty?}")
-          file_copy_client.request_copy(diff) unless diff.empty?
-        end
-      end
-
-
-      all_threads.each { |t| t.abort_on_exception = true }
-      all_threads.each { |t| t.join }
-      # Should never reach this line.
-    end
-    module_function :run_backup_server
-
-  end # module ContentServer
-end # module BBFS
-
+  # Handling thread exceptions.
+  Params.boolean('abort_on_exception', true, 'Any exception in any thread will abort the run.')
+end # module ContentServer

data/lib/content_server/backup_server.rb

@@ -0,0 +1,120 @@
+require 'fileutils'
+require 'set'
+require 'thread'
+
+require 'content_data'
+require 'content_server/content_receiver'
+require 'content_server/queue_indexer'
+require 'content_server/queue_copy'
+require 'content_server/remote_content'
+require 'file_indexing'
+require 'file_monitoring'
+require 'log'
+require 'networking/tcp'
+require 'params'
+require 'process_monitoring/thread_safe_hash'
+require 'process_monitoring/monitoring'
+require 'process_monitoring/monitoring_info'
+
+# Content server. Monitors files, index local files, listen to backup server content,
+# copy changes and new files to backup server.
+module ContentServer
+  # Backup server specific flags
+  Params.string('content_server_hostname', nil, 'IP or DNS of backup server.')
+  Params.integer('content_server_data_port', 3333, 'Port to copy content data from.')
+  Params.integer('content_server_files_port', 4444, 'Listening port in backup server for files')
+
+  Params.integer('backup_check_delay', 5, 'Delay in seconds between two content vs backup checks.')
+
+  def run_backup_server
+    Thread.abort_on_exception = true
+    all_threads = []
+
+    @process_variables = ThreadSafeHash::ThreadSafeHash.new
+    @process_variables.set('server_name', 'backup_server')
+
+    # # # # # # # # # # # #
+    # Initialize/Start monitoring
+    monitoring_events = Queue.new
+    fm = FileMonitoring::FileMonitoring.new
+    fm.set_event_queue(monitoring_events)
+    # Start monitoring and writing changes to queue
+    all_threads << Thread.new do
+      fm.monitor_files
+    end
+
+    # # # # # # # # # # # # # #
+    # Initialize/Start local indexer
+    local_server_content_data_queue = Queue.new
+    queue_indexer = QueueIndexer.new(monitoring_events,
+                                     local_server_content_data_queue,
+                                     Params['local_content_data_path'])
+    # Start indexing on demand and write changes to queue
+    all_threads << queue_indexer.run
+
+    # # # # # # # # # # # # # # # # # # # # # # # # # # #
+    # Initialize/Start backup server content data sender
+    dynamic_content_data = ContentData::DynamicContentData.new
+    #content_data_sender = ContentDataSender.new(
+    #    Params['remote_server'],
+    #    Params['remote_listening_port'])
+    # Start sending to backup server
+    all_threads << Thread.new do
+      while true do
+        Log.info 'Waiting on local server content data queue.'
+        cd = local_server_content_data_queue.pop
+        #    content_data_sender.send_content_data(cd)
+        dynamic_content_data.update(cd)
+      end
+    end
+
+    Params['backup_destination_folder'] = File.expand_path(Params['monitoring_paths'][0]['path'])
+    content_server_dynamic_content_data = ContentData::DynamicContentData.new
+    remote_content = ContentServer::RemoteContentClient.new(content_server_dynamic_content_data,
+                                                            Params['content_server_hostname'],
+                                                            Params['content_server_data_port'],
+                                                            Params['backup_destination_folder'])
+    all_threads.concat(remote_content.run())
+
+    file_copy_client = FileCopyClient.new(Params['content_server_hostname'],
+                                          Params['content_server_files_port'],
+                                          dynamic_content_data,
+                                          @process_variables)
+    all_threads.concat(file_copy_client.threads)
+
+    # Each
+    all_threads << Thread.new do
+      loop do
+        sleep(Params['backup_check_delay'])
+        local_cd = dynamic_content_data.last_content_data()
+        remote_cd = content_server_dynamic_content_data.last_content_data()
+        diff = ContentData::ContentData.remove(local_cd, remote_cd)
+        Log.debug2("Files to send? #{!diff.empty?}")
+        #file_copy_client.request_copy(diff) unless diff.empty?
+        if !diff.empty?
+          Log.info('Backup and remote contents need a sync:')
+          Log.info("Backup content:\n#{local_cd}")
+          Log.info("Remote content:\n#{remote_cd}")
+          Log.info("Missing contents:\n#{diff}")
+          Log.info('Requesting a copy')
+          file_copy_client.request_copy(diff)
+        end
+      end
+    end
+
+    if Params['enable_monitoring']
+      mon = Monitoring::Monitoring.new(@process_variables)
+      Log.add_consumer(mon)
+      all_threads << mon.thread
+      monitoring_info = MonitoringInfo::MonitoringInfo.new(@process_variables)
+    end
+
+    all_threads.each { |t| t.abort_on_exception = true }
+    all_threads.each { |t| t.join }
+    # Should never reach this line.
+  end
+  module_function :run_backup_server
+
+end # module ContentServer
+
+

data/lib/content_server/content_receiver.rb

@@ -2,61 +2,60 @@ require 'log'
 require 'params'
 require 'socket'
 
-module BBFS
-  module ContentServer
+module ContentServer
 
-    class ContentDataReceiver
-      def initialize queue, port
-        @queue = queue
-        @port = port
-      end
+  class ContentDataReceiver
+    def initialize queue, port
+      @queue = queue
+      @port = port
+    end
 
-      def run
-        Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
-          while size_of_data = sock.read(4)
-            size_of_data = size_of_data.unpack("l")[0]
-            Log.debug3 "Size of data: #{size_of_data}"
-            data = sock.read(size_of_data)
-            #Log.debug3 "Data received: #{data}"
-            unmarshaled_data = Marshal.load(data)
-            #Log.debug3 "Unmarshaled data: #{unmarshaled_data}"
-            @queue.push unmarshaled_data
-            Log.debug3 "Socket closed? #{sock.closed?}."
-            break if sock.closed?
-            Log.debug1 'Waiting on sock.read'
-          end
-          Log.debug1 'Exited, socket closed or read returned nil.'
+    def run
+      Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
+        while size_of_data = sock.read(4)
+          size_of_data = size_of_data.unpack("l")[0]
+          Log.debug3 "Size of data: #{size_of_data}"
+          data = sock.read(size_of_data)
+          #Log.debug3 "Data received: #{data}"
+          unmarshaled_data = Marshal.load(data)
+          #Log.debug3 "Unmarshaled data: #{unmarshaled_data}"
+          @queue.push unmarshaled_data
+          Log.debug3 "Socket closed? #{sock.closed?}."
+          break if sock.closed?
+          Log.debug1 'Waiting on sock.read'
         end
+        Log.debug1 'Exited, socket closed or read returned nil.'
       end
     end
+  end
 
-    class ContentDataSender
+  class ContentDataSender
 
-      def initialize host, port
-        @host = host
-        @port = port
-        open_socket
-      end
+    def initialize host, port
+      @host = host
+      @port = port
+      open_socket
+    end
 
-      def open_socket
-        Log.debug1 "Connecting to content server #{@host}:#{@port}."
-        @tcp_socket = TCPSocket.new(@host, @port)
-      end
+    def open_socket
+      Log.debug1 "Connecting to content server #{@host}:#{@port}."
+      @tcp_socket = TCPSocket.new(@host, @port)
+    end
 
-      def send_content_data content_data
-        open_socket if @tcp_socket.closed?
-        #Log.debug3 "Data to send: #{content_data}"
-        marshal_data = Marshal.dump(content_data)
-        Log.debug3 "Marshaled size: #{marshal_data.length}."
-        data_size = [marshal_data.length].pack("l")
-        #Log.debug3 "Marshaled data: #{marshal_data}."
-        if data_size.nil? || marshal_data.nil?
-          Log.debug3 'Send data is nil!!!!!!!!'
-        end
-        @tcp_socket.write data_size
-        @tcp_socket.write marshal_data
+    def send_content_data content_data
+      open_socket if @tcp_socket.closed?
+      #Log.debug3 "Data to send: #{content_data}"
+      marshal_data = Marshal.dump(content_data)
+      Log.debug3 "Marshaled size: #{marshal_data.length}."
+      data_size = [marshal_data.length].pack("l")
+      #Log.debug3 "Marshaled data: #{marshal_data}."
+      if data_size.nil? || marshal_data.nil?
+        Log.debug3 'Send data is nil!!!!!!!!'
       end
+      @tcp_socket.write data_size
+      @tcp_socket.write marshal_data
     end
-
   end
+
 end
+