content_server 1.1.0 → 1.2.0

data/lib/content_server/queue_copy.rb

@@ -2,16 +2,17 @@ require 'thread'
 
 require 'content_server/file_streamer'
 require 'file_indexing/index_agent'
-require 'content_server/globals'
+require 'content_server/server'
 require 'log'
 require 'networking/tcp'
 require 'params'
 
-
 module ContentServer
   Params.integer('ack_timeout', 5, 'Timeout of ack from backup server in seconds.')
-
+  Params.integer('local_timeout', 60, 'Timeout of content being under copy process.')
+  Params.integer('max_copy_streams', 5, 'max contents being copied at once.')
   # Copy message types.
+  :SEND_COPY_MESSAGE
   :ACK_MESSAGE
   :COPY_MESSAGE
   :SEND_COPY_MESSAGE
@@ -20,6 +21,119 @@ module ContentServer
   :ABORT_COPY # Asks the sender to abort file copy.
   :RESET_RESUME_COPY # Sends the stream sender to resend chunk or resume from different offset.
 
+  class FileCopyManager
+    def initialize(copy_input_queue, file_streamer)
+      @copy_input_queue = copy_input_queue
+      @file_streamer = file_streamer
+      @max_contents_under_copy = Params['max_copy_streams']
+      @contents_under_copy = {}
+      @contents_to_copy = {}
+      @contents_to_copy_queue = Queue.new
+      @keeper = Mutex.new
+      @clean_time_out_thread = clean_time_out_thread
+    end
+
+    # Add content to copy process. If already in copy process or waiting for copy then skip.
+    # If no open places for copy then put in waiting list
+    def add_content(checksum, path)
+      Log.debug2("Try to add content:#{checksum} to copy waiting list")
+      @keeper.synchronize{
+        # if content is being copied or waiting then skip it
+        if !@contents_under_copy[checksum]
+          if !@contents_to_copy[checksum]
+            if @contents_under_copy.size < @max_contents_under_copy
+              @contents_under_copy[checksum] = [path, false, Time.now]
+              $process_vars.set('contents under copy', @contents_under_copy.size)
+              @copy_input_queue.push([:SEND_ACK_MESSAGE, checksum])
+              $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
+            else
+              # no place in copy streams. Add to waiting list
+              Log.debug2("add content:#{checksum} to copy waiting list")
+              @contents_to_copy[checksum] = true  # replace with a set
+              @contents_to_copy_queue.push([checksum, path])
+              $process_vars.set('contents to copy queue', @contents_to_copy_queue.size)
+            end
+          else
+            Log.debug2("content:#{checksum} already in waiting list. skipping.")
+          end
+        else
+          Log.debug2("content:#{checksum} is being copied. skipping.")
+        end
+      }
+    end
+
+    def receive_ack(checksum)
+      @keeper.synchronize{
+        content_record = @contents_under_copy[checksum]
+        if content_record
+          if !content_record[1]
+            path = content_record[0]
+            Log.debug1("Streaming to backup server. content: #{checksum} path:#{path}.")
+            @file_streamer.start_streaming(checksum, path)
+            # updating Ack
+            content_record[1] = true
+          else
+            Log.warning("File already received ack: #{checksum}")
+          end
+        else
+          Log.warning("File was aborted or copied: #{checksum}")
+        end
+      }
+    end
+
+    def remove_content(checksum)
+      @keeper.synchronize{
+        Log.debug3("removing checksum:#{checksum} from contents under copy")
+        @contents_under_copy.delete(checksum)
+        $process_vars.set('contents under copy', @contents_under_copy.size)
+        #1 place is became available. Put another file in copy process if waiting
+        if (@contents_to_copy_queue.size > 0)
+          new_content = @contents_to_copy_queue.pop
+          $process_vars.set('contents to copy queue', @contents_to_copy_queue.size)
+          @contents_to_copy.delete(new_content[0])
+          @contents_under_copy[new_content[0]] = [new_content[1], false, Time.now]
+          $process_vars.set('contents under copy', @contents_under_copy.size)
+          @copy_input_queue.push([:SEND_ACK_MESSAGE, new_content[0]])
+          $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
+        end
+      }
+    end
+
+    # clean timed out contents
+    def clean_time_out_thread
+      @thread = Thread.new do
+        loop {
+          sleep 10
+          @keeper.synchronize{
+            # clean timed out contents
+            time_now = Time.now
+            new_contents_under_copy = {}
+            @contents_under_copy.each_key { |checksum|
+              if time_now - @contents_under_copy[checksum][2] > Params['local_timeout']
+                @contents_under_copy.delete(checksum)
+                $process_vars.set('contents under copy', @contents_under_copy.size)
+                Log.warning("Content:#{checksum} has timed out on copy process")
+                @file_streamer.abort_streaming(checksum)
+                if (@contents_to_copy_queue.size > 0)
+                  new_content = @contents_to_copy_queue.pop
+                  $process_vars.set('contents to copy queue', @contents_to_copy_queue.size)
+                  @contents_to_copy.delete(new_content[0])
+                  new_contents_under_copy[new_content[0]] = new_content[1]
+                end
+              end
+            }
+            new_contents_under_copy.each_key { |checksum|
+              @contents_under_copy[checksum.clone] = [new_contents_under_copy[checksum].clone, false, time_now]
+              $process_vars.set('contents under copy', @contents_under_copy.size)
+              @copy_input_queue.push([:SEND_ACK_MESSAGE, checksum])
+              $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
+            }
+          }
+        }
+      end
+    end
+  end
+
   # Simple copier, gets inputs events (files to copy), requests ack from backup to copy
   # then copies one file.
   class FileCopyServer
@@ -33,16 +147,19 @@ module ContentServer
       @copy_prepare = {}
       @file_streamer = FileStreamer.new(method(:send_chunk))
       Log.debug3("initialize FileCopyServer on port:#{port}")
+      @file_copy_manager = FileCopyManager.new(@copy_input_queue, @file_streamer)
     end
 
     def send_chunk(*arg)
       @copy_input_queue.push([:COPY_CHUNK, arg])
+      $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
     end
 
     def receive_message(addr_info, message)
       # Add ack message to copy queue.
       Log.debug2("Content server Copy message received: #{message}")
       @copy_input_queue.push(message)
+      $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
     end
 
     def run()
@@ -51,44 +168,38 @@ module ContentServer
       threads << Thread.new do
         while true do
           Log.debug1 'Waiting on copy files events.'
-          message_type, message_content = @copy_input_queue.pop
+          (message_type, message_content) = @copy_input_queue.pop
+          $process_vars.set('Copy File Queue Size', @copy_input_queue.size)
+          Log.debug1("Content copy message:#{[message_type, message_content]}")
 
-          if message_type == :COPY_MESSAGE
-            Log.debug1 "Copy files event: #{message_content}"
-            # Prepare source,dest map for copy.
+          if message_type == :SEND_ACK_MESSAGE
+            Log.debug1("Sending ack for: #{message_content}")
+            @backup_tcp.send_obj([:ACK_MESSAGE, [message_content, Time.now.to_i]])
+          elsif message_type == :COPY_MESSAGE
             message_content.each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path|
-              if !@copy_prepare.key?(checksum) || !@copy_prepare[checksum][1]
-                @copy_prepare[checksum] = [path, false]
-                Log.debug1("Sending ack for: #{checksum}")
-                @backup_tcp.send_obj([:ACK_MESSAGE, [checksum, Time.now.to_i]])
-              end
+              @file_copy_manager.add_content(checksum, path)
             }
           elsif message_type == :ACK_MESSAGE
             # Received ack from backup, copy file if all is good.
             # The timestamp is of local content server! not backup server!
             timestamp, ack, checksum = message_content
-
             Log.debug1("Ack (#{ack}) received for content: #{checksum}, timestamp: #{timestamp} " \
                        "now: #{Time.now.to_i}")
 
             # Copy file if ack (does not exists on backup and not too much time passed)
             if ack
               if (Time.now.to_i - timestamp < Params['ack_timeout'])
-                if !@copy_prepare.key?(checksum) || @copy_prepare[checksum][1]
-                  Log.warning("File was aborted, copied, or started copy just now: #{checksum}")
-                else
-                  path = @copy_prepare[checksum][0]
-                  Log.info "Streaming to backup server. content: #{checksum} path:#{path}."
-                  @file_streamer.start_streaming(checksum, path)
-                  # Ack received, setting prepare to true
-                  @copy_prepare[checksum][1] = true
-                end
+                @file_copy_manager.receive_ack(checksum)
               else
                 Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
                            "timeout: #{Params['ack_timeout']}")
+                # remove only content under copy
+                @file_copy_manager.remove_content(checksum)
               end
             else
               Log.debug1('Ack is false');
+              # remove content under copy and content in waiting list
+              @file_copy_manager.remove_content(checksum)
             end
           elsif message_type == :COPY_CHUNK_FROM_REMOTE
             checksum = message_content
@@ -97,21 +208,20 @@ module ContentServer
             # We open the message here for printing info and deleting copy_prepare!
             file_checksum, offset, file_size, content, content_checksum = message_content
             Log.debug1("Send chunk for file #{file_checksum}, offset: #{offset} " \
-                         "filesize: #{file_size}.")
+                         "filesize: #{file_size}, checksum:#{content_checksum}")
             # Blocking send.
             @backup_tcp.send_obj([:COPY_CHUNK, message_content])
             if content.nil? and content_checksum.nil?
-              @copy_prepare.delete(file_checksum)
+              # Sending enf of file and removing file from list
+              @file_copy_manager.remove_content(file_checksum)
             end
           elsif message_type == :ABORT_COPY
             Log.debug1("Aborting file copy: #{message_content}")
-            if @copy_prepare.key?(message_content)
-              Log.debug1("Aborting: #{@copy_prepare[message_content][0]}")
-              @copy_prepare.delete(message_content)
-            end
             @file_streamer.abort_streaming(message_content)
+            # remove only content under copy
+            @file_copy_manager.remove_content(message_content)
           elsif message_type == :RESET_RESUME_COPY
-            file_checksum, new_offset = message_content
+            (file_checksum, new_offset) = message_content
             Log.debug1("Resetting/Resuming file (#{file_checksum}) copy to #{new_offset}")
             @file_streamer.reset_streaming(file_checksum, new_offset)
           else
@@ -124,20 +234,17 @@ module ContentServer
   end  # class QueueCopy
 
   class FileCopyClient
-    def initialize(host, port, dynamic_content_data)
+    def initialize(host, port)
       @local_queue = Queue.new
-      @dynamic_content_data = dynamic_content_data
       @tcp_client = Networking::TCPClient.new(host, port, method(:handle_message))
       @file_receiver = FileReceiver.new(method(:done_copy),
                                         method(:abort_copy),
                                         method(:reset_copy))
       @local_thread = Thread.new do
         loop do
-          pop_queue = @local_queue.pop
-          if Params['enable_monitoring']
-            ::ContentServer::Globals.process_vars.set('File Copy Client queue', @local_queue.size)
-          end
-          handle(@local_queue.pop)
+          pop_data = @local_queue.pop
+          $process_vars.set('File Copy Client queue', @local_queue.size)
+          handle(pop_data)
         end
       end
       @local_thread.abort_on_exception = true
@@ -163,42 +270,40 @@ module ContentServer
     end
 
     def done_copy(local_file_checksum, local_path)
-      if Params['enable_monitoring']
-        ::ContentServer::Globals.process_vars.inc('num_files_received')
-      end
+      $process_vars.inc('num_files_received')
       Log.debug1("Done copy file: #{local_path}, #{local_file_checksum}")
     end
 
     def handle_message(message)
       Log.debug3('QueueFileReceiver handle message')
       @local_queue.push(message)
-      if Params['enable_monitoring']
-        ::ContentServer::Globals.process_vars.set('File Copy Client queue', @local_queue.size)
-      end
+      $process_vars.set('File Copy Client queue', @local_queue.size)
     end
 
     # This is a function which receives the messages (file or ack) and return answer in case
     # of ack. Note that it is being executed from the class thread only!
     def handle(message)
       message_type, message_content = message
+      Log.debug1("backup copy message: Type #{message_type}.  message: #{message_content}")
       if message_type == :SEND_COPY_MESSAGE
-        Log.debug1("Requesting files to copy.")
-        Log.debug2("Files requested: #{message_content.to_s}")
         bytes_written = @tcp_client.send_obj([:COPY_MESSAGE, message_content])
         Log.debug2("Sending copy message succeeded? bytes_written: #{bytes_written}.")
       elsif message_type == :COPY_CHUNK
-        Log.debug1('Chunk received.')
         if @file_receiver.receive_chunk(*message_content)
           file_checksum, offset, file_size, content, content_checksum = message_content
           @tcp_client.send_obj([:COPY_CHUNK_FROM_REMOTE, file_checksum])
+        else
+          file_checksum, offset, file_size, content, content_checksum = message_content
+          Log.error("receive_chunk failed for chunk checksum:#{content_checksum}")
         end
       elsif message_type == :ACK_MESSAGE
         checksum, timestamp = message_content
-        # Here we should check file existence
-        Log.info("Returning ack for content: #{checksum}, timestamp: #{timestamp}")
-        Log.debug1("Ack: #{!@dynamic_content_data.exists?(checksum)}")
+        # check if checksum exists in final destination
+        dest_path = FileReceiver.destination_filename(Params['backup_destination_folder'][0]['path'], checksum)
+        need_to_copy = !File.exists?(dest_path)
+        Log.debug1("Returning ack for content:'#{checksum}' timestamp:'#{timestamp}' Ack:'#{need_to_copy}'")
         @tcp_client.send_obj([:ACK_MESSAGE, [timestamp,
-                                             !@dynamic_content_data.exists?(checksum),
+                                             need_to_copy,
                                              checksum]])
       elsif message_type == :ABORT_COPY
         @tcp_client.send_obj([:ABORT_COPY, message_content])

data/lib/content_server/queue_indexer.rb

@@ -1,6 +1,6 @@
 require 'file_indexing/index_agent'
 require 'file_indexing/indexer_patterns'
-require 'content_server/globals'
+require 'content_server/server'
 require 'log'
 require 'params'
 
@@ -10,9 +10,8 @@ module ContentServer
   # content data updates into output queue.
   class QueueIndexer
 
-    def initialize(input_queue, local_dynamic_content_data)
+    def initialize(input_queue)
       @input_queue = input_queue
-      @local_dynamic_content_data = local_dynamic_content_data
     end
 
     # index files and add to copy queue
@@ -24,20 +23,25 @@ module ContentServer
         while true do
           Log.debug1 'Waiting on index input queue.'
           (state, is_dir, path, mtime, size) = @input_queue.pop
+          $process_vars.set('monitor to index queue size', @input_queue.size)
           Log.debug1 "index event: state:#{state}, dir?#{is_dir}, path:#{path}, mtime:#{mtime}, size:#{size}."
           if state == FileMonitoring::FileStatEnum::STABLE && !is_dir
             # Calculating checksum
-            instance_stats = @local_dynamic_content_data.stats_by_location([Params['local_server_name'], path])
+            instance_stats = nil  # definition
+            $local_content_data_lock.synchronize{
+              instance_stats = $local_content_data.stats_by_location([Params['local_server_name'], path])
+            }
             Log.debug1("instance !#{instance_stats}! mtime: #{mtime.to_i}, size: #{size}")
             if instance_stats.nil? || mtime.to_i != instance_stats[1] || size != instance_stats[0]
-              Log.info "Indexing file:'#{path}'."
+              Log.debug1 "Indexing file:'#{path}'."
               checksum = calc_SHA1(path)
-              if Params['enable_monitoring']
-                ::ContentServer::Globals.process_vars.inc('indexed_files')
-              end
+              $process_vars.inc('indexed_files')
+              $indexed_file_count += 1
               Log.debug1("Index info:checksum:#{checksum} size:#{size} time:#{mtime.to_i}")
               Log.debug1('Adding index to content data. put in queue for dynamic update.')
-              @local_dynamic_content_data.add_instance(checksum, size, Params['local_server_name'], path, mtime.to_i)
+              $local_content_data_lock.synchronize{
+                $local_content_data.add_instance(checksum, size, Params['local_server_name'], path, mtime.to_i)
+              }
             else
               Log.info("Skip file #{path} indexing (shallow check passed)")
             end
@@ -46,12 +50,16 @@ module ContentServer
             Log.debug2("NonExisting/Changed (file): #{path}")
             # Remove file but only when non-existing.
             Log.debug1("File to remove: #{path}")
-            @local_dynamic_content_data.remove_instance([Params['local_server_name'],path])
+            $local_content_data_lock.synchronize{
+              $local_content_data.remove_instance(Params['local_server_name'], path)
+            }
           elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
             Log.debug2("NonExisting/Changed (dir): #{path}")
             # Remove directory but only when non-existing.
             Log.debug1("Directory to remove: #{path}")
-            @local_dynamic_content_data.remove_directory(path, Params['local_server_name'])
+            $local_content_data_lock.synchronize{
+              $local_content_data.remove_directory(Params['local_server_name'], path)
+            }
           else
             Log.debug1("This case should not be handled: #{state}, #{is_dir}, #{path}.")
           end

data/lib/content_server/remote_content.rb

@@ -1,6 +1,5 @@
 require 'thread'
 
-require 'content_data/dynamic_content_data'
 require 'log'
 require 'networking/tcp'
 require 'params'
@@ -10,42 +9,51 @@ module ContentServer
   Params.integer('remote_content_fetch_timeout', 10, 'Remote content desired freshness in seconds.')
   Params.integer('remote_content_save_timeout', 60*60, 'Remote content force refresh in seconds.')
 
-  # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
+# Remote content Client
+  # Remote Content (client) - Periodically requests remote content (content server) from Remote Content Server.
+  # Stores locally the remote content data structure.
+  # Remote content client protocol based on time.
+  # Each n seconds it requests content from Remote Content Client connects to server (tcp/ip) and waits for request.
+  # remote_content_save_timeout is pre defined in config-backup_server.yml
   class RemoteContentClient
-    def initialize(dynamic_content_data, host, port, local_backup_folder)
-      @dynamic_content_data = dynamic_content_data
+    # initializes class variables
+    # @param host [ host] host address
+    # @param port[ port]  port TCP/IP port
+    # @param local_backup_folder [local_backup_folder] pre defined folder for backup
+    def initialize(host, port, local_backup_folder)
       @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
       @last_fetch_timestamp = nil
       @last_save_timestamp = nil
+      @last_content_data_id = nil
       @content_server_content_data_path = File.join(local_backup_folder, 'remote',
                                                     host + '_' + port.to_s)
       Log.debug3("Initialized RemoteContentClient: host:#{host}   port:#{port}  local_backup_folder:#{local_backup_folder}")
     end
 
+    # Receives content data,  and writes it to file
+    # Unchanged data will not be saved
+    # @param message [Message] ContentData object
     def receive_content(message)
-      Log.debug1("Backup server received Remote content data:#{message.to_s}")
       Log.info("Backup server received Remote content data")
-      @dynamic_content_data.update(message)
       @last_fetch_timestamp = Time.now.to_i
 
-      save_time_span = Params['remote_content_save_timeout']
-      if !@last_save_timestamp.nil?
-        save_time_span = Time.now.to_i - @last_save_timestamp
-      end
-
-      if save_time_span >= Params['remote_content_save_timeout']
-      	@last_save_timestamp = Time.now.to_i
-        write_to = File.join(@content_server_content_data_path,
-                             @last_save_timestamp.to_s + '.cd')
+      # Update remote content data and write to file if changed ContentData received
+      if(message.unique_id != @last_content_data_id)
+        path = File.join(@content_server_content_data_path, @last_save_timestamp.to_s + '.cd')
         FileUtils.makedirs(@content_server_content_data_path) unless \
               File.directory?(@content_server_content_data_path)
-        count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
-        Log.debug1("Written content data to file:#{write_to}.")
+        $remote_content_data_lock.synchronize{
+          $remote_content_data = message
+          $remote_content_data.to_file(path)
+          @last_content_data_id = message.unique_id   # save last content data ID
+        }
+        Log.debug1("Written content data to file:#{path}.")
       else
         Log.debug1("No need to write remote content data, it has not changed.")
       end
     end
 
+    # Runs Remote content client
     def run()
       Log.debug1("Running remote content client.")
       threads = []
@@ -55,13 +63,13 @@ module ContentServer
         loop do
           # if need content data
           sleep_time_span = Params['remote_content_save_timeout']
-          if !@last_fetch_timestamp.nil?
+          if @last_fetch_timestamp
             sleep_time_span = Time.now.to_i - @last_fetch_timestamp
           end
           Log.debug1("sleep_time_span: #{sleep_time_span}")
           if sleep_time_span >= Params['remote_content_save_timeout']
             # Send ping!
-            bytes_written = @remote_tcp.send_obj(nil)
+            @remote_tcp.send_obj(nil)
             Log.info("sending ping request for remote content data!")
           end
           sleep(sleep_time_span) if sleep_time_span > 0
@@ -70,21 +78,31 @@ module ContentServer
     end
   end
 
+# Remote Content Server
+   # Simple TCP server.
+   #It connects to server and listens to incoming requests. After receiving the request it sends local content data structure.
   class RemoteContentServer
-    def initialize(dynamic_content_data, port)
-      @dynamic_content_data = dynamic_content_data
+    # initializes class variables
+    def initialize(port)
       @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
       Log.debug3("initialize RemoteContentServer on port:#{port}")
     end
 
+    # content_requested
+    # @param addr_info [addr_info] address information
+    # @param message [message] Content Data
     def content_requested(addr_info, message)
       # Send response.
       Log.info("Content server received content data request")
-      Log.debug1("Sending content data:#{@dynamic_content_data.last_content_data}")
-      @tcp_server.send_obj(@dynamic_content_data.last_content_data)
+      $local_content_data_lock.synchronize{
+        Log.debug1("Sending content data:#{$local_content_data}")
+        @tcp_server.send_obj($local_content_data)
+      }
       Log.info('Content server sent content data')
     end
 
+    # Open TCP/IP Thread
+    # @return [run_server] (see #run_server)run TCP server definition in tcp.rb
     def tcp_thread
       return @tcp_server.tcp_thread if @tcp_server != nil
       nil

data/lib/content_server/server.rb

@@ -0,0 +1,91 @@
+require 'log'
+require 'params'
+require 'process_monitoring/thread_safe_hash'
+
+
+module ContentServer
+
+  # Flags combined for content and backup server.
+  Params.string('local_server_name', `hostname`.strip, 'local server name')
+  Params.path('tmp_path', '~/.bbfs/tmp', 'tmp path for temporary files')
+
+  Params.path('local_content_data_path', '', 'ContentData file path.')
+
+  # Monitoring
+  Params.boolean('enable_monitoring', false, 'Whether to enable process monitoring or not.')
+  Params.integer('process_vars_delay', 3, 'pulling time of variables')
+
+  # Handling thread exceptions.
+  Params.boolean('abort_on_exception', true, 'Any exception in any thread will abort the run.')
+
+  Params.integer('data_flush_delay', 300, 'Number of seconds to delay content data file flush to disk.')
+
+  #using module method fot globals initialization due to the use of 'Params'.
+  def init_globals
+    $process_vars = ThreadSafeHash::ThreadSafeHashMonitored.new(Params['enable_monitoring'])
+    $tmp_content_data_file = nil  # will be init during execution
+    $testing_memory_active = false
+    $testing_memory_log = nil
+    $indexed_file_count = 0
+    $local_content_data = nil
+    $local_content_data_lock = nil
+    $remote_content_data_lock = nil
+    $remote_content_data = nil
+  end
+
+  def handle_program_termination(exception)
+    #Write exception message to console
+    message = "\nInterrupt or Exit happened in server:'#{Params['service_name']}'.\n" +
+      "Exception type:'#{exception.class}'.\n" +
+      "Exception message:'#{exception.message}'.\n" +
+      "Stopping process.\n" +
+      "Backtrace:\n#{exception.backtrace.join("\n")}"
+    puts(message)
+
+    # force write content data to file
+    if $local_content_data
+      puts("\nForce writing local content data to #{Params['local_content_data_path']}.")
+      $local_content_data.to_file($tmp_content_data_file)
+      File.rename($tmp_content_data_file, Params['local_content_data_path'])
+    end
+
+    #Write exception message to log and mail(if enabled)
+    Log.error(message)
+  end
+
+  def monitor_general_process_vars
+    objects_counters = {}
+    objects_counters["Time"] = Time.now.to_i
+    while true do
+      current_objects_counters = {}
+      sleep(Params['process_vars_delay'])
+      time = Time.now
+      $process_vars.set('time', time)
+      current_objects_counters['Time'] = time.to_i
+      count = ObjectSpace.each_object(String).count
+      $process_vars.set('String count', count)
+      current_objects_counters['String'] = count
+      count = ObjectSpace.each_object(ContentData::ContentData).count
+      $process_vars.set('ContentData count', count)
+      current_objects_counters['ContentData'] = count
+      dir_count = ObjectSpace.each_object(FileMonitoring::DirStat).count
+      $process_vars.set('DirStat count', dir_count)
+      current_objects_counters['DirStat'] = dir_count
+      file_count = ObjectSpace.each_object(FileMonitoring::FileStat).count
+      $process_vars.set('FileStat count', file_count-dir_count)
+      current_objects_counters['FileStat'] = file_count-dir_count
+
+      # Generate report and update global counters
+      report = ""
+      current_objects_counters.each_key { |type|
+        objects_counters[type] = 0 unless objects_counters[type]
+        diff =  current_objects_counters[type] - objects_counters[type]
+        report += "Type:#{type} raised in:#{diff}   \n"
+        objects_counters[type] = current_objects_counters[type]
+      }
+      Log.info("MEM REPORT at:#{time}:\n#{report}\n")
+    end
+  end
+
+  module_function :init_globals, :handle_program_termination, :monitor_general_process_vars
+end

data/lib/content_server/version.rb

@@ -1,3 +1,3 @@
   module ContentServer
-    VERSION = "1.1.0"
+    VERSION = "1.2.0"
   end

data/lib/content_server.rb

@@ -1,17 +1,2 @@
-require 'content_server/content_server'
-require 'content_server/backup_server'
-
 module ContentServer
-  # Flags combined for content and backup server.
-  Params.path('local_content_data_path', '', 'ContentData file path.')
-
-  # Monitoring
-  Params.boolean('enable_monitoring', false, 'Whether to enable process monitoring or not.')
-  Params.integer('process_vars_delay', 3, 'pulling time of variables')
-
-  # Handling thread exceptions.
-  Params.boolean('abort_on_exception', true, 'Any exception in any thread will abort the run.')
-
-  Params.integer('data_flush_delay', 300, 'Number of seconds to delay content data file flush to disk.')
-
 end # module ContentServer