content_server 1.1.0 → 1.2.0

data/bin/backup_server

@@ -7,13 +7,16 @@
 
 # NOTE this file mainly is a copy of content_server
 
+require 'content_server'  #todo: yarondbb need it?
+require 'content_server/backup_server' # specific backup server impl
+require 'content_server/server' # shared file between content\backup
 require 'log'
 require 'params'
 require 'run_in_background'
-require 'content_server'
 
 Params.init ARGV
 Log.init
+ContentServer.init_globals
 
 Thread.abort_on_exception = true  # TODO (genadyp) should be treated globally? by param for example.
 
@@ -21,28 +24,13 @@ retries = 0  # number of retries to run a server
 
 begin
   RunInBackground.run { ContentServer.run_backup_server }
-rescue SystemExit, SignalException => exc
+rescue SystemExit, SignalException => exception
   # TODO (genadyp) do we need to trap signals by types?
-  puts("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
-                "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
-  puts("Force writing local content data to #{Params['local_content_data_path']}.")
-  ContentServer::local_dynamic_content_data.last_content_data.to_file(ContentServer::tmp_content_data_file)
-  sleep(0.1)  # Added to prevent mv access issue
-  ::FileUtils.mv(ContentServer::tmp_content_data_file, Params['local_content_data_path'])
-
-
-  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
-            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  ContentServer.handle_program_termination(exception)
   Log.flush
   exit
-rescue Exception => exc
-  puts("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n#{exc.backtrace.join("\n")}")
-  puts("Force writing local content data to #{Params['local_content_data_path']}.")
-  ContentServer::local_dynamic_content_data.last_content_data.to_file(ContentServer::tmp_content_data_file)
-  sleep(0.1)  # Added to prevent mv access issue
-  ::FileUtils.mv(ContentServer::tmp_content_data_file, Params['local_content_data_path'])
-
-  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n#{exc.backtrace.join("\n")}")
+rescue Exception => exception
+  ContentServer.handle_program_termination(exception)
   if retries > 0
     Log.debug1("Restarting (retries:#{retries}).")
   else

data/bin/content_server

@@ -4,13 +4,16 @@
 # The files in those directories are indexed (calculating their SHA1).
 # Each unique content is backed up to the remote (backup) server.
 
+require 'content_server'  #todo: yarondbb need it?
+require 'content_server/content_server'  # specific content server impl
+require 'content_server/server'  # shared file between content\backup
 require 'log'
 require 'params'
 require 'run_in_background'
-require 'content_server'
 
 Params.init ARGV
 Log.init
+ContentServer.init_globals
 
 Thread.abort_on_exception = true  # TODO (genadyp) should be treated globally? by param for example.
 
@@ -18,28 +21,13 @@ retries = 0  # number of retries to run a server
 
 begin
   RunInBackground.run { ContentServer.run_content_server }
-rescue SystemExit, SignalException => exc
+rescue SystemExit, SignalException => exception
   # TODO (genadyp) do we need to trap signals by types?
-  #force dump of content data to file
-  puts("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
-           "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
-  puts("Force writing local content data to #{Params['local_content_data_path']}.")
-  ContentServer::local_dynamic_content_data.last_content_data.to_file(ContentServer::tmp_content_data_file)
-  sleep(0.1)  # Added to prevent mv access issue
-  ::FileUtils.mv(ContentServer::tmp_content_data_file, Params['local_content_data_path'])
-
-  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
-            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  ContentServer.handle_program_termination(exception)
   Log.flush
   exit
-rescue Exception => exc
-  puts("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n#{exc.backtrace.join("\n")}")
-  puts("Force writing local content data to #{Params['local_content_data_path']}.")
-  ContentServer::local_dynamic_content_data.last_content_data.to_file(ContentServer::tmp_content_data_file)
-  sleep(0.1)  # Added to prevent mv access issue
-  ::FileUtils.mv(ContentServer::tmp_content_data_file, Params['local_content_data_path'])
-
-  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n#{exc.backtrace.join("\n")}")
+rescue Exception => exception
+  ContentServer.handle_program_termination(exception)
   if retries > 0
     Log.debug1("Restarting (retries:#{retries}).")
   else

data/bin/testing_memory

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+# Runs a memory test:
+# 1. Run content server or backup server
+# 2. Generate files.
+# 3. Monitor files.
+# 4. Index files.
+# 5. Report memory of process at different phases\times
+# Examples:
+#   testing_server --conf_file=~/.bbfs/etc/backup_testing_memory.yml --server_to_test=backup
+#   testing_server --conf_file=~/.bbfs/etc/content_testing_memory.yml --server_to_test=content
+
+# TODO add command line help/usage
+
+require 'params'
+require 'run_in_background'
+require 'testing_memory/testing_memory'
+
+Params.string('server_to_test', nil, 'Server that will be run under test_server. Values are: content, backup')
+
+Params.init ARGV
+Log.init
+ContentServer.init_globals
+
+Thread.abort_on_exception = true
+
+retries = 0  # number of retries to run a server
+
+begin
+  case Params['server_to_test']
+    when 'content'
+      RunInBackground.run { TestingMemory.run_content_memory_server }
+    when 'backup'
+      RunInBackground.run { TestingMemory.run_backup_memory_server }
+    else
+      raise ArgumentError.new "Incorrect server_to_test parameter value: #{Params['server_to_test']}"
+  end
+  retries -=1
+
+rescue SystemExit, SignalException => exc
+  # TODO (genadyp) do we need to trap signals by types?
+  puts("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+                "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.flush
+  exit
+rescue Exception => exc
+  puts("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
+                "#{exc.backtrace.join("\n")}")
+  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
+                "#{exc.backtrace.join("\n")}")
+  if retries > 0
+    Log.debug1("Restarting (retries:#{retries}).")
+  else
+    Log.debug1("Exiting...")
+    Log.flush
+  end
+  retries -= 1
+end while retries >= 0

data/bin/testing_server

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+# Runs content server. This server monitors a set of directories (blobs/patterns).
+# The files in those directories are indexed (calculating their SHA1).
+# Each unique content is backed up to the remote (backup) server.
+# How to run on content side:
+# testing_server --conf_file=~/.bbfs/etc/content_testing_server.yml --server_to_test=content
+# How to run on backup side:
+# testing_server --conf_file=~/.bbfs/etc/backup_testing_server.yml --server_to_test=backup
+
+# TODO add command line help/usage
+
+require 'params'
+require 'run_in_background'
+require 'testing_server'
+
+Params.string('server_to_test', nil, 'Server that will be run under test_server. Values are: content, backup')
+
+Params.init ARGV
+Log.init
+ContentServer.init_globals
+
+Thread.abort_on_exception = true
+
+retries = 0  # number of retries to run a server
+
+begin
+  case Params['server_to_test']
+  when 'content'
+    RunInBackground.run { TestingServer.run_content_testing_server }
+  when 'backup'
+    RunInBackground.run { TestingServer.run_backup_testing_server }
+  else
+    raise ArgumentError.new "Incorrect server_to_test parameter value: #{Params['server_to_test']}"
+  end
+
+rescue SystemExit, SignalException => exc
+  # TODO (genadyp) do we need to trap signals by types?
+  puts("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+                "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.error("Interrupt or Exit happened in #{Params['service_name']} server: #{exc.class}, " +
+            "stopping process.\nBacktrace:\n#{exc.backtrace.join("\n")}")
+  Log.flush
+  exit
+rescue Exception => exc
+  puts("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
+                "#{exc.backtrace.join("\n")}")
+  Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
+                "#{exc.backtrace.join("\n")}")
+  if retries > 0
+    Log.debug1("Restarting (retries:#{retries}).")
+  else
+    Log.debug1("Exiting...")
+    Log.flush
+  end
+  retries -= 1
+end while retries >= 0

data/ext/run_in_background/mkrf_conf.rb

@@ -0,0 +1,34 @@
+# This extension used to install platform specific gem dependencies
+# for more information:
+#     http://stackoverflow.com/questions/4596606/rubygems-how-do-i-add-platform-specific-dependency/10249133#10249133
+#     https://github.com/openshift/os-client-tools/blob/master/express/ext/mkrf_conf.rb
+
+require 'rubygems'
+require 'rubygems/command.rb'
+require 'rubygems/dependency_installer.rb'
+#begin
+#Gem::Command.build_args = ARGV
+#rescue NoMethodError
+#end
+inst = Gem::DependencyInstaller.new
+begin
+  if Gem::win_platform?
+    if Gem::Specification.find_all_by_name('sys-uname').empty?
+      inst.install 'sys-uname'
+    end
+    if Gem::Specification.find_all_by_name('win32-service').empty?
+      inst.install 'win32-service'
+    end
+  else
+    if Gem::Specification.find_all_by_name('daemons').empty?
+      inst.install 'daemons'
+    end
+  end
+rescue Exception => e
+  puts e.to_s
+  exit(1)
+end
+
+f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w")   # create dummy rakefile to indicate success
+f.write("task :default\n")
+f.close