webroar 0.5.0 → 0.6.1

data/src/ruby_lib/analyzer/resources_analyzer.rb

@@ -19,8 +19,8 @@
 module Webroar
   module Analyzer
     class ResourceAnalyzer
-      include WithExceptionHandling   
-      
+
+      include WithExceptionHandling
       def initialize(starling_pid)
         # @apps is a hash having application name as key and id as value. It contains all the application running under WebROaR.
         @apps = Hash.new
@@ -33,106 +33,63 @@ module Webroar
         load_apps
         initialize_pids(starling_pid || 0)
       end
-      
-      def load_apps
-        begin
-          apps = App.find(:all,:select => "id, name")
-          apps.each do |app|
-            @apps[app.name] = app.id
-            @pids[app.id] = Array.new
-          end
-          # for WebROaR head, Starling server and WebROaR analyzer there will be only process, but to keep the processsing same saving pid in array.
-          unless @apps[@webroar_head]
-            app = App.create({:name => @webroar_head})
-            @apps[app.name] = app.id
-            @pids[app.id] = Array.new
-          end
-          unless @apps[@webroar_analyzer]
-            app = App.create({:name => @webroar_analyzer})
-            @apps[app.name] = app.id
-            @pids[app.id] = Array.new
-          end
-          unless @apps[@starling_server]
-            app = App.create({:name => @starling_server})
-            @apps[app.name] = app.id
-            @pids[app.id] = Array.new
-          end
-        rescue Exception => e
-          Logger.error(e)
-          Logger.error(e.backtrace.join("\n"))
-          raise
+
+      # Reload instance variables
+      def reload_apps
+        apps = nil
+        with_exception_handling("App entry App.find") do
+          apps = App.find(:all)
         end
-      end
-      
-      def initialize_pids(starling_pid)
-        begin
-          @pids[@apps[@webroar_analyzer]] <<= Process.pid          
-          @pids[@apps[@starling_server]] <<= starling_pid.to_i
-          
-          grep_str_head = 'webroar-head'
-          cmd = "ps -e -opid,comm | grep #{grep_str_head} | tail -1"
-          pid = `#{cmd}`.split(' ')[0]
-          if pid
-            @pids[@apps[@webroar_head]] <<= pid.to_i
-          end
-        rescue Exception => e
-          Logger.error(e)
-          Logger.error(e.backtrace.join("\n"))
-          raise
+
+        new_apps = Hash.new
+        new_pids = Hash.new
+
+        apps.each do |app|
+          new_apps[app.name] = app.id
+          new_pids[app.id] = @pids[app.id] || Array.new
         end
+
+        @apps = new_apps
+        @pids = new_pids
+
       end
-      
+
       # Registering pid with given application.
       def set_pid(pid, app_name)
-        Logger.info "Got pid = #{pid} for Application = #{app_name}"
+        WLogger.info "Got pid = #{pid} for Application = #{app_name}"
         begin
-          
+
+          reload_apps unless app_id = @apps[app_name]
           unless app_id = @apps[app_name]
+            WLogger.error("Application #{app_name} not found.")
+            return
             #TODO: Here chances of race condition in record creation with MessageAnalyzer#process_messages
-            app = nil
-            with_exception_handling("App.find for application #{app_name}") do
-              app = App.find(:first, :conditions=>["name = ?", app_name])
-            end            
-            if app
-              app_id = app.id
-            else   
-              with_exception_handling("App create for application #{app_name}") do
-                app = App.create({:name => app_name})
-              end
-              app_id = app.id if app              
-            end
-            
-            if app_id
-              @apps[app_name] = app_id 
-              @pids[app_id] = Array.new
-            end
-            
           end
-                
-          @pids[app_id] <<= pid if app_id
-          #Logger.debug "After adding PID for #{app_id}, pid array is #{@pids[app_id].join(',')}"
-          
+
+          @pids[app_id] <<= pid
+          #WLogger.debug "After adding PID for #{app_id}, pid array is #{@pids[app_id].join(',')}"
+
         rescue Exception => e
-          Logger.error(e)
-          Logger.error(e.backtrace.join("\n"))
+          WLogger.error(e)
+          WLogger.error(e.backtrace.join("\n"))
           return
         end
       end
-      
+
       # Determines processor usage and memory usage for the registered application.
       # Usually called at every 60 seconds.
       def take_sample
         begin
           wall_time = Time.now
           @pids.each_pair do |app_id, pid_array|
-            #Logger.debug "taking sample for app - #{app_id}, pids are #{pid_array.join(',')}"
+            #WLogger.debug "taking sample for app - #{app_id}, pids are #{pid_array.join(',')}"
             if pid_array.length > 0
               cmd = "ps -o pid,pcpu,rss -p #{pid_array.join(' ')}"
-              #Logger.debug cmd
+              #WLogger.debug cmd
               result = `#{cmd}`
-              #Logger.debug result
+              #WLogger.debug result
               result = result.split(" ")
-              #Logger.debug result
+              #WLogger.debug result
               interval = 3
               pid = Array.new
               pcpu = Array.new
@@ -153,25 +110,56 @@ module Webroar
               mem.collect!{ |e| e.to_i}
               # In dynamic load balancing some processors might been killed. And ps command returns output for live process only.
               # By applying *and* operation of set theory, it keeps only running process id in array.
-              #Logger.debug "Before anding app_id is #{app_id}, pids array is #{@pids[app_id].join(',')}"
+              #WLogger.debug "Before anding app_id is #{app_id}, pids array is #{@pids[app_id].join(',')}"
               @pids[app_id] &= pid
-              #Logger.debug "After anding app_id is #{app_id}, pids array is #{@pids[app_id].join(',')}"
+              #WLogger.debug "After anding app_id is #{app_id}, pids array is #{@pids[app_id].join(',')}"
               sum_pcpu = 0
               pcpu.each { |e| sum_pcpu += e}
               sum_mem = 0
               mem.each { |e| sum_mem += e}
               # Storing wall_time taken before the begining of block, due to that there is slight difference in actual wall_time vs resouce usage
               with_exception_handling("Resource usage entry creation for app_id=#{app_id}") do
-                ResourceUsage.create({:app_id => app_id, :cpu_usage => sum_pcpu, :memory_usage => sum_mem, :wall_time => wall_time})                
-              end                    
+                ResourceUsage.create({:app_id => app_id, :cpu_usage => sum_pcpu, :memory_usage => sum_mem, :wall_time => wall_time})
+              end
             end # if
           end #  do |app_id, pid_array|
         rescue Exception => e
-          Logger.error(e)
-          Logger.error(e.backtrace.join("\n"))
+          WLogger.error(e)
+          WLogger.error(e.backtrace.join("\n"))
           return
         end
       end  # def take_sample
+
+      private
+
+      def load_apps
+        apps = nil
+        with_exception_handling("App entry App.find") do
+          apps = App.find(:all)
+        end
+
+        apps.each do |app|
+          @apps[app.name] = app.id
+          @pids[app.id] = Array.new
+        end
+
+      end
+
+      def initialize_pids(starling_pid)
+        begin
+          @pids[@apps[@webroar_analyzer]] <<= Process.pid
+          @pids[@apps[@starling_server]] <<= starling_pid.to_i
+
+          grep_str_head = 'webroar-head'
+          cmd = "ps -e -opid,comm | grep #{grep_str_head} | tail -1"
+          pid = `#{cmd}`.split(' ')[0]
+          @pids[@apps[@webroar_head]] <<= pid.to_i if pid
+        rescue Exception => e
+          WLogger.error(e)
+          WLogger.error(e.backtrace.join("\n"))
+          raise
+        end
+      end
     end # class ResourceAnalyzer
   end #module Analyzer
 end # module Webroar

data/src/ruby_lib/analyzer/webroar_analyzer.rb

@@ -16,15 +16,132 @@
 # You should have received a copy of the GNU General Public License
 # along with WebROaR.  If not, see <http://www.gnu.org/licenses/>.
 
-require File.expand_path(File.join(File.dirname(__FILE__), 'user_defined_exception.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'with_exception_handling'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'db_connect.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__) , 'message_reader.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'message_analyzer.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'resources_analyzer.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'webroar_analyzer_script_runner.rb'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'logger.rb'))
+module Webroar
+  module Analyzer
+    # ScriptRunner class design inspired from Starling runner
+    class ScriptRunner
+      def self.run
+        new
+      end
 
+      private
 
-Webroar::Analyzer::ScriptRunner.run
+      # Reading configuration file. Opening log file and PID file on given path.
+      def initialize
+        @@instance = self
+        @configuration = YAML.load(File.open(File.join(WEBROAR_ROOT,'conf','server_internal_config.yml')))
+        @process = ProcessHelper.new(@configuration["webroar_analyzer_script"]["log_file"], @configuration["webroar_analyzer_script"]["pid_file"])
+        WLogger.set_log_file(@configuration["webroar_analyzer_script"]["log_file"])
 
+        pid = @process.running?
+        if pid
+          STDERR.puts "There is already a webroar_analyzer process running (pid #{pid}), exiting."
+          exit(1)
+        elsif pid.nil?
+          STDERR.puts "Cleaning up stale pidfile at #{@configuration[:pid_file]}."
+        end
+        start
+      end
+
+      def get_starling_pid
+        # read starling process id
+        pid_file = YAML.load(File.open(File.join(WEBROAR_ROOT, 'conf', 'starling_server_config.yml')))["starling"]["pid_file"]
+        pid = File.read(pid_file).chomp.to_i rescue nil
+      end
+
+      def start
+        #drop_privileges
+
+        @process.daemonize if @configuration["webroar_analyzer_script"]["daemonize"]
+
+        setup_signal_traps
+        @process.write_pid_file
+
+        WLogger.info "Starting webroar-analyzer."
+        DBConnect.db_up(@configuration["webroar_analyzer_script"]["environment"])
+        @analyzer = ResourceAnalyzer.new(get_starling_pid)
+        @processor = MessageAnalyzer.new(MessageReader.new(@configuration["starling"]), @analyzer, @configuration["webroar_analyzer_script"]["sampling_rate"])
+
+        @message_processor = Thread.new {
+          while true
+            @processor.process_messages
+            sleep(60)
+          end
+        }
+
+        @stale_sample_writer = Thread.new {
+          while true
+            @processor.process_stale_samples
+            sleep(60)
+          end
+        }
+
+        @resource_monitor = Thread.new {
+          while true
+            @processor.process_pid
+            @analyzer.take_sample
+            sleep(60)
+          end
+        }
+
+        @exception_processor = Thread.new {
+          while true
+            @processor.process_exceptions
+            sleep(60)
+          end
+        }
+
+        @message_processor.join
+        @stale_sample_writer.join
+        @resource_monitor.join
+        @exception_processor.join
+
+      end
+
+      def shutdown
+        begin
+          WLogger.info "Got shutdown signal. Processing remaining messages in queue..."
+          Thread.kill(@message_processor)
+          Thread.kill(@stale_sample_writer)
+          Thread.kill(@resource_monitor)
+          Thread.kill(@exception_processor)
+
+          @processor.write_all_samples
+
+          WLogger.info "Messages processed."
+          WLogger.info "Stopping Starling server...."
+          pid = get_starling_pid
+          if pid
+            system("kill -INT #{pid} 2>/dev/null")
+            if $? == 0
+              WLogger.info "Stopped."
+            else
+              WLogger.info "Failed."
+            end
+          else
+            WLogger.info 'Failed to retrieve pid.'
+          end
+          @process.remove_pid_file
+          #TODO: Write data purger
+          WLogger.info "Webroar-analyzer process stopped."
+        rescue Object => e
+          WLogger.error(e)
+          WLogger.error(e.backtrace.join("\n"))
+          WLogger.error "There was an error shutting down: #{e}"
+          exit(70)
+        end
+        exit(0)
+      end
+
+      def setup_signal_traps
+        Signal.trap("INT") { shutdown }
+        Signal.trap("TERM") { shutdown }
+        Signal.trap("USR1") {
+          @processor.reload_apps
+          @analyzer.reload_apps
+        }
+
+      end
+    end #Runner
+  end # Analyzer
+end # Webroar

data/src/ruby_lib/analyzer/with_exception_handling.rb

@@ -25,29 +25,18 @@ module Webroar
         begin
           block.call
         rescue ActiveRecord::StatementInvalid, Exception => e
-          log_message_with_check(:info, log_message)        
-          log_message_with_check(:error, "#{e.message}. try no #{trial+1}")     
+          WLogger.info(log_message)        
+          WLogger.error("#{e.message}. try no #{trial+1}")     
           if trial < MAX_TRIAL
             trial += 1            
             sleep(2)          
             retry
           end
-          log_message_with_check(:info, log_message)  
-          log_message_with_check(:error, e) 
-          log_message_with_check(:error, e.backtrace.join("\n"))
+          WLogger.info(log_message)  
+          WLogger.error(e) 
+          WLogger.error(e.backtrace.join("\n"))
         end
-      end
-      
-      def log_message_with_check(level, message)
-        if defined? Webroar::Analyzer::Logger
-          case level
-            when :info
-              Webroar::Analyzer::Logger.info(message)
-            when :error
-              Webroar::Analyzer::Logger.error(message)
-          end  
-        end
-      end
+      end      
     end
   end
 end

data/src/ruby_lib/exception_tracker/webroar_exception.rb

@@ -16,9 +16,6 @@
 # You should have received a copy of the GNU General Public License
 # along with WebROaR.  If not, see <http://www.gnu.org/licenses/>.
 
-#ADMIN_PANEL_ROOT = File.join(WEBROAR_ROOT, 'src', 'admin_panel').freeze
-#arequire File.join(ADMIN_PANEL_ROOT,'vendor', 'rails', 'activerecord', 'lib', 'active_record')
-
 module Webroar
   module ExceptionTracker
     def trace_rescue_action(method_name, parent_name, method_type)

data/src/ruby_lib/rack/adapter/rails.rb

@@ -23,7 +23,7 @@ module Webroar
         $0 = 'webroar' # Some application using it to retrive executable name.
         load_application
         
-        @rails_app = if defined?(ActionController::Dispatcher) and ActionController::Dispatcher.instance_methods.include?('call')
+        @rails_app = if self.class.rack_based?
           ActionController::Dispatcher.new
         elsif ::Rails::VERSION::MAJOR >= 3
           ::Rails.application
@@ -34,15 +34,20 @@ module Webroar
         @file_server = ::Rack::File.new(::File.join(@root, "public"))
       end
       
+      def self.rack_based?
+        rails_version = ::Rails::VERSION
+        return false if rails_version::MAJOR != 2
+        return false if rails_version::MINOR < 2
+        return false if rails_version::MINOR == 2 && rails_version::TINY < 3
+        true # >= 2.2.3 and < 3.0.0
+      end
+      
       def load_application
         ENV['RAILS_ENV'] = @env
         
         require "#{@root}/config/environment"
-#        TODO: figure out the way to set relative URL for images defined inside the css
         if ::Rails::VERSION::MAJOR >= 3
-          ::Rails.application.config.relative_url_root = @prefix
-          # used to add prefix to asset_path (for image,stylesheet and javascript path)
-          ::Rails.application.config.action_controller.asset_path = proc { |asset_path| "#{@prefix}#{asset_path}" }
+          ActionController::Base.config.relative_url_root = @prefix
         else
           require 'dispatcher'
           if ActionController::Base.respond_to?('relative_url_root=') 

data/src/ruby_lib/ruby_interface/version.rb

@@ -20,12 +20,15 @@ module Webroar
   
   module VERSION #:nodoc:
     MAJOR    = "0"
-    MINOR    = "5"
-    TINY     = "0"
-    
+    MINOR    = "6"
+    TINY     = "1"
+
     STRING   = [MAJOR, MINOR, TINY].join('.')
   end
-  
+  ruby_version = RUBY_VERSION.split(".")
+  RUBY_MAJOR    = ruby_version[0]
+  RUBY_MINOR    = ruby_version[1]
+  RUBY_TINY     = ruby_version[2]
   NAME    = 'WebROaR'.freeze
   SERVER  = "#{NAME}-#{VERSION::STRING}".freeze
   

data/src/ruby_lib/webroar_app_loader.rb

@@ -15,10 +15,23 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with WebROaR.  If not, see <http://www.gnu.org/licenses/>.
+
+$: << "." if not $:.include? "."
 begin
-  require 'rubygems'
-  gem 'starling', '>=0.10.0'
-  require 'starling'
+  require 'yaml'
+  $g_options['analytics'] = true
+
+  server_conf = YAML.load(File.open(File.join($g_options["webroar_root"], "conf", "server_internal_config.yml")))
+  if(server_conf and server_conf["webroar"] and !server_conf["webroar"]["analyzer"].nil?)
+    $g_options['analytics'] = server_conf["webroar"]["analyzer"]
+  end
+
+  if $g_options['analytics']
+    require 'rubygems'
+    gem 'starling', '>=0.10.0'
+    require 'starling'
+  end
+  
   require File.join(File.expand_path(File.dirname(__FILE__)), 'ruby_interface', 'ruby_interface')
   
   #turn it on to see the Ruby exception
@@ -54,22 +67,24 @@ begin
       #    STDERR.reopen(STDOUT)
       #    STDERR.sync = true
       #    puts '...........file opened..............'
-      require File.join(File.dirname(__FILE__), 'profiler', 'message_dispatcher') #to send pid of worker
-#      TODO: Add Exception tracking for Rails 3
-      if $pid_sent and $g_options["app_name"].strip != 'Admin Panel' and $g_options["app_type"] == "rails" 
-        require File.join(File.dirname(__FILE__), 'exception_tracker', 'webroar_exception.rb')
-      else
-        Webroar.log_info("Exception notification would not work.")
-      end
-#      TODO: Add profiling support for Rails 3
-      if $g_options["app_profiling"] == "yes" and $g_options["app_type"] == "rails"  
-        if $pid_sent
-          require File.expand_path(File.join($g_options["webroar_root"], 'src', 'ruby_lib', 'profiler', 'webroar_profiling.rb'))     
-        else
-          Webroar.log_info("Profiling did not started.")
-        end
-      end
       
+      if $g_options['analytics']
+        require File.join(File.dirname(__FILE__), 'profiler', 'message_dispatcher') #to send pid of worker
+  #      TODO: Add Exception tracking for Rails 3
+        if $pid_sent and $g_options["app_name"].strip != 'Admin Panel' and $g_options["app_type"] == "rails" 
+          require File.join(File.dirname(__FILE__), 'exception_tracker', 'webroar_exception.rb')
+        elsif $g_options["app_name"].strip != 'Admin Panel'
+          Webroar.log_info("Exception notification would not work.")
+        end
+  #      TODO: Add profiling support for Rails 3
+        if $g_options["app_profiling"] == "yes" and $g_options["app_type"] == "rails"  
+          if $pid_sent
+            require File.expand_path(File.join($g_options["webroar_root"], 'src', 'ruby_lib', 'profiler', 'webroar_profiling.rb'))     
+          else
+            Webroar.log_info("Profiling did not started.")
+          end
+        end
+      end # analytics
       # Taken from Mongrel cgi_multipart_eof_fix
       # Ruby 1.8.5 has a security bug in cgi.rb, we need to patch it.
       version = RUBY_VERSION.split('.').map { |i| i.to_i }      

data/src/vendor/libebb/ebb.c

@@ -153,9 +153,9 @@ session_cache_store(void *data, gnutls_datum_t key, gnutls_datum_t value)
   cache->value.data = (void*)cache->value_storage;
 
   cache->node.key = &cache->key;
-  cache->node.value = &cache;
+  cache->node.value = cache;
 
-  rbtree_insert(tree, (rbtree_node)cache);
+  rbtree_insert(tree, &cache->node);
 
   //printf("session_cache_store\n");
 
@@ -203,6 +203,13 @@ session_cache_remove (void *data, gnutls_datum_t key)
   return 0;
 }
 
+static void session_cache_deinit(rbtree tree){
+  while(tree->root && tree->root->key){
+    struct session_cache *cache = tree->root->value;
+    session_cache_remove (tree, cache->key);
+  }
+}
+
 static void 
 on_handshake(struct ev_loop *loop ,ev_io *watcher, int revents)
 {
@@ -631,6 +638,13 @@ ebb_server_unlisten(ebb_server *server)
     server->port[0] = '\0';
     server->listening = FALSE;
   }
+#ifdef HAVE_GNUTLS
+  if(server->secure){
+    session_cache_deinit(&server->session_cache);
+    gnutls_certificate_free_credentials (server->credentials);
+    gnutls_global_deinit ();
+  }
+#endif
 }
 
 /**
@@ -654,7 +668,6 @@ ebb_server_init(ebb_server *server, struct ev_loop *loop)
   server->secure = FALSE;
 
 #ifdef HAVE_GNUTLS
-  rbtree_init(&server->session_cache, session_cache_compare);
   server->credentials = NULL;
 #endif
 
@@ -683,7 +696,8 @@ ebb_server_set_secure (ebb_server *server, const char *cert_file, const char *ke
 {
   server->secure = TRUE;
   gnutls_global_init();
-  gnutls_certificate_allocate_credentials(&server->credentials);
+  gnutls_certificate_allocate_credentials(&server->credentials);  
+  rbtree_init(&server->session_cache, session_cache_compare);
   /* todo gnutls_certificate_free_credentials */
   int r = gnutls_certificate_set_x509_key_file( server->credentials
                                               , cert_file
@@ -694,6 +708,7 @@ ebb_server_set_secure (ebb_server *server, const char *cert_file, const char *ke
     error("loading certificates");
     return -1;
   }
+
   return 1;
 }
 #endif /* HAVE_GNUTLS */

data/src/vendor/libev/ev.c

@@ -2894,7 +2894,7 @@ infy_add (EV_P_ ev_stat *w)
       if ((errno == ENOENT || errno == EACCES) && strlen (w->path) < 4096)
         {
           char path [4096];
-          strcpy (path, w->path);
+          memcpy (path, w->path, strlen(w->path)+1);
 
           do
             {

data/src/worker/wkr_controller.c

@@ -105,7 +105,7 @@ int connect_to_head(wkr_t *w){
     
     memset(&addr, 0, sizeof(addr));
     addr.sun_family = AF_UNIX;
-    strcpy(addr.sun_path, w->tmp->ctl_path.str);
+    memcpy(addr.sun_path, w->tmp->ctl_path.str, w->tmp->ctl_path.len+1);
     
     int len = sizeof(addr.sun_family)+strlen(addr.sun_path);
 #ifdef __APPLE__

data/src/worker/wkr_http.c

@@ -200,7 +200,7 @@ int init_ruby_interpreter(wkr_t *w) {
     retval = -1;
     goto err;
   }
-  strcpy(v[0],"webroar-worker");
+  memcpy(v[0],"webroar-worker", strlen("webroar-worker")+1);
 
   v[1] =  (char*) malloc(sizeof(char)*(Config->Worker.File.app_loader.len +1));
   if(!v[1]) {
@@ -208,7 +208,7 @@ int init_ruby_interpreter(wkr_t *w) {
     retval = -1;
     goto err;
   }
-  strcpy(v[1], Config->Worker.File.app_loader.str);
+  memcpy(v[1], Config->Worker.File.app_loader.str, Config->Worker.File.app_loader.len+1);
 
   //RUBY_INIT_STACK;
   ruby_init();

data/src/worker/wkr_main.c

@@ -84,8 +84,8 @@ void crash_handler(int sig) {
     //TODO: add application name
     sprintf(core_file_name, "/tmp/webroar-worker");
     if ( get_timestamp(timestamp) == 0 ) {
-      strcat(core_file_name, "-");
-      strcat(core_file_name, timestamp);
+      memcpy(core_file_name + strlen(core_file_name), "-", 2);
+      memcpy(core_file_name + strlen(core_file_name), timestamp, strlen(timestamp)+1);
     }
 #ifdef __APPLE__
     sprintf(cmd, "gcore -c %s %ld", core_file_name, (long) getppid() );
@@ -104,10 +104,10 @@ void crash_handler(int sig) {
   // get void*'s for all entries on the stack
   size = backtrace(array, Config->Worker.stack_tace);
   bt_symbols = backtrace_symbols(array, size);
-  strcpy(bt_string, "\n");
+  memcpy(bt_string, "\n", 2);
   for(i = 0; i < size; i++) {
-    strcat(bt_string, bt_symbols[i]);
-    strcat(bt_string, "\n");
+    memcpy(bt_string + strlen(bt_string), bt_symbols[i], strlen(bt_symbols[i]) + 1);
+    memcpy(bt_string + strlen(bt_string), "\n", 2);
   }
   LOG_ERROR(FATAL, "Obtained %zd stack frames.%s", size, bt_string);
   free(bt_symbols);