webroar 0.3.1 → 0.4.0

data/src/head/wr_application.c

@@ -20,21 +20,20 @@
 
 // Worker count
 static unsigned int worker_count = 0;
+extern config_t *Config;
 
 /************** Private Functions ******************/
 
 // Check whether application already exist
-static inline wr_app_t* wr_app_exist(wr_svr_t *server, const char *app_name){
+wr_app_t* wr_app_exist(wr_svr_t *server, const char *app_name){
   wr_app_t* app = server->apps, *tmp_app = NULL;
-  
   while(app) {
     if(strcmp(app_name, app->conf->name.str)==0)
       return app;
     tmp_app = app;
     app = app->next;
   }
-  
-  if(strcmp(app_name, WR_STATIC_FILE_SERVER_NAME) == 0){
+  if(strcmp(app_name, Config->Application.Static_server.name.str) == 0){
     return server->static_app;
   }
   
@@ -42,7 +41,7 @@ static inline wr_app_t* wr_app_exist(wr_svr_t *server, const char *app_name){
 }
 
 // Check whether pending worker exist
-static inline wr_pending_wkr_t* wr_pending_worker_exist(wr_app_t *app, const int pid){
+wr_pending_wkr_t* wr_pending_worker_exist(wr_app_t *app, const int pid){
   int i;
   for(i = 0 ; i < WR_QUEUE_SIZE(app->q_pending_workers); i++){
     wr_pending_wkr_t* pending = wr_queue_fetch(app->q_pending_workers);
@@ -64,69 +63,121 @@ void wr_app_wkr_add_cb(struct ev_loop *loop, ev_timer *w, int revents) {
   }
 }
 
+/** Set flag to TRUE to kill single pending worker */
+void wr_app_kill_pending_wkr(wr_app_t* app, const int flag){
+  int pid = 0;
+  wr_pending_wkr_t *pending;
+
+  while(WR_QUEUE_SIZE(app->q_pending_workers) > 0) {
+    pending = wr_queue_fetch(app->q_pending_workers);
+    if(pending){
+      pid = *pending;
+      free(pending);
+    }
+    LOG_INFO("wr_app_kill_pending_wkr: killing worker, pid = %d", pid);
+    if(pid > 0)
+      kill(pid ,SIGKILL);
+    if(flag) break;
+  }
+  app->high_ratio = TOTAL_WORKER_COUNT(app) * Config->Application.max_req_ratio;
+}
+
+void wr_app_add_error_msg(wr_app_t* app){
+  int err_msg_len = 0;
+  char err_msg[512];
+
+  LOG_DEBUG(DEBUG,"Some problem occurred while starting Application %s.", app->conf->name.str);
+  if(app->ctl){
+    scgi_header_add(app->ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
+    err_msg_len = sprintf(err_msg,"The application could not be started due to the following error. Please refer '/var/log/webroar/%s.log' and the application log file for more details.", app->conf->name.str);
+    scgi_body_add(app->ctl->scgi, err_msg, err_msg_len);
+    wr_ctl_resp_write(app->ctl);        
+  }
+  app->timeout_counter = 0;
+  app->ctl = NULL; 
+}
+
+/** Remove application from application list */
+int wr_app_remove(wr_svr_t* server, const char* app_name) {
+  LOG_FUNCTION
+  wr_app_t* app = server->apps, *tmp_app = NULL;
+  
+  LOG_DEBUG(DEBUG, "Removing application %s", app_name);
+  
+  while(app) {
+    if(strcmp(app_name, app->conf->name.str)==0)
+      break;
+    tmp_app = app;
+    app = app->next;
+  }
+  if(app) {
+    if(tmp_app) {      
+      tmp_app->next = app->next;
+    } else {
+      server->apps = app->next;
+    }
+    
+    app->next = NULL;
+    wr_app_free(app);
+    wr_app_conf_remove(app_name);
+    return 0;
+  } else if(strcmp(app_name, Config->Application.Static_server.name.str) == 0){
+    wr_app_free(server->static_app);
+    server->static_app = NULL;
+    wr_app_conf_remove(app_name);
+    return 0;
+  } else {
+    LOG_ERROR(WARN,"Aapplication %s didn't found in list", app_name);
+    sprintf(server->err_msg, "Application '%s' is not found.", app_name);
+    return -1;
+  }
+}
+
 /** Callback function to add worker timeout */
 void wr_app_wkr_add_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
   LOG_FUNCTION
   wr_app_t* app = (wr_app_t*) w->data;
-  char err_msg[512];
-  int err_msg_len = 0;
-  int pid = 0;
   
+  LOG_ERROR(SEVERE,"wr_app_wkr_add_timeout_cb");
+
   // Stop add timeout timer and increament timeout counter
   ev_timer_stop(loop, &app->t_add_timeout);
   app->timeout_counter ++;
 
+  if(app->timeout_counter > Config->Server.Worker.add_trials){
+    LOG_ERROR(SEVERE,"Reset worker timeout counter for %s.", app->conf->name.str);
+    app->timeout_counter = 0;
+    return;
+  }else if(app->timeout_counter == Config->Server.Worker.add_trials){
+    LOG_ERROR(SEVERE,"worker timeout counter for %s exceeds limit.", app->conf->name.str);
+
+    app->timeout_counter ++;
+
+    wr_app_kill_pending_wkr(app, FALSE);
+
+    app->t_add_timeout.repeat = Config->Server.Worker.add_wait;
+    ev_timer_again(loop, &app->t_add_timeout);
+  }
+
   // Kill oldest pending worker
-  if(WR_QUEUE_SIZE(app->q_pending_workers) > 0) {
-    wr_pending_wkr_t *pending = wr_queue_fetch(app->q_pending_workers);
-    if(pending){
-      pid = *pending;
-      free(pending);
-    } 
-    LOG_INFO("wr_app_wkr_add_timeout_cb: killing worker, pid = %d", pid);
-    if(pid > 0)
-      kill(pid ,SIGKILL);
-    LOG_DEBUG(DEBUG,"app->pending_wkr = %d", WR_QUEUE_SIZE(app->q_pending_workers));
-    if(WR_QUEUE_SIZE(app->q_pending_workers) > 0) {
-      ev_timer_again(loop, &app->t_add_timeout);
-    }
+  wr_app_kill_pending_wkr(app, TRUE);
 
-    // Update high load ratio
-    app->high_ratio = TOTAL_WORKER_COUNT(app) * WR_MAX_REQ_RATIO;
+  if(WR_QUEUE_SIZE(app->q_pending_workers) > 0) {
+    ev_timer_again(loop, &app->t_add_timeout);
   }
   
   // If application restarted, rollback all the changes.
   if(app->state == WR_APP_RESTART){
+    wr_application_list_free(app->conf->new);
+    app->conf->new = NULL;
     app->state = WR_APP_ACTIVE;
     // Send error response
-    LOG_DEBUG(DEBUG,"Some problem occurred while starting Application %s.", app->conf->name.str);
-    if(app->ctl){
-      scgi_header_add(app->ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
-      err_msg_len = sprintf(err_msg,"The application could not be started due to the following error. Please refer '/var/log/webroar/%s.log' and the application log file for more details.", app->conf->name.str);
-      scgi_body_add(app->ctl->scgi, err_msg, err_msg_len);
-      wr_ctl_resp_write(app->ctl); 
-    }
-    app->timeout_counter = 0;
-    app->ctl = NULL;
+    wr_app_add_error_msg(app);
     return;
   }else if(app->state == WR_APP_NEW){
-    // Try out upto MAX timeout count
-/*
-    if(WR_QUEUE_SIZE(app->q_pending_workers) == 0 && app->timeout_counter < WR_MAX_ADD_TIMEOUT_COUNTER){
-      wr_app_wkr_add(app);
-    }else{
-*/
     if(WR_QUEUE_SIZE(app->q_pending_workers) == 0){
       // Send error response
-      LOG_DEBUG(DEBUG,"Some problem occurred while starting Application %s.", app->conf->name.str);
-      if(app->ctl){
-        scgi_header_add(app->ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
-        err_msg_len = sprintf(err_msg,"The application could not be started due to the following error. Please refer '/var/log/webroar/%s.log' and the application log file for more details.", app->conf->name.str);
-        scgi_body_add(app->ctl->scgi, err_msg, err_msg_len);
-        wr_ctl_resp_write(app->ctl);        
-      }
-      app->timeout_counter = 0;
-      app->ctl = NULL;
+      wr_app_add_error_msg(app);
       wr_app_remove(app->svr, app->conf->name.str);
     }
     return;
@@ -160,8 +211,8 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
   LOG_DEBUG(DEBUG,"Pending_wkr = %d, app->q_workers->q_count = %d",
             WR_QUEUE_SIZE(app->q_pending_workers), WR_QUEUE_SIZE(app->q_workers));
   if(app->q_workers->q_count > app->conf->min_worker) {
-    char cmd[WR_LONG_LONG_STR_LEN];
-    char pid_list[WR_LONG_LONG_STR_LEN], pid_c[WR_SHORT_STR_LEN];
+    char cmd[STR_SIZE512];
+    char pid_list[STR_SIZE512], pid_c[STR_SIZE32];
     int i,index;
     i = 0;
     index = (app->q_workers->q_front + i) % app->q_workers->q_max_size;
@@ -170,7 +221,7 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
     
     // Get pid of the worker consuming more resident memory
 #ifdef __APPLE__
-    //sprintf(cmd,"ps -o pid -m -p %s | head -n2 | tail -n1 | cut -c-6 > %s",pid_list, WR_HIGH_RSS_PID_FILE);
+    //sprintf(cmd,"ps -o pid -m -p %s | head -n2 | tail -n1 | cut -c-6 > %s",pid_list, Config->Server.File.high_rss.str);
 	/* TODO: when any shell command is executed using system(), process goes into wait state. It is 
 	 observed on only Mac. When tried calling syste() at various interval like after port binding,
 	 controller initialization, forking required worker, daemonizing, activating event loop etc, in all 
@@ -178,7 +229,7 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 	 goes into infinite wait.	   
 	 In this case we would simply pick the first worker from queue and remove it.
 	 */
-    FILE *wfp = fopen(WR_HIGH_RSS_PID_FILE, "w");
+    FILE *wfp = fopen(Config->Server.File.high_rss.str, "w");
     if(wfp) {
       fprintf(wfp,"%d", tmp_worker->pid); 
       fclose(wfp);
@@ -193,18 +244,18 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
       sprintf(pid_c,",%d", tmp_worker->pid);
       strcat(pid_list, pid_c);
     }
-    sprintf(cmd,"ps -o pid --sort=rss -p %s | tail -n1 | cut -c-6 > %s",pid_list, WR_HIGH_RSS_PID_FILE);
+    sprintf(cmd,"ps -o pid --sort=rss -p %s | tail -n1 | cut -c-6 > %s",pid_list, Config->Server.File.high_rss.str);
     LOG_DEBUG(DEBUG,"Formed command to remove worker is %s",cmd);
     system(cmd);
 #endif
 	
     // Read pid from file
-    FILE *fp = fopen(WR_HIGH_RSS_PID_FILE, "r");
+    FILE *fp = fopen(Config->Server.File.high_rss.str, "r");
     if(fp) {
       unsigned pid = 0;
       fscanf(fp, "%u", &pid);
       fclose(fp);
-      remove(WR_HIGH_RSS_PID_FILE);
+      remove(Config->Server.File.high_rss.str);
       int flag = 1;
 
       // Check for worker in list of free workers. If found remove it.
@@ -217,7 +268,8 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
           if(tmp_worker->pid == pid) {
             LOG_DEBUG(DEBUG,"Removing from free worker id=%d", tmp_worker->id);
             forecasted_count--;
-            wr_wkr_remove(tmp_worker, 1);
+            tmp_worker->state = WKR_STATE_ERROR;
+            wr_wkr_free(tmp_worker);
             LOG_DEBUG(DEBUG,"Worker removed from free worker.");
             flag = 0;
             break;
@@ -236,8 +288,7 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
           if(tmp_worker->pid == pid) {
             forecasted_count--;
             LOG_DEBUG(DEBUG,"Remove active status id = %d", tmp_worker->id);
-            if(tmp_worker->state & WR_WKR_ACTIVE)
-              tmp_worker->state -= WR_WKR_ACTIVE;
+            tmp_worker->state = WKR_STATE_EXPIRED;
             break;
           }
         }
@@ -251,37 +302,19 @@ void wr_app_wkr_remove_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 
 /** Reload the application */
-static inline int wr_app_reload(wr_app_t *app){
+int wr_app_reload(wr_app_t *app){
   LOG_FUNCTION
   wr_wkr_t *worker;
-  wr_app_conf_t *app_conf;
   short count;
 
   // Remove an old application from the resolver list.
   wr_req_resolver_remove(app->svr, app);
 
   // Update the application configuration.
-  app_conf = wr_conf_app_update(app->svr->conf,
-                   app->conf->name.str,
-                   app->svr->err_msg);
-  if(app_conf == NULL){
-    LOG_DEBUG(WARN, "Error: %s",app->svr->err_msg);
-    if(app->ctl){
-      scgi_body_add(app->ctl->scgi, app->svr->err_msg, strlen(app->svr->err_msg));
-      scgi_header_add(app->ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
-      wr_ctl_resp_write(app->ctl);
-    }
-    app->state = WR_APP_ACTIVE;
-    return FALSE;
-  }
-
-  // Remove old application specification.
-  app->conf->next = NULL;
-  wr_conf_app_free(app->conf);
-  app->conf = app_conf;
-
+  wr_conf_app_update(app->conf);
+  
   // Add the updated application to resolver list.
-  wr_req_resolver_add(app->svr, app, app_conf);
+  wr_req_resolver_add(app->svr, app);
 
   // Remove workers based on following logic:
   // If all the workers are free keep a single worker to process the requests and remove all others.
@@ -291,85 +324,44 @@ static inline int wr_app_reload(wr_app_t *app){
   while(WR_QUEUE_SIZE(app->q_free_workers) > 0){
     worker = (wr_wkr_t*)wr_queue_fetch(app->q_free_workers);
     // The worker is already removed from free workers list so do not pass the flag.
-    wr_wkr_remove(worker, 0);
+    worker->state = WKR_STATE_ERROR;
+    wr_wkr_free(worker);
   }
 
   // Mark all existing workers to OLD worker.
   for(count = 0; count < WR_QUEUE_SIZE(app->q_workers) ; count++) {
     worker = (wr_wkr_t*)wr_queue_fetch(app->q_workers);
     wr_queue_insert(app->q_workers, worker);
-    worker->state |= WR_WKR_OLD;
-    worker->state -= WR_WKR_ACTIVE;
+    worker->state = WKR_STATE_EXPIRED;
   } 
 
   app->state = WR_APP_RESTARTING;
   return TRUE;
 }
 
-/*************** Application function definition *********/
-
-/** Destroy application */
-void wr_app_free(wr_app_t* app) {
-  LOG_FUNCTION
-  wr_app_t* tmp_app;
-  wr_wkr_t* worker;
-
-  while(app) {
-    tmp_app = app->next;
-    app->state = WR_APP_DESTROY;
-    LOG_DEBUG(4,"Destroying application %s...", app->conf->name.str);
-    LOG_DEBUG(DEBUG,"Worker count = %d", WR_QUEUE_SIZE(app->q_workers));
-    //Destroy workers
-    while(worker = (wr_wkr_t*)wr_queue_fetch(app->q_workers)) {
-      if(app->svr->is_running==0)
-        worker->state |= WR_WKR_HANG;
-      wr_wkr_free(worker);
-    }
-
-    wr_queue_free(app->q_free_workers);
-    wr_queue_free(app->q_workers);
-    wr_queue_free(app->q_pending_workers);
-
-    wr_req_t *req;
-    WR_QUEUE_FETCH(app->q_messages, req)  ;
-    while(req) {
-      wr_conn_err_resp(req->conn, WR_HTTP_STATUS_500);
-      WR_QUEUE_FETCH(app->q_messages, req)  ;
-    }
-
-    wr_queue_free(app->q_messages);
-
-    wr_req_resolver_remove(app->svr, app);
-
-    ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add);
-    ev_timer_stop(app->svr->ebb_svr.loop, &app->t_remove);
-    ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add_timeout);
-
-    free(app);
-    app = tmp_app;
-  }
-}
-
-/** Display application structure */
-void wr_app_print(wr_app_t*app) {
-  while(app) {
-    LOG_DEBUG(4,"Application %s", app->conf->name.str);
-    app = app->next;
-  }
-}
-
 /** Create worker for application */
 int wr_app_wkr_add(wr_app_t *app) {
   if(WR_QUEUE_SIZE(app->q_pending_workers) < WR_QUEUE_MAX_SIZE(app->q_pending_workers)) {
-    int retval = wr_wkr_create(app->svr, app->conf);
+    if(app->timeout_counter >= Config->Server.Worker.add_trials){
+      LOG_ERROR(SEVERE, "Could not fork worker because previous %d workers got timed out.",
+                Config->Server.Worker.add_trials);
+      return FALSE;
+    }
+    config_application_list_t *conf = app->conf;
+    
+    if(app->state == WR_APP_RESTART) conf = app->conf->new; 
+    
+    int retval = wr_wkr_create(app->svr, conf);
     if(retval > 0){
       wr_pending_wkr_t *pending = wr_malloc(wr_pending_wkr_t);
       *pending = retval;
       wr_queue_insert(app->q_pending_workers, pending);
-      app->high_ratio = TOTAL_WORKER_COUNT(app) * WR_MAX_REQ_RATIO;
-      ev_timer_again(app->svr->ebb_svr.loop, &app->t_add_timeout);
-      LOG_INFO("PID of created worker = %d, Rails application=%s",
-               retval, app->conf->path.str);
+      app->high_ratio = TOTAL_WORKER_COUNT(app) * Config->Application.max_req_ratio;
+      if (Config->Server.Worker.add_timeout > 0) {
+        ev_timer_again(app->svr->ebb_svr.loop, &app->t_add_timeout);
+      }
+      
+      LOG_INFO("PID of created worker = %d", retval);
       return TRUE;
     }else{
       LOG_ERROR(SEVERE,"Could not fork process to start new worker.");
@@ -379,23 +371,23 @@ int wr_app_wkr_add(wr_app_t *app) {
 }
 
 /** Insert application based on application configuration */
-static int wr_app_insert(wr_svr_t* server, wr_app_conf_t* config, wr_ctl_t *ctl) {
+int wr_app_insert(wr_svr_t* server, config_application_list_t* config, wr_ctl_t *ctl) {
   LOG_FUNCTION
   wr_app_t* app = wr_malloc(wr_app_t);
-
+  
   if(!app) {
     LOG_ERROR(WARN, "%s() application object allocation failed. Returning ...", __FUNCTION__);
     return FALSE;
   }
-
-  // Queue size is WR_ALLOWED_MAX_WORKERS + 1 to accommodate temporary extra 
+  
+  // Queue size is Config->Server.Worker.max + 1 to accommodate temporary extra 
   // worker, created during application restart 
-  app->q_free_workers     = wr_queue_new(WR_ALLOWED_MAX_WORKERS + 1);
-  app->q_workers         = wr_queue_new(WR_ALLOWED_MAX_WORKERS + 1);
-  app->q_pending_workers = wr_queue_new(WR_MAX_PENDING_WKR);
-
-  app->q_messages        = wr_queue_new(WR_MSG_QUE_SIZE);
-
+  app->q_free_workers     = wr_queue_new(Config->Server.Worker.max + 1);
+  app->q_workers         = wr_queue_new(Config->Server.Worker.max + 1);
+  app->q_pending_workers = wr_queue_new(Config->Server.Worker.pending);
+  
+  app->q_messages        = wr_queue_new(Config->Application.msg_queue_size);
+  
   if(app->q_workers == NULL || app->q_pending_workers == NULL ||
      app->q_free_workers == NULL || app->q_messages == NULL) {
     free(app);
@@ -415,19 +407,21 @@ static int wr_app_insert(wr_svr_t* server, wr_app_conf_t* config, wr_ctl_t *ctl)
   /* set application object in control, it would be used at time of freeing control object */
   if(ctl)    ctl->app = app;
   
-  if(strcmp(config->name.str, WR_STATIC_FILE_SERVER_NAME) == 0){
+  if(strcmp(config->name.str, Config->Application.Static_server.name.str) == 0){
     app->next = NULL;
     server->static_app = app;
   }else{
-    wr_req_resolver_add(server, app, config);
+    wr_req_resolver_add(server, app);
     app->next = server->apps;
     server->apps = app;
   }
-
-  ev_timer_init (&app->t_add, wr_app_wkr_add_cb, 0., WR_HIGH_LOAD_LIMIT);
-  ev_timer_init (&app->t_remove, wr_app_wkr_remove_cb, 0., WR_LOW_LOAD_LIMIT);
-  ev_timer_init (&app->t_add_timeout, wr_app_wkr_add_timeout_cb, 0., WR_WKR_ADD_TIMEOUT);
-
+  
+  ev_timer_init (&app->t_add, wr_app_wkr_add_cb, 0., Config->Application.high_load);
+  ev_timer_init (&app->t_remove, wr_app_wkr_remove_cb, 0., Config->Application.low_load);
+  if (Config->Server.Worker.add_timeout > 0) {
+    ev_timer_init (&app->t_add_timeout, wr_app_wkr_add_timeout_cb, 0., Config->Server.Worker.add_timeout);
+  }
+  
   LOG_DEBUG(4,"%s() Application Added:%s", __FUNCTION__, config->name.str);
 
   wr_app_wkr_balance(app);
@@ -435,17 +429,73 @@ static int wr_app_insert(wr_svr_t* server, wr_app_conf_t* config, wr_ctl_t *ctl)
   return TRUE;
 }
 
+/*************** Application function definition *********/
+
+/** Destroy application */
+void wr_app_free(wr_app_t* app) {
+  LOG_FUNCTION
+  wr_app_t* tmp_app;
+  wr_wkr_t* worker;
+
+  while(app) {
+    tmp_app = app->next;
+    app->state = WR_APP_DESTROY;
+    LOG_DEBUG(4,"Destroying application %s...", app->conf->name.str);
+    LOG_DEBUG(DEBUG,"Worker count = %d", WR_QUEUE_SIZE(app->q_workers));
+    //Destroy workers
+    while(worker = (wr_wkr_t*)wr_queue_fetch(app->q_workers)) {
+      if(app->svr->is_running == 0)
+        worker->state = WKR_STATE_ERROR;
+      wr_wkr_free(worker);
+    }
+
+    wr_queue_free(app->q_free_workers);
+    wr_queue_free(app->q_workers);
+    wr_queue_free(app->q_pending_workers);
+
+    wr_req_t *req;
+    WR_QUEUE_FETCH(app->q_messages, req)  ;
+    while(req) {
+      wr_conn_err_resp(req->conn, WR_HTTP_STATUS_500);
+      WR_QUEUE_FETCH(app->q_messages, req)  ;
+    }
+
+    wr_queue_free(app->q_messages);
+
+    wr_req_resolver_remove(app->svr, app);
+
+    ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add);
+    ev_timer_stop(app->svr->ebb_svr.loop, &app->t_remove);
+    if (Config->Server.Worker.add_timeout > 0) {
+      ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add_timeout);
+    }
+    free(app);
+    app = tmp_app;
+  }
+}
+
+/** Display application structure */
+void wr_app_print(wr_app_t*app) {
+  while(app) {
+    LOG_DEBUG(4,"Application %s", app->conf->name.str);
+    app = app->next;
+  }
+}
+
 /** Balance number of workers */
 void wr_app_wkr_balance(wr_app_t *app){
   // Maintain minimum number of workers
-  //while(TOTAL_WORKER_COUNT(app) < app->conf->min_worker && app->timeout_counter < WR_MAX_ADD_TIMEOUT_COUNTER){
+  //while(TOTAL_WORKER_COUNT(app) < app->conf->min_worker && app->timeout_counter < Config->Server.Worker.add_trials){
   while(TOTAL_WORKER_COUNT(app) < app->conf->min_worker){
     if(wr_app_wkr_add(app) == FALSE)   break;
-    app->low_ratio = TOTAL_WORKER_COUNT(app) * WR_MIN_REQ_RATIO;
+    app->low_ratio = TOTAL_WORKER_COUNT(app) * Config->Application.min_req_ratio;
   }
 
-  if(WR_QUEUE_SIZE(app->q_workers) >= app->conf->min_worker && app->state == WR_APP_RESTART)
-    app->state == WR_APP_ACTIVE;
+  if(WR_QUEUE_SIZE(app->q_workers) >= app->conf->min_worker && app->state == WR_APP_RESTART){
+    app->state = WR_APP_ACTIVE;
+    wr_application_list_free(app->conf->new);
+    app->conf->new = NULL;
+  }
   
   // Create worker if application is high loaded
 /*
@@ -455,6 +505,37 @@ void wr_app_wkr_balance(wr_app_t *app){
 */
 }
 
+/** Got worker add error */
+int wr_app_wkr_error(wr_svr_t *server, const wr_ctl_msg_t *ctl_msg) {
+  LOG_FUNCTION
+  const char* app_name = ctl_msg->msg.wkr.app_name.str;
+  wr_app_t* app = wr_app_exist(server, app_name);
+  
+  if(app == NULL){
+    return -1;
+  }
+  
+  if(app->state == WR_APP_RESTART){    
+    app->state = WR_APP_ACTIVE;
+    wr_application_list_free(app->conf->new);
+    app->conf->new = NULL;
+    // Send error response
+    wr_app_add_error_msg(app);
+  }else if(app->state == WR_APP_NEW){
+    // Send error response
+    wr_app_add_error_msg(app);
+    wr_app_remove(app->svr, app->conf->name.str);
+  }else{
+    wr_pending_wkr_t *pending = wr_pending_worker_exist(app, atoi(ctl_msg->msg.wkr.pid.str));
+    if(pending != NULL)     free(pending);
+    app->timeout_counter = 0;
+      
+    if(WR_QUEUE_SIZE(app->q_pending_workers) <= 0 && Config->Server.Worker.add_timeout > 0)
+      ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add_timeout);
+  }
+  return 0;
+}
+
 /** Add newly created worker to application */
 int wr_app_wkr_insert(wr_svr_t *server, wr_wkr_t *worker,const wr_ctl_msg_t *ctl_msg) {
   LOG_FUNCTION
@@ -478,15 +559,16 @@ int wr_app_wkr_insert(wr_svr_t *server, wr_wkr_t *worker,const wr_ctl_msg_t *ctl
       scgi_body_add(worker->ctl->scgi, "Either worker add timeout or worker PID does not match.",
                     strlen("Either worker add timeout or worker PID does not match."));
       return -1;      
+    }else{
+      free(pending);
     }
 
     worker->id = ++worker_count;
     worker->app = app;
-    if(!(worker->state & WR_WKR_ACTIVE))      worker->state += WR_WKR_ACTIVE;
-
+    
     app->timeout_counter = 0;
       
-    if(WR_QUEUE_SIZE(app->q_pending_workers) <= 0)
+    if(WR_QUEUE_SIZE(app->q_pending_workers) <= 0 && Config->Server.Worker.add_timeout > 0)
       ev_timer_stop(app->svr->ebb_svr.loop, &app->t_add_timeout);
 
     if(app->state == WR_APP_RESTART){
@@ -516,37 +598,6 @@ int wr_app_wkr_insert(wr_svr_t *server, wr_wkr_t *worker,const wr_ctl_msg_t *ctl
   return -1;
 }
 
-/** Remove application from application list */
-int wr_app_remove(wr_svr_t* server, const char* app_name) {
-  LOG_FUNCTION
-  wr_app_t* app = server->apps, *tmp_app = NULL;
-
-  LOG_DEBUG(DEBUG, "Removing application %s", app_name);
-  
-  while(app) {
-    if(strcmp(app_name, app->conf->name.str)==0)
-      break;
-    tmp_app = app;
-    app = app->next;
-  }
-  if(app) {
-    if(tmp_app) {      
-      tmp_app->next = app->next;
-    } else {
-      server->apps = app->next;
-    }
-
-    app->next = NULL;
-    wr_app_free(app);
-    wr_app_conf_remove(server->conf, app_name);
-    return 0;
-  } else {
-    LOG_ERROR(WARN,"Aapplication %s didn't found in list", app_name);
-    sprintf(server->err_msg, "Application '%s' is not found.", app_name);
-    return -1;
-  }
-}
-
 /** Check load balance to add the worker */
 void wr_app_chk_load_to_add_wkr(wr_app_t *app) {
   if(TOTAL_WORKER_COUNT(app) < app->conf->max_worker) {
@@ -581,7 +632,7 @@ void wr_app_chk_load_to_remove_wkr(wr_app_t *app) {
 /** Initialize the applications */
 void wr_app_init(wr_svr_t *server) {
   LOG_FUNCTION
-  wr_app_conf_t *app = server->conf->apps;
+  config_application_list_t *app = Config->Application.list;
 
   while(app) {
     wr_app_insert(server, app, NULL);
@@ -595,7 +646,7 @@ void wr_app_add_cb(wr_ctl_t *ctl, const wr_ctl_msg_t *ctl_msg) {
   LOG_FUNCTION
 
   wr_svr_t* server = ctl->svr;
-  wr_app_conf_t* app_conf = NULL;
+  config_application_list_t* app_conf = NULL;
   wr_app_t* app = wr_app_exist(server, ctl_msg->msg.app.app_name.str);
   
   // Reset the error message
@@ -604,10 +655,9 @@ void wr_app_add_cb(wr_ctl_t *ctl, const wr_ctl_msg_t *ctl_msg) {
      /* set application object in control, it would be used at time of freeing control object */
      ctl->app = app;
      sprintf(ctl->svr->err_msg, "Application '%s' is already running.", ctl_msg->msg.app.app_name.str); 
-  }else if(ctl && server && server->conf) {
-    app_conf = wr_conf_app_read(server->conf,
-                           ctl_msg->msg.app.app_name.str,
-                           ctl->svr->err_msg);
+  }else if(ctl && server) {
+    app_conf = wr_conf_app_read(ctl_msg->msg.app.app_name.str,
+                           ctl->svr->err_msg, FALSE);
     if(app_conf!=NULL) {
       if(wr_app_insert(ctl->svr, app_conf, ctl) == TRUE)        return;
     } else if(ctl->svr->err_msg[0] == 0) {
@@ -635,64 +685,72 @@ void wr_app_remove_cb(wr_ctl_t *ctl, const wr_ctl_msg_t *ctl_msg) {
 /** Allication reload callback */
 void wr_app_reload_cb(wr_ctl_t *ctl, const wr_ctl_msg_t *ctl_msg){
   LOG_FUNCTION
-  wr_app_conf_t* app_config = NULL;
   wr_app_t *app = wr_app_exist(ctl->svr, ctl_msg->msg.app.app_name.str);
 
-  // Read new application configuration.
-  app_config = wr_conf_app_update(ctl->svr->conf,
-                                  ctl_msg->msg.app.app_name.str,
-                                  ctl->svr->err_msg);
-
-  LOG_INFO("Reload the application %s", ctl_msg->msg.app.app_name.str);
-  // Report error on not getting the application configuration.
-  if(app_config == NULL){
-    LOG_ERROR(WARN, "Error: %s",ctl->svr->err_msg);
-    scgi_body_add(ctl->scgi, ctl->svr->err_msg, strlen(ctl->svr->err_msg));
-    scgi_header_add(ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
-    wr_ctl_resp_write(ctl);
-    // Add old application configuration to server configuration.
-    if(app){
-      LOG_DEBUG(WARN,"Replace the application configuration with old one.");
-      wr_conf_app_replace(app->svr->conf, app->conf);
-    }
-    return;
-  }
-  
   if(app){
-    int i;
-    wr_app_conf_t *tmp_app_conf = app->conf;
-    // Set variables to restart the application.
-    LOG_DEBUG(DEBUG,"Set variables to restart an existing application.");
-    app->conf = app_config;
+    
+    LOG_INFO("Reload the application %s", ctl_msg->msg.app.app_name.str);
+    // Read new application configuration.
+    // Report error on not getting the application configuration.
+    if(wr_conf_app_read(ctl_msg->msg.app.app_name.str, ctl->svr->err_msg, TRUE) == NULL){
+      LOG_ERROR(WARN, "Error: %s",ctl->svr->err_msg);
+      scgi_body_add(ctl->scgi, ctl->svr->err_msg, strlen(ctl->svr->err_msg));
+      scgi_header_add(ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
+      wr_ctl_resp_write(ctl);
+      return;
+    }
+    
     app->state = WR_APP_RESTART;
+    app->timeout_counter = 0;
     while(WR_QUEUE_SIZE(app->q_pending_workers) > 0){
       wr_pending_wkr_t* pending = wr_queue_fetch(app->q_pending_workers);
       free(pending);
     }
 
     app->ctl = ctl;
-    LOG_DEBUG(4,"%s() Application Added:%s", __FUNCTION__, app->conf->name.str);
+    LOG_DEBUG(4,"%s() Application Added:%s", __FUNCTION__, app->conf->new->name.str);
     
     // Add single worker with updated application.
     LOG_DEBUG(DEBUG, "Add first worker on application restart.");
-    wr_app_wkr_add(app);
-    
-    // Replace the application configuration with older configuration object.
-    wr_conf_app_replace(app->svr->conf, tmp_app_conf);
-    app->conf = tmp_app_conf;
+    wr_app_wkr_add(app);   
     return;
   }else{
     // If application didn't found, report an error and create new application.
     LOG_ERROR(WARN,"Aapplication %s didn't found in list", ctl_msg->msg.app.app_name.str);
     sprintf(ctl->svr->err_msg, "Application '%s' is not found.", ctl_msg->msg.app.app_name.str);
-    scgi_body_add(ctl->scgi,
-                  "Couldn't remove application. But trying to start application.",
+    scgi_body_add(ctl->scgi, "Couldn't remove application. But trying to start application.",
                   strlen("Couldn't remove application. But trying to start application."));
-    if(wr_app_insert(ctl->svr, app_config, ctl) == TRUE)     return;
+
+    config_application_list_t *app_config = wr_conf_app_read(ctl_msg->msg.app.app_name.str, ctl->svr->err_msg, FALSE);
+    if(app_config){
+      if(wr_app_insert(ctl->svr, app_config, ctl) == TRUE)     return;
+    }else{
+      LOG_ERROR(WARN, "Error: %s",ctl->svr->err_msg);
+      scgi_body_add(ctl->scgi, ctl->svr->err_msg, strlen(ctl->svr->err_msg));
+    }
   }
 
   // Return ERROR status.
   scgi_header_add(ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
   wr_ctl_resp_write(ctl);
 }
-  
+
+/** Application configuration requset */
+void wr_app_conf_req_cb(wr_ctl_t *ctl, const wr_ctl_msg_t *ctl_msg){
+  config_application_list_t* app_conf = NULL;
+  wr_app_t* app = wr_app_exist(ctl->svr, ctl_msg->msg.app.app_name.str);
+
+  if(app && app->conf->scgi){
+    scgi_build(ctl->scgi);
+    scgi_free(ctl->scgi);
+    ctl->destroy_scgi = FALSE;
+    if(app->state == WR_APP_RESTART){
+      ctl->scgi = app->conf->new->scgi;
+    }else{ 
+      ctl->scgi = app->conf->scgi;
+    }
+  }else{
+    scgi_header_add(ctl->scgi, "STATUS", strlen("STATUS"), "ERROR", strlen("ERROR"));
+  }
+  wr_ctl_resp_write(ctl);
+}