iodine 0.4.19 → 0.5.0

data/ext/iodine/extconf.rb

@@ -33,7 +33,7 @@ else
   puts 'using an unknown (old?) compiler... who knows if this will work out... we hope.'
 end
 
-$CFLAGS = '-std=c11 -O3 -Wall -DSERVER_DELAY_IO=1 -DNO_CHILD_REAPER=1'
+$CFLAGS = "-std=c11 -O2 -Wall #{ENV['CFLAGS']}"
 RbConfig::MAKEFILE_CONFIG['CC'] = $CC = ENV['CC'] if ENV['CC']
 RbConfig::MAKEFILE_CONFIG['CPP'] = $CPP = ENV['CPP'] if ENV['CPP']
 

data/ext/iodine/facil.c

@@ -8,14 +8,24 @@ Feel free to copy, use and enjoy according to the license provided.
 
 #include "evio.h"
 #include "facil.h"
-#include "fio_list.h"
+#include "fio_hashmap.h"
+#include "fiobj4sock.h"
+
+#include "fio_mem.h"
 
 #include <errno.h>
+#include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#if !defined(__GNUC__) && !defined(__clang__)
+#define __attribute__(...)
+#endif
 
 /* *****************************************************************************
 Data Structures
@@ -35,31 +45,34 @@ struct connection_data_s {
   protocol_s *protocol;
   time_t active;
   uint8_t timeout;
+  spn_lock_i scheduled;
   spn_lock_i lock;
 };
 
 static struct facil_data_s {
   spn_lock_i global_lock;
   uint8_t need_review;
-  ssize_t capacity;
-  time_t last_cycle;
+  uint8_t spindown;
+  uint16_t active;
+  uint16_t threads;
   pid_t parent;
+  pool_pt thread_pool;
+  ssize_t capacity;
+  size_t connection_count;
+  void (*on_idle)(void);
+  void (*on_finish)(void);
+  struct timespec last_cycle;
   struct connection_data_s conn[];
 } * facil_data;
 
 #define fd_data(fd) (facil_data->conn[(fd)])
 #define uuid_data(uuid) fd_data(sock_uuid2fd((uuid)))
-#define uuid_prt_meta(uuid) prt_meta(uuid_data((uuid)).protocol)
+// #define uuid_prt_meta(uuid) prt_meta(uuid_data((uuid)).protocol)
 
-static inline void clear_connection_data_unsafe(intptr_t uuid,
-                                                protocol_s *protocol) {
-  uuid_data(uuid) = (struct connection_data_s){.active = facil_data->last_cycle,
-                                               .protocol = protocol,
-                                               .lock = uuid_data(uuid).lock};
-}
 /** locks a connection's protocol returns a pointer that need to be unlocked. */
 inline static protocol_s *protocol_try_lock(intptr_t fd,
                                             enum facil_protocol_lock_e type) {
+  errno = 0;
   if (spn_trylock(&fd_data(fd).lock))
     goto would_block;
   protocol_s *pr = fd_data(fd).protocol;
@@ -84,6 +97,146 @@ inline static void protocol_unlock(protocol_s *pr,
   spn_unlock(&prt_meta(pr).locks[type]);
 }
 
+/* *****************************************************************************
+Internal Protocol Names
+***************************************************************************** */
+static const char *LISTENER_PROTOCOL_NAME =
+    "listening protocol __facil_internal__";
+
+static const char *CONNECTOR_PROTOCOL_NAME = "connect protocol __internal__";
+
+static const char *TIMER_PROTOCOL_NAME = "timer protocol __facil_internal__";
+
+static const char *CLUSTER_LISTEN_PROTOCOL_NAME =
+    "cluster listening protocol __facil_internal__";
+
+static const char *CLUSTER_CONNECTION_PROTOCOL_NAME =
+    "cluster connection __facil_internal__";
+
+static inline int is_counted_protocol(protocol_s *p) {
+  return p->service != TIMER_PROTOCOL_NAME &&
+         p->service != CLUSTER_LISTEN_PROTOCOL_NAME &&
+         p->service != CLUSTER_CONNECTION_PROTOCOL_NAME;
+}
+
+/* *****************************************************************************
+Event deferring (declarations)
+***************************************************************************** */
+
+static void deferred_on_close(void *uuid_, void *pr_);
+static void deferred_on_shutdown(void *arg, void *arg2);
+static void deferred_on_ready(void *arg, void *arg2);
+static void deferred_on_data(void *uuid, void *arg2);
+static void deferred_ping(void *arg, void *arg2);
+
+/* *****************************************************************************
+Overriding `defer` to use `evio` when waiting for events
+***************************************************************************** */
+
+/* if the FIO_DEDICATED_SYSTEM is defined threads are activated more often. */
+#if FIO_DEDICATED_SYSTEM
+void defer_thread_wait(pool_pt pool, void *p_thr) {
+  (void)pool;
+  (void)p_thr;
+  evio_wait(EVIO_TICK);
+}
+#endif
+
+/* *****************************************************************************
+Event Handlers (evio)
+***************************************************************************** */
+void sock_flush_defer(void *arg, void *ignored) {
+  (void)ignored;
+  switch (sock_flush((intptr_t)arg)) {
+  case 1:
+    evio_add_write(sock_uuid2fd((intptr_t)arg), (void *)arg);
+    break;
+  case 0:
+    defer(deferred_on_ready, arg, NULL);
+    break;
+  }
+}
+
+void evio_on_ready(void *arg) { defer(sock_flush_defer, arg, NULL); }
+void evio_on_close(void *arg) { sock_force_close((intptr_t)arg); }
+void evio_on_error(void *arg) { sock_force_close((intptr_t)arg); }
+void evio_on_data(void *arg) { defer(deferred_on_data, arg, NULL); }
+
+/* *****************************************************************************
+Mock Protocol Callbacks and Service Funcions
+***************************************************************************** */
+static void mock_on_ev(intptr_t uuid, protocol_s *protocol) {
+  (void)uuid;
+  (void)protocol;
+}
+
+static void mock_on_data(intptr_t uuid, protocol_s *protocol) {
+  facil_quite(uuid);
+  (void)protocol;
+}
+
+static void mock_on_close(intptr_t uuid, protocol_s *protocol) {
+  (void)(protocol);
+  (void)(uuid);
+}
+
+static void mock_on_finish(intptr_t uuid, void *udata) {
+  (void)(udata);
+  (void)(uuid);
+}
+
+static void mock_ping(intptr_t uuid, protocol_s *protocol) {
+  (void)(protocol);
+  sock_force_close(uuid);
+}
+
+static void mock_idle(void) {}
+
+/* Support for the default pub/sub cluster engine */
+#pragma weak pubsub_cluster_init
+void pubsub_cluster_init(void) {}
+#pragma weak pubsub_cluster_on_fork_start
+void pubsub_cluster_on_fork_start(void) {}
+#pragma weak pubsub_cluster_on_fork_end
+void pubsub_cluster_on_fork_end(void) {}
+#pragma weak pubsub_cluster_cleanup
+void pubsub_cluster_cleanup(void) {}
+
+/* other cleanup concern */
+#pragma weak fio_cli_end
+void fio_cli_end(void) {}
+
+/* perform initialization for external services. */
+static void facil_external_init(void) {
+  sock_on_fork();
+  pubsub_cluster_on_fork_start();
+  fio_malloc_after_fork();
+  defer_on_fork();
+}
+
+/* perform stage 2 initialization for external services. */
+static void facil_external_init2(void) { pubsub_cluster_on_fork_end(); }
+
+/* perform cleanup for external services. */
+static void facil_external_cleanup(void) {}
+
+#pragma weak http_lib_init
+void http_lib_init(void) {}
+#pragma weak http_lib_cleanup
+void http_lib_cleanup(void) {}
+
+/* perform initialization for external services. */
+static void facil_external_root_init(void) {
+  http_lib_init();
+  pubsub_cluster_init();
+}
+/* perform cleanup for external services. */
+static void facil_external_root_cleanup(void) {
+  http_lib_cleanup();
+  pubsub_cluster_cleanup();
+  fio_cli_end();
+}
+
 /* *****************************************************************************
 Deferred event handlers
 ***************************************************************************** */
@@ -97,54 +250,80 @@ postpone:
   defer(deferred_on_close, uuid_, pr_);
 }
 
-static void deferred_on_ready(void *arg, void *arg2) {
-  if (!uuid_data(arg).protocol)
+static void deferred_on_shutdown(void *arg, void *arg2) {
+  if (!uuid_data(arg).protocol) {
     return;
-  protocol_s *pr = protocol_try_lock(sock_uuid2fd(arg), FIO_PR_LOCK_WRITE);
-  if (!pr)
+  }
+  protocol_s *pr = protocol_try_lock(sock_uuid2fd(arg), FIO_PR_LOCK_TASK);
+  if (!pr) {
+    if (errno == EBADF)
+      return;
     goto postpone;
-  pr->on_ready((intptr_t)arg, pr);
-  protocol_unlock(pr, FIO_PR_LOCK_WRITE);
+  }
+  uuid_data(arg).active = facil_data->last_cycle.tv_sec;
+  uuid_data(arg).timeout = 8;
+  /* TODO: 0.7.0 catch the return valuse to set timeout and maybe keep open */
+  pr->on_shutdown((intptr_t)arg, pr);
+  pr->ping = mock_ping;
+  protocol_unlock(pr, FIO_PR_LOCK_TASK);
+  sock_close((intptr_t)arg);
   return;
 postpone:
-  defer(deferred_on_ready, arg, NULL);
+  defer(deferred_on_shutdown, arg, NULL);
   (void)arg2;
 }
 
-static void deferred_on_shutdown(void *arg, void *arg2) {
-  if (!uuid_data(arg).protocol)
+static void deferred_on_ready(void *arg, void *arg2) {
+  if (!uuid_data(arg).protocol) {
     return;
+  }
   protocol_s *pr = protocol_try_lock(sock_uuid2fd(arg), FIO_PR_LOCK_WRITE);
-  if (!pr)
+  if (!pr) {
+    if (errno == EBADF)
+      return;
     goto postpone;
-  pr->on_shutdown((intptr_t)arg, pr);
+  }
+  pr->on_ready((intptr_t)arg, pr);
   protocol_unlock(pr, FIO_PR_LOCK_WRITE);
-  sock_close((intptr_t)arg);
   return;
 postpone:
-  defer(deferred_on_shutdown, arg, NULL);
+  defer(deferred_on_ready, arg, NULL);
   (void)arg2;
 }
 
-static void deferred_on_data(void *arg, void *arg2) {
-  if (!uuid_data(arg).protocol)
+static void deferred_on_data(void *uuid, void *arg2) {
+  if (!uuid_data(uuid).protocol || sock_isclosed((intptr_t)uuid)) {
     return;
-  protocol_s *pr = protocol_try_lock(sock_uuid2fd(arg), FIO_PR_LOCK_TASK);
-  if (!pr)
+  }
+  protocol_s *pr = protocol_try_lock(sock_uuid2fd(uuid), FIO_PR_LOCK_TASK);
+  if (!pr) {
+    if (errno == EBADF)
+      return;
     goto postpone;
-  pr->on_data((intptr_t)arg, pr);
+  }
+  spn_unlock(&uuid_data(uuid).scheduled);
+  pr->on_data((intptr_t)uuid, pr);
   protocol_unlock(pr, FIO_PR_LOCK_TASK);
+  if (!spn_trylock(&uuid_data(uuid).scheduled)) {
+    evio_add_read(sock_uuid2fd((intptr_t)uuid), uuid);
+  }
   return;
 postpone:
-  defer(deferred_on_data, arg, NULL);
-  (void)arg2;
+  if (arg2) {
+    /* the event is being forced, so force rescheduling */
+    defer(deferred_on_data, (void *)uuid, (void *)1);
+  } else {
+    /* the protocol was locked, so there might not be any need for the event */
+    evio_add_read(sock_uuid2fd((intptr_t)uuid), uuid);
+  }
+  return;
 }
 
 static void deferred_ping(void *arg, void *arg2) {
   if (!uuid_data(arg).protocol ||
       (uuid_data(arg).timeout &&
        (uuid_data(arg).timeout >
-        (facil_data->last_cycle - uuid_data(arg).active)))) {
+        (facil_data->last_cycle.tv_sec - uuid_data(arg).active)))) {
     return;
   }
   protocol_s *pr = protocol_try_lock(sock_uuid2fd(arg), FIO_PR_LOCK_WRITE);
@@ -158,22 +337,6 @@ postpone:
   (void)arg2;
 }
 
-/* *****************************************************************************
-Event Handlers (evio)
-***************************************************************************** */
-static void sock_flush_defer(void *arg, void *ignored) {
-  (void)ignored;
-  sock_flush((intptr_t)arg);
-}
-
-void evio_on_ready(void *arg) {
-  defer(sock_flush_defer, arg, NULL);
-  defer(deferred_on_ready, arg, NULL);
-}
-void evio_on_close(void *arg) { sock_force_close((intptr_t)arg); }
-void evio_on_error(void *arg) { sock_force_close((intptr_t)arg); }
-void evio_on_data(void *arg) { defer(deferred_on_data, arg, NULL); }
-
 /* *****************************************************************************
 Forcing IO events
 ***************************************************************************** */
@@ -181,7 +344,8 @@ Forcing IO events
 void facil_force_event(intptr_t uuid, enum facil_io_event ev) {
   switch (ev) {
   case FIO_EVENT_ON_DATA:
-    evio_on_data((void *)uuid);
+    spn_trylock(&uuid_data(uuid).scheduled);
+    defer(deferred_on_data, (void *)uuid, (void *)1);
     break;
   case FIO_EVENT_ON_TIMEOUT:
     defer(deferred_ping, (void *)uuid, NULL);
@@ -192,21 +356,41 @@ void facil_force_event(intptr_t uuid, enum facil_io_event ev) {
   }
 }
 
+/**
+ * Temporarily prevents `on_data` events from firing.
+ *
+ * The `on_data` event will be automatically rescheduled when (if) the socket's
+ * outgoing buffer fills up or when `facil_force_event` is called with
+ * `FIO_EVENT_ON_DATA`.
+ */
+void facil_quite(intptr_t uuid) {
+  if (sock_isvalid(uuid))
+    spn_trylock(&uuid_data(uuid).scheduled);
+}
+
 /* *****************************************************************************
 Socket callbacks
 ***************************************************************************** */
 
 void sock_on_close(intptr_t uuid) {
+  // fprintf(stderr, "INFO: facil.io, on-close called for %u (set to %p)\n",
+  //         (unsigned int)sock_uuid2fd(uuid), (void
+  //         *)uuid_data(uuid).protocol);
   spn_lock(&uuid_data(uuid).lock);
   protocol_s *old_protocol = uuid_data(uuid).protocol;
-  clear_connection_data_unsafe(uuid, NULL);
+  uuid_data(uuid) = (struct connection_data_s){.lock = uuid_data(uuid).lock};
   spn_unlock(&uuid_data(uuid).lock);
-  if (old_protocol)
+  if (old_protocol) {
+    if (is_counted_protocol(old_protocol)) {
+      spn_sub(&facil_data->connection_count, 1);
+    }
     defer(deferred_on_close, (void *)uuid, old_protocol);
+  }
 }
 
 void sock_touch(intptr_t uuid) {
-  uuid_data(uuid).active = facil_data->last_cycle;
+  if (facil_data && facil_data->active)
+    uuid_data(uuid).active = facil_data->last_cycle.tv_sec;
 }
 
 /* *****************************************************************************
@@ -214,13 +398,18 @@ Initialization and Cleanup
 ***************************************************************************** */
 static spn_lock_i facil_libinit_lock = SPN_LOCK_INIT;
 
+static void facil_cluster_cleanup(void); /* cluster data cleanup */
+
 static void facil_libcleanup(void) {
   /* free memory */
   spn_lock(&facil_libinit_lock);
   if (facil_data) {
+    facil_external_root_cleanup();
+    facil_cluster_cleanup();
+    defer_perform(); /* perform any lingering cleanup tasks */
     munmap(facil_data,
-           sizeof(*facil_data) +
-               (facil_data->capacity * sizeof(struct connection_data_s)));
+           sizeof(*facil_data) + ((size_t)facil_data->capacity *
+                                  sizeof(struct connection_data_s)));
     facil_data = NULL;
   }
   spn_unlock(&facil_libinit_lock);
@@ -228,19 +417,26 @@ static void facil_libcleanup(void) {
 
 static void facil_lib_init(void) {
   ssize_t capa = sock_max_capacity();
-  if (capa < 0)
-    perror("ERROR: socket capacity unknown / failure"), exit(ENOMEM);
+  if (capa < 0) {
+    perror("ERROR: socket capacity unknown / failure");
+    exit(ENOMEM);
+  }
   size_t mem_size =
-      sizeof(*facil_data) + (capa * sizeof(struct connection_data_s));
+      sizeof(*facil_data) + ((size_t)capa * sizeof(struct connection_data_s));
   spn_lock(&facil_libinit_lock);
   if (facil_data)
     goto finish;
   facil_data = mmap(NULL, mem_size, PROT_READ | PROT_WRITE | PROT_EXEC,
                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-  if (!facil_data)
-    perror("ERROR: Couldn't initialize the facil.io library"), exit(0);
+  if (!facil_data) {
+    perror("ERROR: Couldn't initialize the facil.io library");
+    exit(0);
+  }
   memset(facil_data, 0, mem_size);
-  *facil_data = (struct facil_data_s){.capacity = capa, .parent = getpid()};
+  *facil_data = (struct facil_data_s){
+      .capacity = capa, .parent = getpid(),
+  };
+  facil_external_root_init();
   atexit(facil_libcleanup);
 #ifdef DEBUG
   if (FACIL_PRINT_STATE)
@@ -248,127 +444,65 @@ static void facil_lib_init(void) {
             "Initialized the facil.io library.\n"
             "facil.io's memory footprint per connection == %lu Bytes X %lu\n"
             "=== facil.io's memory footprint: %lu ===\n\n",
-            sizeof(struct connection_data_s), facil_data->capacity, mem_size);
+            (unsigned long)sizeof(struct connection_data_s),
+            (unsigned long)facil_data->capacity, (unsigned long)mem_size);
 #endif
 finish:
   spn_unlock(&facil_libinit_lock);
-  time(&facil_data->last_cycle);
-}
-
-/* *****************************************************************************
-Mock Protocol Callbacks and Service Funcions
-***************************************************************************** */
-static void mock_on_ev(intptr_t uuid, protocol_s *protocol) {
-  (void)uuid;
-  (void)protocol;
-}
-
-static void mock_on_close(intptr_t uuid, protocol_s *protocol) {
-  (void)(protocol);
-  (void)(uuid);
-}
-
-static void mock_on_finish(intptr_t uuid, void *udata) {
-  (void)(udata);
-  (void)(uuid);
-}
-
-static void mock_ping(intptr_t uuid, protocol_s *protocol) {
-  (void)(protocol);
-  sock_force_close(uuid);
+  clock_gettime(CLOCK_REALTIME, &facil_data->last_cycle);
 }
-static void mock_idle(void) {}
-
-/* Support for the default pub/sub cluster engine */
-#pragma weak pubsub_cluster_init
-void pubsub_cluster_init(void) {}
 
-/*
- * Support for the (removed) file caching service.
+/** Sets to shutdown flag for the current process.
+ *
+ * If a cluster is used and this function is called for a worker, a new worker
+ * will respawn.
  *
- * It's possible to re-add the service from the `unused` folder.
+ * This MUST be signal safe (don't call any functions, just uswe flags).
  */
-#pragma weak fio_cfd_clear
-void fio_cfd_clear(void) {}
-
-/* perform initialization for external services. */
-static void facil_external_init(void) { pubsub_cluster_init(); }
-
-/* perform cleanup for external services. */
-static void facil_external_cleanup(void) { fio_cfd_clear(); }
+static void facil_stop(void) {
+  if (!facil_data)
+    return;
+  facil_data->active = 0;
+}
 
 /* *****************************************************************************
 The listenning protocol
 ***************************************************************************** */
 #undef facil_listen
 
-static const char *listener_protocol_name =
-    "listening protocol __facil_internal__";
-
 struct ListenerProtocol {
   protocol_s protocol;
-  protocol_s *(*on_open)(intptr_t uuid, void *udata);
+  void (*on_open)(void *uuid, void *udata);
   void *udata;
-  void *rw_udata;
-  sock_rw_hook_s *(*set_rw_hooks)(intptr_t uuid, void *udata);
-  void (*on_finish_rw)(intptr_t uuid, void *rw_udata);
   void (*on_start)(intptr_t uuid, void *udata);
   void (*on_finish)(intptr_t uuid, void *udata);
-  char port[16];
+  char *port;
+  char *address;
+  uint8_t quite;
 };
 
-static sock_rw_hook_s *listener_set_rw_hooks(intptr_t uuid, void *udata) {
-  (void)udata;
-  (void)uuid;
-  return NULL; /* (sock_rw_hook_s *)&SOCK_DEFAULT_HOOKS; */
-}
-
 static void listener_ping(intptr_t uuid, protocol_s *plistener) {
   // fprintf(stderr, "*** Listener Ping Called for %ld\n", sock_uuid2fd(uuid));
-  uuid_data(uuid).active = facil_data->last_cycle;
+  uuid_data(uuid).active = facil_data->last_cycle.tv_sec;
   return;
   (void)plistener;
 }
 
-/* called asynchronously after a new connection was opened */
-static void listener_deferred_on_open(void *uuid_, void *srv_uuid_) {
-  intptr_t uuid = (intptr_t)uuid_;
-  intptr_t srv_uuid = (intptr_t)srv_uuid_;
-  struct ListenerProtocol *listener =
-      (struct ListenerProtocol *)protocol_try_lock(sock_uuid2fd(srv_uuid),
-                                                   FIO_PR_LOCK_WRITE);
-  if (!listener) {
-    if (sock_isvalid(uuid) && sock_isvalid(srv_uuid))
-      defer(listener_deferred_on_open, uuid_, srv_uuid_);
-    else
-      sock_close(uuid);
-    return;
-  }
-  {
-    sock_rw_hook_s *hooks = listener->set_rw_hooks(uuid, listener->rw_udata);
-    if (hooks)
-      sock_rw_hook_set(uuid, hooks);
-  }
-  protocol_s *pr = listener->on_open(uuid, listener->udata);
-  protocol_unlock((protocol_s *)listener, FIO_PR_LOCK_WRITE);
-  facil_attach(uuid, pr);
-  if (!pr)
-    sock_close(uuid);
-}
-
 static void listener_on_data(intptr_t uuid, protocol_s *plistener) {
-  intptr_t new_client;
-  if ((new_client = sock_accept(uuid)) == -1) {
-    if (errno == ECONNABORTED || errno == ECONNRESET)
-      defer(deferred_on_data, (void *)uuid, NULL);
-    else if (errno != EWOULDBLOCK)
+  for (int i = 0; i < 4; ++i) {
+    intptr_t new_client = sock_accept(uuid);
+    if (new_client == -1) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN || errno == ECONNABORTED ||
+          errno == ECONNRESET)
+        return;
       perror("ERROR: socket accept error");
-    return;
+      return;
+    }
+    // to defer or not to defer...? TODO: answer the question
+    struct ListenerProtocol *listener = (struct ListenerProtocol *)plistener;
+    defer(listener->on_open, (void *)new_client, listener->udata);
   }
-  defer(listener_deferred_on_open, (void *)new_client, (void *)uuid);
-  defer(deferred_on_data, (void *)uuid, NULL);
-  return;
-  (void)plistener;
+  facil_force_event(uuid, FIO_EVENT_ON_DATA);
 }
 
 static void free_listenner(void *li) { free(li); }
@@ -376,10 +510,17 @@ static void free_listenner(void *li) { free(li); }
 static void listener_on_close(intptr_t uuid, protocol_s *plistener) {
   struct ListenerProtocol *listener = (void *)plistener;
   listener->on_finish(uuid, listener->udata);
-  listener->on_finish_rw(uuid, listener->rw_udata);
-  if (FACIL_PRINT_STATE)
-    fprintf(stderr, "* (%d) Stopped listening on port %s\n", getpid(),
-            listener->port);
+  if (FACIL_PRINT_STATE && facil_data->parent == getpid()) {
+    if (listener->port) {
+      fprintf(stderr, "* Stopped listening on port %s\n", listener->port);
+    } else {
+      fprintf(stderr, "* Stopped listening on Unix Socket %s\n",
+              listener->address);
+    }
+  }
+  if (!listener->port) {
+    unlink(listener->address);
+  }
   free_listenner(listener);
 }
 
@@ -388,41 +529,56 @@ listener_alloc(struct facil_listen_args settings) {
   if (!settings.on_start)
     settings.on_start = mock_on_finish;
   if (!settings.on_finish)
-    settings.on_finish = (void (*)(intptr_t, void *))mock_on_close;
-  if (!settings.on_finish_rw)
-    settings.on_finish_rw = mock_on_finish;
-  if (!settings.set_rw_hooks)
-    settings.set_rw_hooks = listener_set_rw_hooks;
-  struct ListenerProtocol *listener = malloc(sizeof(*listener));
+    settings.on_finish = mock_on_finish;
+  size_t port_len = 0;
+  size_t addr_len = 0;
+  if (settings.port) {
+    port_len = strlen(settings.port) + 1;
+  }
+  if (settings.address) {
+    addr_len = strlen(settings.address) + 1;
+  }
+  struct ListenerProtocol *listener =
+      malloc(sizeof(*listener) + addr_len + port_len);
+
   if (listener) {
     *listener = (struct ListenerProtocol){
-        .protocol.service = listener_protocol_name,
+        .protocol.service = LISTENER_PROTOCOL_NAME,
         .protocol.on_data = listener_on_data,
         .protocol.on_close = listener_on_close,
         .protocol.ping = listener_ping,
-        .on_open = settings.on_open,
+        .on_open = (void (*)(void *, void *))settings.on_open,
         .udata = settings.udata,
         .on_start = settings.on_start,
         .on_finish = settings.on_finish,
-        .on_finish_rw = settings.on_finish_rw,
-        .set_rw_hooks = settings.set_rw_hooks,
-        .rw_udata = settings.rw_udata,
     };
-    size_t tmp = strlen(settings.port);
-    memcpy(listener->port, settings.port, tmp + 1);
+    if (settings.port) {
+      listener->port = (char *)(listener + 1);
+      memcpy(listener->port, settings.port, port_len);
+    }
+    if (settings.address) {
+      listener->address = (char *)(listener + 1);
+      listener->address += port_len;
+      memcpy(listener->address, settings.address, addr_len);
+    }
     return listener;
   }
   return NULL;
 }
 
-inline static void listener_on_start(size_t fd) {
-  intptr_t uuid = sock_fd2uuid(fd);
-  if (uuid < 0)
-    fprintf(stderr, "ERROR: listening socket dropped?\n"), kill(0, SIGINT),
-        exit(4);
-  if (evio_add(fd, (void *)uuid) < 0)
-    perror("Couldn't register listening socket"), kill(0, SIGINT), exit(4);
-  fd_data(fd).active = facil_data->last_cycle;
+inline static void listener_on_start(int fd) {
+  intptr_t uuid = sock_fd2uuid((int)fd);
+  if (uuid < 0) {
+    fprintf(stderr, "ERROR: listening socket dropped?\n");
+    kill(0, SIGINT);
+    exit(4);
+  }
+  if (evio_add(fd, (void *)uuid) < 0) {
+    perror("Couldn't register listening socket");
+    kill(0, SIGINT);
+    exit(4);
+  }
+  fd_data(fd).active = facil_data->last_cycle.tv_sec;
   // call the on_init callback
   struct ListenerProtocol *listener =
       (struct ListenerProtocol *)uuid_data(uuid).protocol;
@@ -439,8 +595,14 @@ though a `srv_stop` function or when a SIGINT/SIGTERM is received).
 int facil_listen(struct facil_listen_args settings) {
   if (!facil_data)
     facil_lib_init();
-  if (settings.on_open == NULL || settings.port == NULL)
+  if (settings.on_open == NULL) {
+    errno = EINVAL;
     return -1;
+  }
+  if (!settings.port || settings.port[0] == 0 ||
+      (settings.port[0] == '0' && settings.port[1] == 0)) {
+    settings.port = NULL;
+  }
   intptr_t uuid = sock_listen(settings.address, settings.port);
   if (uuid == -1) {
     return -1;
@@ -451,8 +613,12 @@ int facil_listen(struct facil_listen_args settings) {
     sock_close(uuid);
     return -1;
   }
-  if (FACIL_PRINT_STATE)
-    fprintf(stderr, "* Listening on port %s\n", settings.port);
+  if (FACIL_PRINT_STATE && facil_data->parent == getpid()) {
+    if (settings.port)
+      fprintf(stderr, "* Listening on port %s\n", settings.port);
+    else
+      fprintf(stderr, "* Listening on Unix Socket at %s\n", settings.address);
+  }
   return 0;
 }
 
@@ -460,42 +626,32 @@ int facil_listen(struct facil_listen_args settings) {
 Connect (as client)
 ***************************************************************************** */
 
-static const char *connector_protocol_name = "connect protocol __internal__";
-
 struct ConnectProtocol {
   protocol_s protocol;
-  protocol_s *(*on_connect)(intptr_t uuid, void *udata);
+  void (*on_connect)(void *uuid, void *udata);
   void (*on_fail)(intptr_t uuid, void *udata);
-  sock_rw_hook_s *(*set_rw_hooks)(intptr_t uuid, void *udata);
   void *udata;
-  void *rw_udata;
   intptr_t uuid;
-  int opened;
+  uint8_t opened;
 };
 
 /* The first `ready` signal is fired when a connection was established */
 static void connector_on_ready(intptr_t uuid, protocol_s *_connector) {
   struct ConnectProtocol *connector = (void *)_connector;
-  // fprintf(stderr, "connector_on_ready called\n");
-  if (!connector->opened) {
-    sock_rw_hook_s *hooks = connector->set_rw_hooks(uuid, connector->rw_udata);
-    if (hooks)
-      sock_rw_hook_set(uuid, hooks);
-  }
-  connector->opened = 1;
   sock_touch(uuid);
-  if (facil_attach(uuid, connector->on_connect(uuid, connector->udata)) == -1)
-    goto error;
-  uuid_data(uuid).protocol->on_ready(uuid, uuid_data(uuid).protocol);
+  if (connector->opened == 0) {
+    connector->opened = 1;
+    facil_set_timeout(uuid, 0); /* remove connection timeout settings */
+    connector->on_connect((void *)uuid, connector->udata);
+    evio_add_write(sock_uuid2fd(uuid), (void *)uuid);
+  }
   return;
-error:
-  sock_close(uuid);
 }
 
 /* If data events reach this protocol, delay their execution. */
 static void connector_on_data(intptr_t uuid, protocol_s *connector) {
   (void)connector;
-  facil_force_event(uuid, FIO_EVENT_ON_DATA);
+  (void)uuid;
 }
 
 /* Failed to connect */
@@ -510,39 +666,43 @@ static void connector_on_close(intptr_t uuid, protocol_s *pconnector) {
 #undef facil_connect
 intptr_t facil_connect(struct facil_connect_args opt) {
   intptr_t uuid = -1;
-  if (!opt.address || !opt.port || !opt.on_connect)
+  if (!opt.on_connect || (!opt.address && !opt.port))
     goto error;
-  if (!opt.set_rw_hooks)
-    opt.set_rw_hooks = listener_set_rw_hooks;
+  if (!opt.timeout)
+    opt.timeout = 30;
   struct ConnectProtocol *connector = malloc(sizeof(*connector));
   if (!connector)
     goto error;
   *connector = (struct ConnectProtocol){
-      .on_connect = opt.on_connect,
+      .on_connect = (void (*)(void *, void *))opt.on_connect,
       .on_fail = opt.on_fail,
       .udata = opt.udata,
-      .protocol.service = connector_protocol_name,
+      .protocol.service = CONNECTOR_PROTOCOL_NAME,
       .protocol.on_data = connector_on_data,
       .protocol.on_ready = connector_on_ready,
       .protocol.on_close = connector_on_close,
-      .set_rw_hooks = opt.set_rw_hooks,
-      .rw_udata = opt.rw_udata,
       .opened = 0,
   };
   uuid = connector->uuid = sock_connect(opt.address, opt.port);
   /* check for errors, always invoke the on_fail if required */
-  if (uuid == -1)
+  if (uuid == -1) {
+    free(connector);
     goto error;
+  }
   if (facil_attach(uuid, &connector->protocol) == -1) {
+    /* facil_attach calls the failure / `on_close` callback */
     sock_close(uuid);
-    goto error;
+    return -1;
   }
+  facil_set_timeout(uuid, opt.timeout);
   return uuid;
 error:
   if (opt.on_fail)
     opt.on_fail(uuid, opt.udata);
   return -1;
 }
+#define facil_connect(...)                                                     \
+  facil_connect((struct facil_connect_args){__VA_ARGS__})
 
 /* *****************************************************************************
 Timers
@@ -562,18 +722,19 @@ typedef struct {
 
 #define prot2timer(protocol) (*((timer_protocol_s *)(protocol)))
 
-static const char *timer_protocol_name = "timer protocol __facil_internal__";
-
 static void timer_on_data(intptr_t uuid, protocol_s *protocol) {
   prot2timer(protocol).task(prot2timer(protocol).arg);
-  evio_reset_timer(sock_uuid2fd(uuid));
   if (prot2timer(protocol).repetitions == 0)
-    return;
+    goto reschedule;
   prot2timer(protocol).repetitions -= 1;
   if (prot2timer(protocol).repetitions)
-    return;
-  evio_remove(sock_uuid2fd(uuid));
+    goto reschedule;
   sock_force_close(uuid);
+  return;
+reschedule:
+  spn_trylock(&uuid_data(uuid).scheduled);
+  evio_set_timer(sock_uuid2fd(uuid), (void *)uuid,
+                 prot2timer(protocol).milliseconds);
 }
 
 static void timer_on_close(intptr_t uuid, protocol_s *protocol) {
@@ -596,7 +757,7 @@ static inline timer_protocol_s *timer_alloc(void (*task)(void *), void *arg,
   timer_protocol_s *t = malloc(sizeof(*t));
   if (t)
     *t = (timer_protocol_s){
-        .protocol.service = timer_protocol_name,
+        .protocol.service = TIMER_PROTOCOL_NAME,
         .protocol.on_data = timer_on_data,
         .protocol.on_close = timer_on_close,
         .protocol.ping = timer_ping,
@@ -610,10 +771,12 @@ static inline timer_protocol_s *timer_alloc(void (*task)(void *), void *arg,
 }
 
 inline static void timer_on_server_start(int fd) {
-  if (evio_add_timer(fd, (void *)sock_fd2uuid(fd),
-                     prot2timer(fd_data(fd).protocol).milliseconds))
-    perror("Couldn't register a required timed event."), kill(0, SIGINT),
-        exit(4);
+  if (evio_set_timer(fd, (void *)sock_fd2uuid(fd),
+                     prot2timer(fd_data(fd).protocol).milliseconds)) {
+    perror("Couldn't register a required timed event.");
+    kill(0, SIGINT);
+    exit(4);
+  }
 }
 
 /**
@@ -645,8 +808,13 @@ int facil_run_every(size_t milliseconds, size_t repetitions,
   if (protocol == NULL)
     goto error;
   facil_attach(uuid, (protocol_s *)protocol);
-  if (evio_isactive() && evio_add_timer(fd, (void *)uuid, milliseconds) == -1)
-    goto error;
+  if (evio_isactive() && evio_set_timer(fd, (void *)uuid, milliseconds) == -1) {
+    /* don't goto error because the protocol is attached. */
+    const int old = errno;
+    sock_force_close(uuid);
+    errno = old;
+    return -1;
+  }
   return 0;
 error:
   if (uuid != -1) {
@@ -668,292 +836,582 @@ error_fin:
 }
 
 /* *****************************************************************************
-Cluster Messaging
+Cluster Messaging - using Unix Sockets
 ***************************************************************************** */
-/* cluster state management */
-static struct {
-  uintptr_t count;
-  fio_list_s handlers;
-  spn_lock_i lock;
-  struct facil_cluster_data_pipe {
-    intptr_t in;
-    intptr_t out;
-  } * pipes;
-} facil_cluster_data = {.handlers.next = &facil_cluster_data.handlers,
-                        .handlers.prev = &facil_cluster_data.handlers,
-                        .lock = SPN_LOCK_INIT};
-
-/* message handler */
-typedef struct {
-  fio_list_s list;
-  void (*on_message)(void *data, uint32_t len);
-  int32_t msg_type;
-} fio_cluster_handler;
-
-/* message sending and protocol. */
-struct facil_cluster_msg_packet {
-  uint16_t count;
-  spn_lock_i lock;
-  struct facil_cluster_msg {
-    uint32_t len;
-    int32_t msg_type;
-  } payload;
-  uint8_t data[];
-};
 
-/* pipe communication protocol object. */
-struct facil_cluster_protocol {
-  protocol_s protocol;
-  size_t mem_size;
-  size_t read;
-  struct facil_cluster_msg *msg;
+#ifdef __BIG_ENDIAN__
+inline static uint32_t cluster_str2uint32(uint8_t *str) {
+  return ((str[0] & 0xFF) | ((((uint32_t)str[1]) << 8) & 0xFF00) |
+          ((((uint32_t)str[2]) << 16) & 0xFF0000) |
+          ((((uint32_t)str[3]) << 24) & 0xFF000000));
+}
+inline static void cluster_uint2str(uint8_t *dest, uint32_t i) {
+  dest[0] = i & 0xFF;
+  dest[1] = (i >> 8) & 0xFF;
+  dest[2] = (i >> 16) & 0xFF;
+  dest[3] = (i >> 24) & 0xFF;
+}
+#else
+inline static uint32_t cluster_str2uint32(uint8_t *str) {
+  return (((((uint32_t)str[0]) << 24) & 0xFF000000) |
+          ((((uint32_t)str[1]) << 16) & 0xFF0000) |
+          ((((uint32_t)str[2]) << 8) & 0xFF00) | (str[3] & 0xFF));
+}
+inline static void cluster_uint2str(uint8_t *dest, uint32_t i) {
+  dest[0] = (i >> 24) & 0xFF;
+  dest[1] = (i >> 16) & 0xFF;
+  dest[2] = (i >> 8) & 0xFF;
+  dest[3] = i & 0xFF;
+}
+#endif
+
+#define CLUSTER_READ_BUFFER 16384
+typedef struct {
+  protocol_s pr;
+  FIOBJ channel;
+  FIOBJ msg;
+  uint32_t exp_channel;
+  uint32_t exp_msg;
+  uint32_t type;
+  int32_t filter;
+  uint32_t length;
+  uint8_t buffer[];
+} cluster_pr_s;
+
+typedef struct {
+  void (*on_message)(int32_t filter, FIOBJ, FIOBJ);
+  FIOBJ channel;
+  FIOBJ msg;
+  int32_t filter;
+} cluster_msg_data_s;
+
+static void cluster_on_new_peer(intptr_t srv, protocol_s *pr);
+static void cluster_on_listening_ping(intptr_t srv, protocol_s *pr);
+static void cluster_on_listening_close(intptr_t srv, protocol_s *pr);
+
+static struct {
+  protocol_s listening;
+  intptr_t root;
+  fio_hash_s clients;
+  fio_hash_s handlers;
+  spn_lock_i lock;
+  uint8_t client_mode;
+  char cluster_name[128];
+} facil_cluster_data = {
+    .lock = SPN_LOCK_INIT,
+    .root = -1,
+    .listening =
+        {
+            .on_close = cluster_on_listening_close,
+            .ping = cluster_on_listening_ping,
+            .on_data = cluster_on_new_peer,
+        },
+};
+
+enum cluster_message_type_e {
+  CLUSTER_MESSAGE_FORWARD,
+  CLUSTER_MESSAGE_JSON,
+  CLUSTER_MESSAGE_SHUTDOWN,
+  CLUSTER_MESSAGE_ERROR,
+  CLUSTER_MESSAGE_PING,
 };
 
-/* cluster handling of message */
-static void facil_cluster_handle_msg(struct facil_cluster_msg *msg) {
-  fio_cluster_handler *hnd;
+static void cluster_deferred_handler(void *msg_data_, void *ignr) {
+  cluster_msg_data_s *data = msg_data_;
+  data->on_message(data->filter, data->channel, data->msg);
+  fiobj_free(data->channel);
+  fiobj_free(data->msg);
+  fio_free(data);
+  (void)ignr;
+}
+
+static void cluster_forward_msg2handlers(cluster_pr_s *c) {
   spn_lock(&facil_cluster_data.lock);
-  fio_list_for_each(fio_cluster_handler, list, hnd,
-                    facil_cluster_data.handlers) {
-    if (hnd->msg_type == msg->msg_type) {
-      hnd->on_message(msg + 1, msg->len - sizeof(*msg));
+  void *target_ =
+      fio_hash_find(&facil_cluster_data.handlers, (FIO_HASH_KEY_TYPE)c->filter);
+  spn_unlock(&facil_cluster_data.lock);
+  // fprintf(stderr, "handler for %d: %p\n", c->filter, target_);
+  if (target_) {
+    cluster_msg_data_s *data = fio_malloc(sizeof(*data));
+    if (!data) {
+      perror("FATAL ERROR: (facil.io cluster) couldn't allocate memory");
+      exit(errno);
     }
+    *data = (cluster_msg_data_s){
+        .on_message = ((cluster_msg_data_s *)(&target_))->on_message,
+        .channel = fiobj_dup(c->channel),
+        .msg = fiobj_dup(c->msg),
+        .filter = c->filter,
+    };
+    defer(cluster_deferred_handler, data, NULL);
   }
-  spn_unlock(&facil_cluster_data.lock);
 }
 
-/* cluster service ID */
-static char *facil_cluster_protocol_id = "facil_io_internal_cluster_protocol";
+static inline FIOBJ cluster_wrap_message(uint32_t ch_len, uint32_t msg_len,
+                                         uint32_t type, int32_t id,
+                                         uint8_t *ch_data, uint8_t *msg_data) {
+  FIOBJ buf = fiobj_str_buf(ch_len + msg_len + 16);
+  fio_cstr_s f = fiobj_obj2cstr(buf);
+  cluster_uint2str(f.bytes, ch_len);
+  cluster_uint2str(f.bytes + 4, msg_len);
+  cluster_uint2str(f.bytes + 8, type);
+  cluster_uint2str(f.bytes + 12, (uint32_t)id);
+  if (ch_len && ch_data) {
+    memcpy(f.bytes + 16, ch_data, ch_len);
+  }
+  if (msg_len && msg_data) {
+    memcpy(f.bytes + 16 + ch_len, msg_data, msg_len);
+  }
+  fiobj_str_resize(buf, ch_len + msg_len + 16);
+  return buf;
+}
 
-/* cluster protocol on_close callback */
-static void facil_cluster_on_close(intptr_t uuid, protocol_s *pr_) {
-  struct facil_cluster_protocol *pr = (struct facil_cluster_protocol *)pr_;
-  if (pr->msg)
-    free(pr->msg);
-  free(pr);
-#if FACIL_PRINT_STATE == 1
-  if (defer_fork_is_active())
-    fprintf(stderr, "ERROR: facil cluster member (%d) closed prematurely.\n",
-            defer_fork_pid());
-#endif
-  (void)uuid;
+static inline void cluster_send2clients(uint32_t ch_len, uint32_t msg_len,
+                                        uint32_t type, int32_t id,
+                                        uint8_t *ch_data, uint8_t *msg_data,
+                                        intptr_t uuid) {
+  if (facil_cluster_data.clients.count == 0)
+    return;
+  FIOBJ forward =
+      cluster_wrap_message(ch_len, msg_len, type, id, ch_data, msg_data);
+  spn_lock(&facil_cluster_data.lock);
+  FIO_HASH_FOR_LOOP(&facil_cluster_data.clients, i) {
+    if (i->obj) {
+      if ((intptr_t)i->key != uuid)
+        fiobj_send_free((intptr_t)i->key, fiobj_dup(forward));
+    }
+  }
+  spn_unlock(&facil_cluster_data.lock);
+  fiobj_free(forward);
+}
+
+static inline void cluster_send2traget(uint32_t ch_len, uint32_t msg_len,
+                                       uint32_t type, int32_t id,
+                                       uint8_t *ch_data, uint8_t *msg_data) {
+  if (facil_cluster_data.client_mode) {
+    FIOBJ forward =
+        cluster_wrap_message(ch_len, msg_len, type, id, ch_data, msg_data);
+    fiobj_send_free(facil_cluster_data.root, forward);
+  } else {
+    cluster_send2clients(ch_len, msg_len, type, id, ch_data, msg_data, 0);
+  }
 }
 
-/* React when a sibling process closes / crashes */
-static void facil_cluster_on_sibling_close(intptr_t uuid, protocol_s *pr_) {
-  if (facil_cluster_data.pipes[defer_fork_pid()].in == uuid)
+/* NOT signal safe. */
+static inline void facil_cluster_signal_children(void) {
+  if (facil_parent_pid() != getpid()) {
+    facil_stop();
     return;
-  if (defer_fork_is_active()) {
-    kill(getpid(), SIGINT);
-#if DEBUG
-    fprintf(stderr,
-            "* Sibling death detected, signaling self exit process %d\n",
-            getpid());
-#endif
   }
-  (void)uuid;
-  (void)pr_;
+  cluster_send2traget(0, 0, CLUSTER_MESSAGE_SHUTDOWN, 0, NULL, NULL);
+  /* TODO: add `on_restart` callback handler. */
+}
+
+static void cluster_on_client_message(cluster_pr_s *c, intptr_t uuid) {
+  switch ((enum cluster_message_type_e)c->type) {
+  case CLUSTER_MESSAGE_JSON: {
+    fio_cstr_s s = fiobj_obj2cstr(c->channel);
+    FIOBJ tmp = FIOBJ_INVALID;
+    if (fiobj_json2obj(&tmp, s.bytes, s.len)) {
+      fiobj_free(c->channel);
+      c->channel = tmp;
+      tmp = FIOBJ_INVALID;
+    } else {
+      fprintf(stderr,
+              "WARNING: (facil.io cluster) JSON channel isn't valid JSON.\n");
+    }
+    s = fiobj_obj2cstr(c->msg);
+    if (fiobj_json2obj(&tmp, s.bytes, s.len)) {
+      fiobj_free(c->msg);
+      c->msg = tmp;
+    } else {
+      fprintf(stderr,
+              "WARNING: (facil.io cluster) JSON message isn't valid JSON.\n");
+    }
+  }
+  /* fallthrough */
+  case CLUSTER_MESSAGE_FORWARD:
+    cluster_forward_msg2handlers(c);
+    break;
+
+  case CLUSTER_MESSAGE_ERROR:
+  case CLUSTER_MESSAGE_SHUTDOWN:
+    facil_stop();
+    facil_cluster_data.root = -1;
+    sock_close(uuid);
+    break;
+
+  case CLUSTER_MESSAGE_PING:
+    /* do nothing, really. */
+    break;
+  }
 }
 
-/* cluster protocol on_data callback */
-static void facil_cluster_on_data(intptr_t uuid, protocol_s *pr_) {
-  struct facil_cluster_protocol *pr = (struct facil_cluster_protocol *)pr_;
-  while (1) {
-    errno = 0;
-    if (pr->read < 8 || pr->read < pr->msg->len) {
-      ssize_t read = sock_read(uuid, (void *)(((uintptr_t)pr->msg) + pr->read),
-                               pr->mem_size - pr->read);
-      if (read <= 0) {
-        if (read < 0)
-          perror("ERROR: cluster pipe read error");
-        return;
-      }
-      pr->read += read;
-      if (pr->read < 8)
-        continue;
-      if (pr->mem_size < pr->msg->len) {
-        void *tmp = realloc(pr->msg, pr->msg->len);
-        if (!tmp)
-          perror("ERROR: (critical) cannot allocate memory for a clustered "
-                 "message."),
-              kill(0, SIGINT), exit(errno);
-        pr->msg = tmp;
-#if defined(FACIL_PRINT_STATE) && FACIL_PRINT_STATE == 1 && defined(DEBUG)
-        fprintf(stderr, "* Cluster (%d) Reallocated msg size to %u\n",
-                defer_fork_pid(), pr->msg->len);
-#endif
-        pr->mem_size = pr->msg->len;
-      }
+static void cluster_on_server_message(cluster_pr_s *c, intptr_t uuid) {
+  switch ((enum cluster_message_type_e)c->type) {
+  case CLUSTER_MESSAGE_JSON:
+  case CLUSTER_MESSAGE_FORWARD: {
+    if (fio_hash_count(&facil_cluster_data.clients)) {
+      fio_cstr_s cs = fiobj_obj2cstr(c->channel);
+      fio_cstr_s ms = fiobj_obj2cstr(c->msg);
+      cluster_send2clients((uint32_t)cs.len, (uint32_t)ms.len, c->type,
+                           c->filter, cs.bytes, ms.bytes, uuid);
     }
-    if ((size_t)(pr->msg->len) <= pr->read) {
-      const size_t msg_len = (size_t)(pr->msg->len);
-      const size_t leftover = (size_t)(pr->read - msg_len);
-      facil_cluster_handle_msg(pr->msg);
-      pr->read = leftover;
-      if (leftover) {
-        memmove(pr->msg, (void *)(((uintptr_t)pr->msg) + msg_len), pr->read);
+    if (c->type == CLUSTER_MESSAGE_JSON) {
+      fio_cstr_s s = fiobj_obj2cstr(c->channel);
+      FIOBJ tmp = FIOBJ_INVALID;
+      if (fiobj_json2obj(&tmp, s.bytes, s.len)) {
+        fiobj_free(c->channel);
+        c->channel = tmp;
+        tmp = FIOBJ_INVALID;
+      } else {
+        fprintf(stderr,
+                "WARNING: (facil.io cluster) JSON message isn't valid JSON.\n");
+      }
+      s = fiobj_obj2cstr(c->msg);
+      if (fiobj_json2obj(&tmp, s.bytes, s.len)) {
+        fiobj_free(c->msg);
+        c->msg = tmp;
+      } else {
+        fprintf(stderr,
+                "WARNING: (facil.io cluster) JSON message isn't valid JSON.\n");
       }
     }
+    cluster_forward_msg2handlers(c);
+    break;
+  }
+  case CLUSTER_MESSAGE_SHUTDOWN:
+  case CLUSTER_MESSAGE_ERROR:
+  case CLUSTER_MESSAGE_PING:
+    /* do nothing, really. */
+    break;
   }
 }
 
-/* Registers process specific information */
-static void facil_cluster_register(void *arg1, void *arg2) {
-  (void)arg1;
-  (void)arg2;
-  if (facil_cluster_data.count < 2)
+static void cluster_on_client_close(intptr_t uuid, protocol_s *pr_) {
+  cluster_pr_s *c = (cluster_pr_s *)pr_;
+  /* no shutdown message received - parent crashed. */
+  if (facil_cluster_data.root == uuid && c->type != CLUSTER_MESSAGE_SHUTDOWN &&
+      facil_data->active) {
+    if (FACIL_PRINT_STATE)
+      fprintf(stderr, "* (%d) Parent Process crash detected!\n", getpid());
+    unlink(facil_cluster_data.cluster_name);
+  }
+  fiobj_free(c->msg);
+  fiobj_free(c->channel);
+  free(c);
+  facil_stop();
+  if (facil_cluster_data.root == uuid)
+    facil_cluster_data.root = -1;
+}
+
+static void cluster_on_server_close(intptr_t uuid, protocol_s *pr_) {
+  cluster_pr_s *c = (cluster_pr_s *)pr_;
+  if (facil_cluster_data.client_mode || facil_parent_pid() != getpid()) {
+    /* we respawned - clean up resources, but don't stop server */
+    fiobj_free(c->msg);
+    fiobj_free(c->channel);
+    free(c);
+    if (facil_cluster_data.root == uuid)
+      facil_cluster_data.root = -1;
     return;
-
-  /* used to watch for sibling crashes and prevent other events from firing */
-  static protocol_s sentinel_protocol = {
-      .service = NULL,
-      .ping = listener_ping,
-      .on_close = facil_cluster_on_sibling_close,
-  };
-  /* close what we don't need and watch the connections we do need */
-  for (size_t i = 0; i < facil_cluster_data.count; i++) {
-    if (i == (size_t)defer_fork_pid())
-      continue;
-    sock_close(facil_cluster_data.pipes[i].in);
-    facil_attach(facil_cluster_data.pipes[i].out, &sentinel_protocol);
   }
-  sock_close(facil_cluster_data.pipes[defer_fork_pid()].out);
+  spn_lock(&facil_cluster_data.lock);
+  fio_hash_insert(&facil_cluster_data.clients, (FIO_HASH_KEY_TYPE)uuid, NULL);
+  // fio_hash_compact(&facil_cluster_data.clients);
+  spn_unlock(&facil_cluster_data.lock);
+  fiobj_free(c->msg);
+  fiobj_free(c->channel);
+  free(c);
+}
 
-  struct facil_cluster_protocol *pr = malloc(sizeof(*pr));
-  if (!pr)
-    perror("ERROR: (critical) cannot allocate memory for cluster."),
-        kill(0, SIGINT), exit(errno);
-  *pr = (struct facil_cluster_protocol){
-      .protocol.service = facil_cluster_protocol_id,
-      .protocol.on_data = facil_cluster_on_data,
-      .protocol.on_close = facil_cluster_on_close,
-      .protocol.ping = listener_ping,
-  };
-  pr->msg = calloc(1, 1024);
-  pr->mem_size = 1024;
-  if (!pr->msg)
-    perror("ERROR: (critical) cannot allocate memory for cluster."),
-        kill(0, SIGINT), exit(errno);
-  facil_attach(facil_cluster_data.pipes[defer_fork_pid()].in, &pr->protocol);
+static void cluster_on_shutdown(intptr_t uuid, protocol_s *pr_) {
+  cluster_send2traget(0, 0, CLUSTER_MESSAGE_SHUTDOWN, 0, NULL, NULL);
+  facil_force_event(uuid, FIO_EVENT_ON_READY);
+  (void)pr_;
+  (void)uuid;
 }
 
-static void facil_cluster_init(uint16_t count) {
-  if (facil_cluster_data.pipes || count < 2)
+static void cluster_on_data(intptr_t uuid, protocol_s *pr_) {
+  cluster_pr_s *c = (cluster_pr_s *)pr_;
+  ssize_t i =
+      sock_read(uuid, c->buffer + c->length, CLUSTER_READ_BUFFER - c->length);
+  if (i <= 0)
     return;
-  facil_cluster_data.pipes =
-      malloc((sizeof(struct facil_cluster_data_pipe) * count));
-  if (!facil_cluster_data.pipes)
-    goto error;
+  c->length += i;
+  i = 0;
+  do {
+    if (!c->exp_channel && !c->exp_msg) {
+      if (c->length - i < 16)
+        break;
+      c->exp_channel = cluster_str2uint32(c->buffer + i);
+      c->exp_msg = cluster_str2uint32(c->buffer + i + 4);
+      c->type = cluster_str2uint32(c->buffer + i + 8);
+      c->filter = (int32_t)cluster_str2uint32(c->buffer + i + 12);
+      if (c->exp_channel) {
+        if (c->exp_channel >= (1024 * 1024 * 16)) {
+          fprintf(
+              stderr,
+              "ERROR: (%d) cluster message name too long (16Mb limit): %u\n",
+              getpid(), (unsigned int)c->exp_channel);
+          facil_stop();
+          return;
+        }
+        c->channel = fiobj_str_buf(c->exp_channel);
+      }
+      if (c->exp_msg) {
+        if (c->exp_msg >= (1024 * 1024 * 64)) {
+          fprintf(
+              stderr,
+              "ERROR: (%d) cluster message data too long (64Mb limit): %u\n",
+              getpid(), (unsigned int)c->exp_msg);
+          facil_stop();
+          return;
+        }
+        c->msg = fiobj_str_buf(c->exp_msg);
+      }
+      i += 16;
+    }
+    if (c->exp_channel) {
+      if (c->exp_channel + i > c->length) {
+        fiobj_str_write(c->channel, (char *)c->buffer + i,
+                        (size_t)(c->length - i));
+        c->exp_channel -= (c->length - i);
+        i = c->length;
+        break;
+      } else {
+        fiobj_str_write(c->channel, (char *)c->buffer + i, c->exp_channel);
+        i += c->exp_channel;
+        c->exp_channel = 0;
+      }
+    }
+    if (c->exp_msg) {
+      if (c->exp_msg + i > c->length) {
+        fiobj_str_write(c->msg, (char *)c->buffer + i, (size_t)(c->length - i));
+        c->exp_msg -= (c->length - i);
+        i = c->length;
+        break;
+      } else {
+        fiobj_str_write(c->msg, (char *)c->buffer + i, c->exp_msg);
+        i += c->exp_msg;
+        c->exp_msg = 0;
+      }
+    }
+    if (facil_cluster_data.client_mode) {
+      cluster_on_client_message(c, uuid);
+    } else {
+      cluster_on_server_message(c, uuid);
+    }
+    fiobj_free(c->msg);
+    fiobj_free(c->channel);
+    c->msg = FIOBJ_INVALID;
+    c->channel = FIOBJ_INVALID;
+  } while (c->length > i);
+  c->length -= i;
+  if (c->length) {
+    memmove(c->buffer, c->buffer + i, c->length);
+  }
+  (void)pr_;
+}
+static void cluster_ping(intptr_t uuid, protocol_s *pr_) {
+  FIOBJ ping = cluster_wrap_message(0, 0, CLUSTER_MESSAGE_PING, 0, NULL, NULL);
+  fiobj_send_free(uuid, ping);
+  (void)pr_;
+}
 
-  /* used to prevent any events from firing  */
-  static protocol_s stub_protocol = {
-      .service = NULL, .ping = listener_ping,
+static void cluster_on_open(intptr_t fd, void *udata) {
+  cluster_pr_s *pr = malloc(sizeof(*pr) + CLUSTER_READ_BUFFER);
+  *pr = (cluster_pr_s){
+      .pr =
+          {
+              .service = CLUSTER_CONNECTION_PROTOCOL_NAME,
+              .on_data = cluster_on_data,
+              .on_shutdown = cluster_on_shutdown,
+              .on_close =
+                  (facil_cluster_data.client_mode ? cluster_on_client_close
+                                                  : cluster_on_server_close),
+              .ping = cluster_ping,
+          },
   };
-  stub_protocol.service = facil_cluster_protocol_id;
-
-  facil_cluster_data.count = count;
-  int p_tmp[2];
-
-  for (size_t i = 0; i < count; i++) {
-    if (pipe(p_tmp))
-      goto error;
-    sock_set_non_block(p_tmp[0]);
-    sock_set_non_block(p_tmp[1]);
-    facil_cluster_data.pipes[i].in = sock_open(p_tmp[0]);
-    facil_attach(facil_cluster_data.pipes[i].in, &stub_protocol);
-    facil_cluster_data.pipes[i].out = sock_open(p_tmp[1]);
-    facil_attach(facil_cluster_data.pipes[i].out, &stub_protocol);
-  }
-  defer(facil_cluster_register, NULL, NULL);
-  return;
-error:
-  perror("ERROR: (facil.io) Couldn't initialize cluster pipes");
-  kill(0, SIGINT), exit(errno);
+  if (facil_cluster_data.root >= 0 && facil_cluster_data.root != fd) {
+    // if (facil_parent_pid() != getpid()) {
+    //   fprintf(stderr,
+    //           "WARNING: non-root process adding client...\n"
+    //           "         adding to clients %p (root == %p)\n",
+    //           (void *)fd, (void *)facil_cluster_data.root);
+    // } else {
+    //   fprintf(stderr, "INFO: root process adding child connection %p\n",
+    //           (void *)fd);
+    // }
+    spn_lock(&facil_cluster_data.lock);
+    fio_hash_insert(&facil_cluster_data.clients, (FIO_HASH_KEY_TYPE)fd,
+                    (void *)fd);
+    spn_unlock(&facil_cluster_data.lock);
+  } else if (facil_parent_pid() != getpid()) {
+    // fprintf(stderr, "INFO: child process registering...%p \n", (void *)fd);
+  }
+  if (facil_attach(fd, &pr->pr) == -1) {
+    fprintf(stderr, "(%d) ", getpid());
+    perror("ERROR: (facil.io cluster) couldn't attach connection");
+    facil_stop();
+  }
+  (void)udata;
 }
 
-static void facil_cluster_destroy(void) {
-  if (!facil_cluster_data.pipes)
-    return;
-  for (size_t i = 0; i < facil_cluster_data.count; i++) {
-    sock_close(facil_cluster_data.pipes[i].out);
-    sock_close(facil_cluster_data.pipes[i].in);
+static void cluster_on_new_peer(intptr_t srv, protocol_s *pr) {
+  intptr_t client = sock_accept(srv);
+  if (client == -1) {
+    // fprintf(stderr,
+    // "ERROR: (facil.io cluster) couldn't accept connection\n");
+  } else {
+    cluster_on_open(client, NULL);
   }
-
-  fio_cluster_handler *hnd;
-  while ((hnd = fio_list_pop(fio_cluster_handler, list,
-                             facil_cluster_data.handlers)))
-    free(hnd);
-  free(facil_cluster_data.pipes);
-  facil_cluster_data.pipes = NULL;
+  (void)pr;
 }
-
-void facil_cluster_set_handler(int32_t msg_type,
-                               void (*on_message)(void *data, uint32_t len)) {
-  fio_cluster_handler *hnd;
-  spn_lock(&facil_cluster_data.lock);
-  fio_list_for_each(fio_cluster_handler, list, hnd,
-                    facil_cluster_data.handlers) {
-    if (hnd->msg_type == msg_type) {
-      hnd->on_message = on_message;
-      spn_unlock(&facil_cluster_data.lock);
-      return;
-    }
+static void cluster_on_listening_close(intptr_t srv, protocol_s *pr) {
+  if (facil_parent_pid() == getpid()) {
+    unlink(facil_cluster_data.cluster_name);
   }
-  hnd = malloc(sizeof(*hnd));
-  *hnd = (fio_cluster_handler){.msg_type = msg_type, .on_message = on_message};
-  fio_list_push(fio_cluster_handler, list, facil_cluster_data.handlers, hnd);
-  spn_unlock(&facil_cluster_data.lock);
+  if (facil_cluster_data.root == srv)
+    facil_cluster_data.root = -1;
+  (void)srv;
+  (void)pr;
+}
+static void cluster_on_listening_ping(intptr_t srv, protocol_s *pr) {
+  sock_touch(srv);
+  (void)pr;
 }
 
-static void facil_msg_free(void *msg_) {
-  struct facil_cluster_msg_packet *msg = msg_;
-  spn_lock(&msg->lock);
-  msg->count--;
-  if (msg->count) {
-    spn_unlock(&msg->lock);
-    return;
+static int cluster_on_start(void) {
+  if (facil_data->active <= 1)
+    return 0;
+  if (facil_parent_pid() == getpid()) {
+    /* Root process, listen to incoming cluster connections */
+    facil_cluster_data.client_mode = 0;
+    facil_cluster_data.listening.service = CLUSTER_LISTEN_PROTOCOL_NAME;
+    if (facil_attach(facil_cluster_data.root, &facil_cluster_data.listening)) {
+      perror("FATAL ERROR: (facil.io) couldn't attach cluster socket");
+    }
+  } else {
+    /* Worker process, connect to Root process */
+    facil_cluster_data.client_mode = 1;
+    fio_hash_free(&facil_cluster_data.clients);
+    facil_cluster_data.clients = (fio_hash_s)FIO_HASH_INIT;
+    if (facil_cluster_data.root != -1) {
+      sock_force_close(facil_cluster_data.root);
+    }
+    facil_cluster_data.root =
+        facil_connect(.address = facil_cluster_data.cluster_name,
+                      .on_connect = cluster_on_open);
+    if (facil_cluster_data.root == -1) {
+      perror(
+          "FATAL ERROR: (facil.io cluster) couldn't connect to cluster socket");
+      fprintf(stderr, "         socket: %s\n", facil_cluster_data.cluster_name);
+      facil_stop();
+      return -1;
+    }
   }
-  free(msg);
+  return 0;
 }
-int facil_cluster_send(int32_t msg_type, void *data, uint32_t len) {
-  if (!defer_fork_is_active()) {
-#ifdef DEBUG
-    fprintf(stderr,
-            "WARNING: (debug, ignored) `facil_cluster_send` can only be "
-            "called while facil.io is running.\n");
+
+static int facil_cluster_init(void) {
+  if (facil_data->active <= 1)
+    return 0;
+  /* create a unique socket name */
+  char *tmp_folder = getenv("TMPDIR");
+  uint32_t tmp_folder_len = 0;
+  if (!tmp_folder || ((tmp_folder_len = (uint32_t)strlen(tmp_folder)) > 100)) {
+#ifdef P_tmpdir
+    tmp_folder = P_tmpdir;
+    if (tmp_folder)
+      tmp_folder_len = (uint32_t)strlen(tmp_folder);
+#else
+    tmp_folder = "/tmp/";
+    tmp_folder_len = 5;
 #endif
-    return -1;
   }
-  if (facil_cluster_data.count < 2)
-    return 0;
-  if (!data || !len)
+  if (tmp_folder_len >= 100)
+    tmp_folder_len = 0;
+  if (tmp_folder_len) {
+    memcpy(facil_cluster_data.cluster_name, tmp_folder, tmp_folder_len);
+    if (facil_cluster_data.cluster_name[tmp_folder_len - 1] != '/')
+      facil_cluster_data.cluster_name[tmp_folder_len++] = '/';
+  }
+  memcpy(facil_cluster_data.cluster_name + tmp_folder_len, "facil-io-sock-",
+         14);
+  tmp_folder_len += 14;
+  tmp_folder_len +=
+      fio_ltoa(facil_cluster_data.cluster_name + tmp_folder_len, getpid(), 8);
+  facil_cluster_data.cluster_name[tmp_folder_len] = 0;
+
+  /* remove if existing */
+  unlink(facil_cluster_data.cluster_name);
+  /* create, bind, listen */
+  facil_cluster_data.root = sock_listen(facil_cluster_data.cluster_name, NULL);
+
+  if (facil_cluster_data.root == -1) {
+    perror("FATAL ERROR: (facil.io cluster) failed to open cluster socket.\n"
+           "             check file permissions");
     return -1;
-  struct facil_cluster_msg_packet *msg = malloc(len + sizeof(*msg));
-  if (!msg)
+  }
+  return 0;
+}
+
+void facil_cluster_set_handler(int32_t filter,
+                               void (*on_message)(int32_t id, FIOBJ ch,
+                                                  FIOBJ msg)) {
+  spn_lock(&facil_cluster_data.lock);
+  fio_hash_insert(&facil_cluster_data.handlers, (uint64_t)filter,
+                  (void *)(uintptr_t)on_message);
+  spn_unlock(&facil_cluster_data.lock);
+}
+
+int facil_cluster_send(int32_t filter, FIOBJ ch, FIOBJ msg) {
+  if (!facil_data) {
+    fprintf(stderr, "ERROR: cluster inactive, can't send message.\n");
     return -1;
-  msg->count = facil_cluster_data.count - 1;
-  msg->lock = SPN_LOCK_INIT;
-  msg->payload.len = len + sizeof(msg->payload);
-  msg->payload.msg_type = msg_type;
-  memcpy(msg->data, data, len);
-  for (int i = 0; i < (int)facil_cluster_data.count; i++) {
-    if (defer_fork_pid() == i)
-      continue;
-    if (sock_write2(.uuid = facil_cluster_data.pipes[i].out, .buffer = msg,
-                    .offset =
-                        ((uintptr_t) &
-                         (((struct facil_cluster_msg_packet *)(0))->payload)),
-                    .length = msg->payload.len, .move = 1,
-                    .dealloc = facil_msg_free))
-      perror("ERROR: Cluster `write` failed");
   }
+  uint32_t type = CLUSTER_MESSAGE_FORWARD;
+
+  if ((!ch || FIOBJ_TYPE_IS(ch, FIOBJ_T_STRING)) &&
+      (!msg || FIOBJ_TYPE_IS(msg, FIOBJ_T_STRING))) {
+    fiobj_dup(ch);
+    fiobj_dup(msg);
+  } else {
+    type = CLUSTER_MESSAGE_JSON;
+    ch = fiobj_obj2json(ch, 0);
+    msg = fiobj_obj2json(msg, 0);
+  }
+  fio_cstr_s cs = fiobj_obj2cstr(ch);
+  fio_cstr_s ms = fiobj_obj2cstr(msg);
+  cluster_send2traget((uint32_t)cs.len, (uint32_t)ms.len, type, filter,
+                      cs.bytes, ms.bytes);
+  fiobj_free(ch);
+  fiobj_free(msg);
   return 0;
 }
 
+static void facil_cluster_cleanup(void) {
+  fio_hash_free(&facil_cluster_data.handlers);
+  fio_hash_free(&facil_cluster_data.clients);
+}
 /* *****************************************************************************
 Running the server
 ***************************************************************************** */
 
+volatile uint8_t facil_signal_children_flag = 0;
+
+static inline void facil_internal_poll(void) {
+  if (facil_signal_children_flag) {
+    facil_signal_children_flag = 0;
+    facil_cluster_signal_children();
+  }
+}
+
+static inline void facil_internal_poll_reset(void) {
+  facil_signal_children_flag = 0;
+}
+
 static void print_pid(void *arg, void *ignr) {
   (void)arg;
   (void)ignr;
@@ -963,7 +1421,7 @@ static void print_pid(void *arg, void *ignr) {
 static void facil_review_timeout(void *arg, void *ignr) {
   (void)ignr;
   protocol_s *tmp;
-  time_t review = facil_data->last_cycle;
+  time_t review = facil_data->last_cycle.tv_sec;
   intptr_t fd = (intptr_t)arg;
 
   uint16_t timeout = fd_data(fd).timeout;
@@ -978,7 +1436,7 @@ static void facil_review_timeout(void *arg, void *ignr) {
   if (prt_meta(tmp).locks[FIO_PR_LOCK_TASK] ||
       prt_meta(tmp).locks[FIO_PR_LOCK_WRITE])
     goto unlock;
-  defer(deferred_ping, (void *)sock_fd2uuid(fd), NULL);
+  defer(deferred_ping, (void *)sock_fd2uuid((int)fd), NULL);
 unlock:
   protocol_unlock(tmp, FIO_PR_LOCK_STATE);
 finish:
@@ -994,75 +1452,432 @@ reschedule:
   defer(facil_review_timeout, (void *)fd, NULL);
 }
 
-static void facil_cycle(void *arg, void *ignr) {
+static void perform_idle(void *arg, void *ignr) {
+  facil_data->on_idle();
+  (void)arg;
   (void)ignr;
+}
+
+/* reactor pattern cycling - common */
+static void facil_cycle_schedule_events(void) {
   static int idle = 0;
-  time(&facil_data->last_cycle);
+  clock_gettime(CLOCK_REALTIME, &facil_data->last_cycle);
+  facil_internal_poll();
   int events;
   if (defer_has_queue()) {
     events = evio_review(0);
-    if (events < 0)
-      goto error;
-    if (events > 0)
+    if (events < 0) {
+      return;
+    }
+    if (events > 0) {
       idle = 1;
+    }
   } else {
-    events = evio_review(512);
+    events = evio_review(EVIO_TICK);
     if (events < 0)
-      goto error;
+      return;
     if (events > 0) {
       idle = 1;
     } else if (idle) {
-      ((struct facil_run_args *)arg)->on_idle();
+      defer(perform_idle, NULL, NULL);
       idle = 0;
     }
   }
-  if (!defer_fork_is_active())
-    return;
-  if (facil_data->need_review) {
+  static time_t last_to_review = 0;
+  if (facil_data->need_review &&
+      facil_data->last_cycle.tv_sec != last_to_review) {
+    last_to_review = facil_data->last_cycle.tv_sec;
     facil_data->need_review = 0;
     defer(facil_review_timeout, (void *)0, NULL);
   }
-  defer(facil_cycle, arg, NULL);
-error:
-  (void)1;
 }
 
-static void facil_init_run(void *arg, void *arg2) {
-  (void)arg;
-  (void)arg2;
+/* reactor pattern cycling during cleanup */
+static void facil_cycle_unwind(void *ignr, void *ignr2) {
+  if (facil_data->connection_count) {
+    facil_cycle_schedule_events();
+    defer(facil_cycle_unwind, ignr, ignr2);
+    return;
+  }
+  pool_pt pool = facil_data->thread_pool;
+  facil_data->thread_pool = NULL;
+  defer_pool_stop(pool);
+  return;
+  (void)ignr;
+  (void)ignr2;
+}
+
+/* reactor pattern cycling */
+static void facil_cycle(void *ignr, void *ignr2) {
+  facil_cycle_schedule_events();
+  if (facil_data->active) {
+    defer(facil_cycle, ignr, ignr2);
+    return;
+  }
+  /* switch to winding down */
+  if (FACIL_PRINT_STATE && !facil_data->spindown) {
+    pid_t pid = getpid();
+    if (pid != facil_data->parent)
+      fprintf(stderr, "* (%d) Detected exit signal.\n", getpid());
+    else
+      fprintf(stderr, "* Server Detected exit signal.\n");
+  }
+  pool_pt pool = facil_data->thread_pool;
+  facil_data->thread_pool = NULL;
+  defer_pool_stop(pool);
+  return;
+  (void)ignr;
+  (void)ignr2;
+}
+
+/**
+OVERRIDE THIS to replace the default `fork` implementation or to inject hooks
+into the forking function.
+
+Behaves like the system's `fork`.
+*/
+#pragma weak facil_fork
+int facil_fork(void) { return (int)fork(); }
+
+/** This will be called by child processes, make sure to unlock any existing
+ * locks.
+ *
+ * Known locks:
+ * * `defer` tasks lock.
+ * * `sock` packet pool lock.
+ * * `sock` connection lock (per connection).
+ * * `facil` global lock.
+ * * `facil` pub/sub lock.
+ * * `facil` connection data lock (per connection data).
+ * * `facil` protocol lock (per protocol object, placed in `rsv`).
+ * * `pubsub` pubsub global lock (should be initialized in facil_external_init.
+ * * `pubsub` pubsub client lock (should be initialized in facil_external_init.
+ */
+static void facil_worker_startup(uint8_t sentinel) {
+  facil_cluster_data.lock = facil_data->global_lock = SPN_LOCK_INIT;
+  facil_internal_poll_reset();
   evio_create();
-  time(&facil_data->last_cycle);
-  for (intptr_t i = 0; i < facil_data->capacity; i++) {
-    if (fd_data(i).protocol) {
-      if (fd_data(i).protocol->service == listener_protocol_name)
-        listener_on_start(i);
-      else if (fd_data(i).protocol->service == timer_protocol_name)
-        timer_on_server_start(i);
-      else
-        evio_add(i, (void *)sock_fd2uuid(i));
+  clock_gettime(CLOCK_REALTIME, &facil_data->last_cycle);
+  facil_external_init();
+  if (facil_data->active == 1) {
+    /* single process */
+    for (int i = 0; i < facil_data->capacity; i++) {
+      errno = 0;
+      fd_data(i).lock = SPN_LOCK_INIT;
+      if (fd_data(i).protocol) {
+        fd_data(i).protocol->rsv = 0;
+        if (fd_data(i).protocol->service == LISTENER_PROTOCOL_NAME)
+          listener_on_start(i);
+        else if (fd_data(i).protocol->service == TIMER_PROTOCOL_NAME)
+          timer_on_server_start(i);
+        else {
+          evio_add(i, (void *)sock_fd2uuid(i));
+        }
+      }
     }
+  } else if (sentinel == 0) {
+    /* child process */
+    for (int i = 0; i < facil_data->capacity; i++) {
+      errno = 0;
+      fd_data(i).lock = SPN_LOCK_INIT;
+      if (fd_data(i).protocol) {
+        fd_data(i).protocol->rsv = 0;
+        if (fd_data(i).protocol->service == LISTENER_PROTOCOL_NAME)
+          listener_on_start(i);
+        else if (fd_data(i).protocol->service == TIMER_PROTOCOL_NAME)
+          timer_on_server_start(i);
+        else {
+          /* prevent normal connections from being shared across workers */
+          intptr_t uuid = sock_fd2uuid(i);
+          if (uuid != -1) {
+            sock_force_close(uuid);
+          } else {
+            sock_on_close(uuid);
+          }
+        }
+      }
+    }
+  } else {
+    /* sentinel process - ignore listening sockets, but keep them open */
+    for (int i = 0; i < facil_data->capacity; i++) {
+      fd_data(i).lock = SPN_LOCK_INIT;
+      if (fd_data(i).protocol) {
+        fd_data(i).protocol->rsv = 0;
+        if (fd_data(i).protocol->service == TIMER_PROTOCOL_NAME)
+          timer_on_server_start(i);
+        else if (fd_data(i).protocol->service != LISTENER_PROTOCOL_NAME) {
+          evio_add(i, (void *)sock_fd2uuid(i));
+        }
+      }
+    }
+  }
+  /* Clear all existing events & flush `facil_cycle` out. */
+  {
+    facil_data->spindown = 1;
+    uint16_t old_active = facil_data->active;
+    facil_data->active = 0;
+    defer_perform();
+    facil_data->active = old_active;
+    facil_data->spindown = 0;
+  }
+  /* called after connection cleanup, as it should open connections. */
+  if (cluster_on_start()) {
+    facil_data->thread_pool = NULL;
+    return;
   }
+  /* call any external startup callbacks. */
+  facil_external_init2();
+  /* add cycling to the defer queue to setup the reactor pattern. */
   facil_data->need_review = 1;
-  facil_external_init();
-  defer(facil_cycle, arg, NULL);
-}
+  defer(facil_cycle, NULL, NULL);
 
-static void facil_cleanup(void *arg) {
+  if (FACIL_PRINT_STATE) {
+    if (sentinel || facil_data->parent == getpid()) {
+      fprintf(stderr,
+              "Server is running %u %s X %u %s, press ^C to stop\n"
+              "* Detected capacity: %zd open file limit\n"
+              "* Root pid: %d\n",
+              facil_data->active, facil_data->active > 1 ? "workers" : "worker",
+              facil_data->threads,
+              facil_data->threads > 1 ? "threads" : "thread",
+              facil_data->capacity, facil_data->parent);
+    } else {
+      defer(print_pid, NULL, NULL);
+    }
+  }
+  facil_data->thread_pool =
+      defer_pool_start((sentinel ? 1 : facil_data->threads));
+  if (facil_data->thread_pool)
+    defer_pool_wait(facil_data->thread_pool);
+}
+
+static void facil_worker_cleanup(void) {
+  facil_data->active = 0;
+  facil_cluster_signal_children();
+  for (int i = 0; i <= facil_data->capacity; ++i) {
+    intptr_t uuid;
+    if (fd_data(i).protocol && is_counted_protocol(fd_data(i).protocol) &&
+        (uuid = sock_fd2uuid(i)) >= 0) {
+      defer(deferred_on_shutdown, (void *)uuid, NULL);
+    }
+  }
+  facil_data->thread_pool = defer_pool_start(facil_data->threads);
+  if (facil_data->thread_pool) {
+    defer(facil_cycle_unwind, NULL, NULL);
+    defer_pool_wait(facil_data->thread_pool);
+    facil_data->thread_pool = NULL;
+  }
   fprintf(stderr, "* %d cleanning up.\n", getpid());
-  intptr_t uuid;
-  for (intptr_t i = 0; i < facil_data->capacity; i++) {
+  /* close leftovers */
+  for (int i = 0; i <= facil_data->capacity; ++i) {
+    intptr_t uuid;
     if (fd_data(i).protocol && (uuid = sock_fd2uuid(i)) >= 0) {
-      defer(deferred_on_shutdown, (void *)uuid, NULL);
+      sock_force_close(uuid);
     }
   }
-  facil_cycle(arg, NULL);
   defer_perform();
-  facil_cycle(arg, NULL);
-  ((struct facil_run_args *)arg)->on_finish();
+  facil_data->on_finish();
   defer_perform();
   evio_close();
-  facil_cluster_destroy();
   facil_external_cleanup();
+
+  if (facil_data->parent == getpid()) {
+    kill(0, SIGINT);
+    while (wait(NULL) != -1)
+      ;
+    if (FACIL_PRINT_STATE) {
+      fprintf(stderr, "\n   ---  Shutdown Complete  ---\n");
+    }
+  }
+}
+
+static void facil_sentinel_task(void *arg1, void *arg2);
+static void *facil_sentinel_worker_thread(void *arg) {
+  errno = 0;
+  pid_t child = facil_fork();
+  if (arg && child) {
+    spn_unlock((spn_lock_i *)arg);
+  }
+  if (child == -1) {
+    perror("FATAL ERROR: couldn't spawn worker.");
+    kill(facil_parent_pid(), SIGINT);
+    facil_stop();
+    return NULL;
+  } else if (child) {
+    int status;
+    waitpid(child, &status, 0);
+#if DEBUG
+    if (facil_data->active) { /* !WIFEXITED(status) || WEXITSTATUS(status) */
+      if (!WIFEXITED(status) || WEXITSTATUS(status)) {
+        fprintf(stderr,
+                "FATAL ERROR: Child worker (%d) crashed. Stopping services in "
+                "DEBUG mode.\n",
+                child);
+      } else {
+        fprintf(
+            stderr,
+            "INFO (FATAL): Child worker (%d) shutdown. Stopping services in "
+            "DEBUG mode.\n",
+            child);
+      }
+      kill(0, SIGINT);
+    }
+#else
+    if (facil_data->active) {
+      if (!WIFEXITED(status) || WEXITSTATUS(status)) {
+        fprintf(stderr,
+                "ERROR: Child worker (%d) crashed. Respawning worker.\n",
+                child);
+      } else {
+        fprintf(stderr,
+                "INFO: Child worker (%d) shutdown. Respawning worker.\n",
+                child);
+      }
+      defer(facil_sentinel_task, NULL, NULL);
+    }
+#endif
+  } else {
+    facil_worker_startup(0);
+    facil_worker_cleanup();
+    exit(0);
+  }
+  return NULL;
+}
+
+#if FIO_SENTINEL_USE_PTHREAD
+static void facil_sentinel_task(void *arg1, void *arg2) {
+  if (!facil_data->active)
+    return;
+  spn_lock_i pre_fork_lock = SPN_LOCK_INIT;
+  spn_lock(&pre_fork_lock);
+  pthread_t sentinel;
+  if (pthread_create(&sentinel, NULL, facil_sentinel_worker_thread,
+                     (void *)&pre_fork_lock)) {
+    perror("FATAL ERROR: couldn't start sentinel thread");
+    exit(errno);
+  }
+  pthread_detach(sentinel);
+  spn_lock(&pre_fork_lock); /* will wait for worker thread to release lock. */
+  facil_cluster_data.listening.on_data(facil_cluster_data.root,
+                                       &facil_cluster_data.listening);
+  (void)arg1;
+  (void)arg2;
+}
+#else
+static void facil_sentinel_task(void *arg1, void *arg2) {
+  if (!facil_data->active)
+    return;
+  spn_lock_i pre_fork_lock = SPN_LOCK_INIT;
+  spn_lock(&pre_fork_lock);
+  void *thrd =
+      defer_new_thread(facil_sentinel_worker_thread, (void *)&pre_fork_lock);
+  defer_free_thread(thrd);
+  spn_lock(&pre_fork_lock); /* will wait for worker thread to release lock. */
+  facil_cluster_data.listening.on_data(facil_cluster_data.root,
+                                       &facil_cluster_data.listening);
+  (void)arg1;
+  (void)arg2;
+}
+#endif
+
+/* handles the SIGUSR1, SIGINT and SIGTERM signals. */
+static void sig_int_handler(int sig) {
+  switch (sig) {
+#if !FACIL_DISABLE_HOT_RESTART
+  case SIGUSR1:
+    facil_signal_children_flag = 1;
+    break;
+#endif
+  case SIGINT:  /* fallthrough */
+  case SIGTERM: /* fallthrough */
+    facil_stop();
+    break;
+  default:
+    break;
+  }
+}
+
+/* setup handling for the SIGUSR1, SIGPIPE, SIGINT and SIGTERM signals. */
+static void facil_setup_signal_handler(void) {
+  /* setup signal handling */
+  struct sigaction act, old;
+
+  act.sa_handler = sig_int_handler;
+  sigemptyset(&act.sa_mask);
+  act.sa_flags = SA_RESTART | SA_NOCLDSTOP;
+
+  if (sigaction(SIGINT, &act, &old)) {
+    perror("couldn't set signal handler");
+    return;
+  };
+
+  if (sigaction(SIGTERM, &act, &old)) {
+    perror("couldn't set signal handler");
+    return;
+  };
+#if !FACIL_DISABLE_HOT_RESTART
+  if (sigaction(SIGUSR1, &act, &old)) {
+    perror("couldn't set signal handler");
+    return;
+  };
+#endif
+
+  act.sa_handler = SIG_IGN;
+  if (sigaction(SIGPIPE, &act, &old)) {
+    perror("couldn't set signal handler");
+    return;
+  };
+}
+
+/*
+ * Zombie Reaping
+ * With thanks to Dr Graham D Shaw.
+ * http://www.microhowto.info/howto/reap_zombie_processes_using_a_sigchld_handler.html
+ */
+static void reap_child_handler(int sig) {
+  (void)(sig);
+  int old_errno = errno;
+  while (waitpid(-1, NULL, WNOHANG) > 0)
+    ;
+  errno = old_errno;
+}
+
+/* initializes zombie reaping for the process */
+void facil_reap_children(void) {
+  struct sigaction sa;
+  sa.sa_handler = reap_child_handler;
+  sigemptyset(&sa.sa_mask);
+  sa.sa_flags = SA_RESTART | SA_NOCLDSTOP;
+  if (sigaction(SIGCHLD, &sa, 0) == -1) {
+    perror("Child reaping initialization failed");
+    kill(0, SIGINT);
+    exit(errno);
+  }
+}
+
+/** returns facil.io's parent (root) process pid. */
+pid_t facil_parent_pid(void) {
+  if (!facil_data)
+    facil_lib_init();
+  return facil_data->parent;
+}
+
+static inline size_t facil_detect_cpu_cores(void) {
+  ssize_t cpu_count = 0;
+#ifdef _SC_NPROCESSORS_ONLN
+  cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
+  if (cpu_count < 0) {
+    if (FACIL_PRINT_STATE) {
+      fprintf(stderr, "WARNING: CPU core count auto-detection failed.\n");
+    }
+    return 0;
+  }
+#else
+  if (1 || FACIL_PRINT_STATE) {
+    fprintf(stderr, "WARNING: CPU core count auto-detection failed.\n");
+  }
+#endif
+  return cpu_count;
 }
 
 #undef facil_run
@@ -1074,60 +1889,106 @@ void facil_run(struct facil_run_args args) {
     args.on_idle = mock_idle;
   if (!args.on_finish)
     args.on_finish = mock_idle;
-#ifdef _SC_NPROCESSORS_ONLN
   if (!args.threads && !args.processes) {
-    ssize_t cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
-    if (cpu_count > 0)
-      args.threads = args.processes = cpu_count;
-  }
+    /* both options set to 0 - default to cores*cores matrix */
+    ssize_t cpu_count = facil_detect_cpu_cores();
+#if FACIL_CPU_CORES_LIMIT
+    if (cpu_count > FACIL_CPU_CORES_LIMIT) {
+      fprintf(
+          stderr,
+          "INFO: Detected %zu cores. Capping auto-detection of cores "
+          "to %zu.\n"
+          "      Avoid this message by setting threads / workers manually.\n"
+          "      To increase auto-detection limit, recompile with:\n"
+          "             -DFACIL_CPU_CORES_LIMIT=%zu \n",
+          (size_t)cpu_count, (size_t)FACIL_CPU_CORES_LIMIT, (size_t)cpu_count);
+      cpu_count = FACIL_CPU_CORES_LIMIT;
+    }
 #endif
-  if (!args.processes)
+    args.threads = args.processes = (int16_t)cpu_count;
+  } else if (args.threads < 0 || args.processes < 0) {
+    /* Set any option that is less than 0 be equal to cores/value */
+    /* Set any option equal to 0 be equal to the other option in value */
+    ssize_t cpu_count = facil_detect_cpu_cores();
+
+    if (cpu_count > 0) {
+      int16_t threads = 0;
+      if (args.threads < 0)
+        threads = (int16_t)(cpu_count / (args.threads * -1));
+      else if (args.threads == 0)
+        threads = -1 * args.processes;
+      if (args.processes < 0)
+        args.processes = (int16_t)(cpu_count / (args.processes * -1));
+      else if (args.processes == 0)
+        args.processes = -1 * args.threads;
+      args.threads = threads;
+    }
+  }
+
+  /* make sure we have at least one process and at least one thread */
+  if (args.processes <= 0)
     args.processes = 1;
-  if (!args.threads)
+  if (args.threads <= 0)
     args.threads = 1;
-  facil_cluster_init(args.processes);
-  if (FACIL_PRINT_STATE) {
-    fprintf(stderr, "Server is running %u %s X %u %s, press ^C to stop\n",
-            args.processes, args.processes > 1 ? "workers" : "worker",
-            args.threads, args.threads > 1 ? "threads" : "thread");
-    defer(print_pid, NULL, NULL);
-  }
-  defer(facil_init_run, &args, NULL);
-  int frk = defer_perform_in_fork(args.processes, args.threads);
-  facil_cleanup(&args);
-  if (frk < 0) {
-    perror("ERROR: couldn't spawn workers");
-  } else if (frk > 0) {
-    exit(0);
+
+  /* listen to SIGINT / SIGTERM */
+  facil_setup_signal_handler();
+
+  /* activate facil, fork if needed */
+  facil_data->active = (uint16_t)args.processes;
+  facil_data->threads = (uint16_t)args.threads;
+  facil_data->on_finish = args.on_finish;
+  facil_data->on_idle = args.on_idle;
+  /* initialize cluster */
+  if (args.processes > 1) {
+    if (facil_cluster_init()) {
+      kill(0, SIGINT);
+      if (FACIL_PRINT_STATE) {
+        fprintf(stderr, "\n   !!!  Crashed trying to "
+                        "start the service  !!!\n");
+      }
+      exit(-1);
+    }
+    for (int i = 0; i < args.processes && facil_data->active; ++i) {
+      facil_sentinel_task(NULL, NULL);
+    }
+    facil_worker_startup(1);
+  } else {
+    facil_worker_startup(0);
   }
-  if (FACIL_PRINT_STATE)
-    fprintf(stderr, "\n   ---  Completed Shutdown  ---\n");
+  facil_worker_cleanup();
 }
 
 /**
  * returns true (1) if the facil.io engine is already running.
  */
-int facil_is_running(void) { return (facil_data && defer_fork_is_active()); }
+int facil_is_running(void) { return (facil_data && facil_data->active > 0); }
 
 /* *****************************************************************************
 Setting the protocol
 ***************************************************************************** */
 
+/* managing the protocol pointer array and the `on_close` callback */
 static int facil_attach_state(intptr_t uuid, protocol_s *protocol,
                               protocol_metadata_s state) {
   if (!facil_data)
     facil_lib_init();
   if (protocol) {
-    if (!protocol->on_close)
+    if (!protocol->on_close) {
       protocol->on_close = mock_on_close;
-    if (!protocol->on_data)
-      protocol->on_data = mock_on_ev;
-    if (!protocol->on_ready)
+    }
+    if (!protocol->on_data) {
+      protocol->on_data = mock_on_data;
+    }
+    if (!protocol->on_ready) {
       protocol->on_ready = mock_on_ev;
-    if (!protocol->ping)
+    }
+    if (!protocol->ping) {
       protocol->ping = mock_ping;
-    if (!protocol->on_shutdown)
+    }
+    if (!protocol->on_shutdown) {
       protocol->on_shutdown = mock_on_ev;
+    }
     prt_meta(protocol) = state;
   }
   spn_lock(&uuid_data(uuid).lock);
@@ -1135,16 +1996,28 @@ static int facil_attach_state(intptr_t uuid, protocol_s *protocol,
     spn_unlock(&uuid_data(uuid).lock);
     if (protocol)
       defer(deferred_on_close, (void *)uuid, protocol);
+    if (uuid == -1)
+      errno = EBADF;
+    else
+      errno = ENOTCONN;
     return -1;
+  } else {
+    if (is_counted_protocol(protocol)) {
+      spn_add(&facil_data->connection_count, 1);
+    }
   }
   protocol_s *old_protocol = uuid_data(uuid).protocol;
   uuid_data(uuid).protocol = protocol;
-  uuid_data(uuid).active = facil_data->last_cycle;
+  uuid_data(uuid).active = facil_data->last_cycle.tv_sec;
   spn_unlock(&uuid_data(uuid).lock);
-  if (old_protocol)
+  if (old_protocol) {
+    if (is_counted_protocol(old_protocol)) {
+      spn_sub(&facil_data->connection_count, 1);
+    }
     defer(deferred_on_close, (void *)uuid, old_protocol);
-  if (evio_isactive())
-    evio_add(sock_uuid2fd(uuid), (void *)uuid);
+  } else if (evio_isactive()) {
+    return evio_add(sock_uuid2fd(uuid), (void *)uuid);
+  }
   return 0;
 }
 
@@ -1169,7 +2042,7 @@ int facil_attach_locked(intptr_t uuid, protocol_s *protocol) {
 /** Sets a timeout for a specific connection (if active). */
 void facil_set_timeout(intptr_t uuid, uint8_t timeout) {
   if (sock_isvalid(uuid)) {
-    uuid_data(uuid).active = facil_data->last_cycle;
+    uuid_data(uuid).active = facil_data->last_cycle.tv_sec;
     uuid_data(uuid).timeout = timeout;
   }
 }
@@ -1184,8 +2057,12 @@ Misc helpers
 /**
 Returns the last time the server reviewed any pending IO events.
 */
-time_t facil_last_tick(void) {
-  return facil_data ? facil_data->last_cycle : time(NULL);
+struct timespec facil_last_tick(void) {
+  if (!facil_data) {
+    facil_lib_init();
+    clock_gettime(CLOCK_REALTIME, &facil_data->last_cycle);
+  }
+  return facil_data->last_cycle;
 }
 
 /**
@@ -1208,15 +2085,14 @@ void facil_protocol_unlock(protocol_s *pr, enum facil_protocol_lock_e type) {
 }
 /** Counts all the connections of a specific type. */
 size_t facil_count(void *service) {
-  long count = 0;
-  void *tmp;
+  size_t count = 0;
   for (intptr_t i = 0; i < facil_data->capacity; i++) {
-    tmp = NULL;
+    void *tmp = NULL;
     spn_lock(&fd_data(i).lock);
     if (fd_data(i).protocol && fd_data(i).protocol->service)
       tmp = (void *)fd_data(i).protocol->service;
     spn_unlock(&fd_data(i).lock);
-    if (tmp != listener_protocol_name && tmp != timer_protocol_name &&
+    if (tmp != LISTENER_PROTOCOL_NAME && tmp != TIMER_PROTOCOL_NAME &&
         (!service || (tmp == service)))
       count++;
   }
@@ -1239,10 +2115,10 @@ struct task {
 };
 
 static inline struct task *alloc_facil_task(void) {
-  return malloc(sizeof(struct task));
+  return fio_malloc(sizeof(struct task));
 }
 
-static inline void free_facil_task(struct task *task) { free(task); }
+static inline void free_facil_task(struct task *task) { fio_free(task); }
 
 static void mock_on_task_done(intptr_t uuid, void *arg) {
   (void)uuid;
@@ -1256,7 +2132,7 @@ static void perform_single_task(void *v_uuid, void *v_task) {
   protocol_s *pr = protocol_try_lock(sock_uuid2fd(v_uuid), task->task_type);
   if (!pr)
     goto defer;
-  if (pr->service == connector_protocol_name) {
+  if (pr->service == CONNECTOR_PROTOCOL_NAME) {
     protocol_unlock(pr, task->task_type);
     goto defer;
   }
@@ -1298,8 +2174,10 @@ static void perform_multi_task(void *v_fd, void *v_task) {
   protocol_s *pr = protocol_try_lock((intptr_t)v_fd, task->task_type);
   if (!pr)
     goto reschedule;
-  if (pr->service == task->service)
-    task->func(sock_fd2uuid((intptr_t)v_fd), pr, task->arg);
+  if (pr->service == task->service) {
+    const intptr_t uuid = sock_fd2uuid((int)(intptr_t)v_fd);
+    task->func(uuid, pr, task->arg);
+  }
   protocol_unlock(pr, task->task_type);
   defer(finish_multi_task, v_fd, v_task);
   return;
@@ -1350,8 +2228,8 @@ complete:
 void facil_defer(struct facil_defer_args_s args) {
   if (!args.fallback)
     args.fallback = mock_on_task_done;
-  if (!args.task_type)
-    args.task_type = FIO_PR_LOCK_TASK;
+  if (!args.type)
+    args.type = FIO_PR_LOCK_TASK;
   if (!args.task || !uuid_data(args.uuid).protocol || args.uuid < 0 ||
       !sock_isvalid(args.uuid))
     goto error;