iodine 0.6.5 → 0.7.0

data/ext/iodine/redis_engine.c

@@ -4,12 +4,17 @@ License: MIT
 
 Feel free to copy, use and enjoy according to the license provided.
 */
-#include "spnlock.inc"
 
-#include "fio_llist.h"
-#include "fiobj4sock.h"
-#include "redis_engine.h"
-#include "resp_parser.h"
+#define FIO_INCLUDE_LINKED_LIST
+#define FIO_INCLUDE_STR
+// #define DEBUG 1
+#include <fio.h>
+
+#include <fiobj.h>
+#include <fiobj4sock.h>
+
+#include <redis_engine.h>
+#include <resp_parser.h>
 
 #define REDIS_READ_BUFFER 8192
 /* *****************************************************************************
@@ -17,282 +22,390 @@ The Redis Engine and Callbacks Object
 ***************************************************************************** */
 
 typedef struct {
-  uintptr_t id_protection;
-  pubsub_engine_s en;
+  fio_pubsub_engine_s en;
   struct redis_engine_internal_s {
-    protocol_s protocol;
-    uintptr_t uuid;
+    fio_protocol_s protocol;
+    intptr_t uuid;
     resp_parser_s parser;
-    uintptr_t is_pub;
+    void (*on_message)(struct redis_engine_internal_s *parser, FIOBJ msg);
     FIOBJ str;
     FIOBJ ary;
-    uintptr_t ary_count;
-    uintptr_t buf_pos;
+    uint32_t ary_count;
+    uint16_t buf_pos;
+    uint16_t nesting;
   } pub_data, sub_data;
-  fio_ls_embd_s callbacks;
-  spn_lock_i lock;
+  subscription_s *publication_forwarder;
+  subscription_s *cmd_forwarder;
+  subscription_s *cmd_reply;
   char *address;
   char *port;
   char *auth;
   FIOBJ last_ch;
   size_t auth_len;
   size_t ref;
+  fio_ls_embd_s queue;
+  fio_lock_i lock;
   uint8_t ping_int;
-  uint8_t sent;
-  uint8_t flag;
+  uint8_t pub_send;
+  volatile uint8_t flag;
   uint8_t buf[];
 } redis_engine_s;
 
 typedef struct {
   fio_ls_embd_s node;
-  void (*callback)(pubsub_engine_s *e, FIOBJ reply, void *udata);
+  void (*callback)(fio_pubsub_engine_s *e, FIOBJ reply, void *udata);
   void *udata;
   size_t cmd_len;
   uint8_t cmd[];
 } redis_commands_s;
 
-/** converst from a `pubsub_engine_s` to a `redis_engine_s`. */
-#define en2redis(e) FIO_LS_EMBD_OBJ(redis_engine_s, en, (e))
+/** converts from a publishing protocol to an `redis_engine_s`. */
+#define pub2redis(pr) FIO_LS_EMBD_OBJ(redis_engine_s, pub_data, (pr))
+/** converts from a subscribing protocol to an `redis_engine_s`. */
+#define sub2redis(pr) FIO_LS_EMBD_OBJ(redis_engine_s, sub_data, (pr))
 
-/** converst from a `protocol_s` to a `redis_engine_s`. */
-#define prot2redis(prot)                                                       \
-  ((FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, protocol, (prot))->is_pub) \
-       ? FIO_LS_EMBD_OBJ(redis_engine_s, pub_data.protocol, (pr))              \
-       : FIO_LS_EMBD_OBJ(redis_engine_s, sub_data.protocol, (pr)))
-
-/** converst from a `resp_parser_s` to the internal data structure. */
+/** converts from a `resp_parser_s` to the internal data structure. */
 #define parser2data(prsr)                                                      \
   FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, (prsr))
 
-#define parser2redis(prsr)                                                     \
-  (parser2data(prsr)->is_pub                                                   \
-       ? FIO_LS_EMBD_OBJ(redis_engine_s, pub_data.parser, (prsr))              \
-       : FIO_LS_EMBD_OBJ(redis_engine_s, sub_data.parser, (prsr)))
+/* releases any resources used by an internal engine*/
+static inline void redis_internal_reset(struct redis_engine_internal_s *i) {
+  i->buf_pos = 0;
+  i->parser = (resp_parser_s){.obj_countdown = 0, .expecting = 0};
+  fiobj_free(i->str);
+  i->str = FIOBJ_INVALID;
+  fiobj_free(i->ary);
+  i->ary = FIOBJ_INVALID;
+  i->ary_count = 0;
+  i->nesting = 0;
+  i->uuid = -1;
+}
 
 /** cleans up and frees the engine data. */
 static inline void redis_free(redis_engine_s *r) {
-  if (spn_sub(&r->ref, 1))
+  if (fio_atomic_sub(&r->ref, 1))
     return;
-  fiobj_free(r->pub_data.ary ? r->pub_data.ary : r->pub_data.str);
-  fiobj_free(r->sub_data.ary ? r->sub_data.ary : r->sub_data.str);
+  redis_internal_reset(&r->pub_data);
+  redis_internal_reset(&r->sub_data);
   fiobj_free(r->last_ch);
-  while (fio_ls_embd_any(&r->callbacks)) {
-    free(FIO_LS_EMBD_OBJ(redis_commands_s, node,
-                         fio_ls_embd_pop(&r->callbacks)));
+  while (fio_ls_embd_any(&r->queue)) {
+    fio_free(
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, fio_ls_embd_pop(&r->queue)));
+  }
+  fio_unsubscribe(r->publication_forwarder);
+  r->publication_forwarder = NULL;
+  fio_unsubscribe(r->cmd_forwarder);
+  r->cmd_forwarder = NULL;
+  fio_unsubscribe(r->cmd_reply);
+  r->cmd_reply = NULL;
+  fio_free(r);
+}
+
+/* *****************************************************************************
+Simple RESP formatting
+***************************************************************************** */
+
+inline static void fiobj2resp2(FIOBJ dest, FIOBJ obj) {
+  if (fiobj_hash_key_in_loop())
+    fiobj2resp2(dest, fiobj_hash_key_in_loop());
+  fio_str_info_s s;
+  switch (FIOBJ_TYPE(obj)) {
+  case FIOBJ_T_NULL:
+    fiobj_str_write(dest, "$-1\r\n", 5);
+    break;
+  case FIOBJ_T_ARRAY:
+    fiobj_str_write(dest, "*", 1);
+    fiobj_str_write_i(dest, fiobj_ary_count(obj));
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+  case FIOBJ_T_HASH:
+    fiobj_str_write(dest, "*", 1);
+    fiobj_str_write_i(dest, fiobj_hash_count(obj) * 2);
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+  case FIOBJ_T_TRUE:
+    fiobj_str_write(dest, "$4\r\ntrue\r\n", 10);
+    break;
+  case FIOBJ_T_FALSE:
+    fiobj_str_write(dest, "$4\r\nfalse\r\n", 11);
+    break;
+#if 0
+    /* Numbers aren't as good for commands as one might think... */
+  case FIOBJ_T_NUMBER:
+    fiobj_str_write(dest, ":", 1);
+    fiobj_str_write_i(dest, fiobj_obj2num(obj));
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+#else
+  case FIOBJ_T_NUMBER: /* overflow */
+#endif
+  case FIOBJ_T_FLOAT:   /* overflow */
+  case FIOBJ_T_UNKNOWN: /* overflow */
+  case FIOBJ_T_STRING:  /* overflow */
+  case FIOBJ_T_DATA:
+    s = fiobj_obj2cstr(obj);
+    fiobj_str_write(dest, "$", 1);
+    fiobj_str_write_i(dest, s.len);
+    fiobj_str_write(dest, "\r\n", 2);
+    fiobj_str_write(dest, s.data, s.len);
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
   }
-  free(r);
+}
+
+static int fiobj2resp_task(FIOBJ o, void *dest_) {
+  fiobj2resp2((FIOBJ)dest_, o);
+  return 0;
 }
 
 /**
- * Defined later, converts FIOBJ objects into a RESP string (client mode).
+ * Converts FIOBJ objects into a RESP string (client mode).
+ */
+static FIOBJ fiobj2resp(FIOBJ dest, FIOBJ obj) {
+  fiobj_each2(obj, fiobj2resp_task, (void *)dest);
+  return dest;
+}
+
+/**
+ * Converts FIOBJ objects into a RESP string (client mode).
  *
  * Don't call `fiobj_free`, object will self-destruct.
  */
-static FIOBJ fiobj2resp_tmp(FIOBJ obj1, FIOBJ obj2);
+static inline FIOBJ fiobj2resp_tmp(FIOBJ obj) {
+  return fiobj2resp(fiobj_str_tmp(), obj);
+}
 
 /* *****************************************************************************
-Command / Callback handling
+RESP parser callbacks
 ***************************************************************************** */
 
-/* the deferred callback handler */
-static void redis_perform_callback(void *e, void *cmd_) {
-  redis_commands_s *cmd = cmd_;
-  FIOBJ reply = (FIOBJ)cmd->node.next;
-  if (cmd->callback)
-    cmd->callback(e, reply, cmd->udata);
-  fiobj_free(reply);
-  // fprintf(stderr, "Handled: %s\n", cmd->cmd);
-  free(cmd);
+/** a local static callback, called when a parser / protocol error occurs. */
+static int resp_on_parser_error(resp_parser_s *parser) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  FIO_LOG_STATE(
+      "ERROR: (redis) parser error - attempting to restart connection.\n");
+  fio_close(i->uuid);
+  return -1;
 }
 
-/* send the necxt command in the queue */
-static void redis_send_cmd_queue(void *r_, void *ignr) {
-  redis_engine_s *r = r_;
-  if (!r->pub_data.uuid)
-    return;
-  spn_lock(&r->lock);
-  if (r->sent == 0 && fio_ls_embd_any(&r->callbacks)) {
-    redis_commands_s *cmd =
-        FIO_LS_EMBD_OBJ(redis_commands_s, node, r->callbacks.prev);
-    intptr_t uuid = r->pub_data.uuid;
-    r->sent = 1;
-    spn_unlock(&r->lock);
-    // fprintf(stderr, "Sending: %s\n", cmd->cmd);
-    sock_write2(.uuid = uuid, .buffer = cmd->cmd, .length = cmd->cmd_len,
-                .dealloc = SOCK_DEALLOC_NOOP);
-  } else {
-    r->sent = 0;
-    spn_unlock(&r->lock);
-  }
-  (void)ignr;
+/** a local static callback, called when the RESP message is complete. */
+static int resp_on_message(resp_parser_s *parser) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  FIOBJ msg = i->ary ? i->ary : i->str;
+  i->on_message(i, msg);
+  /* cleanup */
+  fiobj_free(msg);
+  i->ary = FIOBJ_INVALID;
+  i->str = FIOBJ_INVALID;
+  return 0;
 }
 
-static void redis_attach_cmd(redis_engine_s *r, redis_commands_s *cmd) {
-  uint8_t schedule = 0;
-  spn_lock(&r->lock);
-  fio_ls_embd_push(&r->callbacks, &cmd->node);
-  if (r->sent == 0) {
-    schedule = 1;
-  }
-  spn_unlock(&r->lock);
-  if (schedule) {
-    defer(redis_send_cmd_queue, r, NULL);
+/** a local helper to add parsed objects to the data store. */
+static inline void resp_add_obj(struct redis_engine_internal_s *dest, FIOBJ o) {
+  if (dest->ary) {
+    fiobj_ary_push(dest->ary, o);
+    --dest->ary_count;
+    if (!dest->ary_count && dest->nesting) {
+      FIOBJ tmp = fiobj_ary_shift(dest->ary);
+      dest->ary_count = fiobj_obj2num(tmp);
+      fiobj_free(tmp);
+      dest->ary = fiobj_ary_shift(dest->ary);
+      --dest->nesting;
+    }
   }
+  dest->str = o;
 }
 
-static void redis_cmd_reply(redis_engine_s *r, FIOBJ reply) {
-  uint8_t schedule = 0;
-  spn_lock(&r->lock);
-  r->sent = 0;
-  fio_ls_embd_s *node = fio_ls_embd_shift(&r->callbacks);
-  schedule = fio_ls_embd_any(&r->callbacks);
-  spn_unlock(&r->lock);
-  if (!node) {
-    /* TODO: possible ping? from server?! not likely... */
-    fprintf(stderr,
-            "WARNING: (redis) received a reply when no command was sent\n");
-    return;
-  }
-  node->next = (void *)fiobj_dup(reply);
-  if (schedule)
-    defer(redis_send_cmd_queue, r, NULL);
-  defer(redis_perform_callback, &r->en,
-        FIO_LS_EMBD_OBJ(redis_commands_s, node, node));
+/** a local static callback, called when a Number object is parsed. */
+static int resp_on_number(resp_parser_s *parser, int64_t num) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_num_new(num));
+  return 0;
 }
-
-/* *****************************************************************************
-Connection Establishment
-***************************************************************************** */
-
-static void redis_on_auth(pubsub_engine_s *e, FIOBJ reply, void *udata) {
-  if (FIOBJ_TYPE_IS(reply, FIOBJ_T_TRUE)) {
-    fio_cstr_s s = fiobj_obj2cstr(reply);
-    fprintf(stderr,
-            "WARNING: (redis) Authentication FAILED.\n"
-            "        %.*s\n",
-            (int)s.len, s.data);
-  }
-  (void)e;
-  (void)udata;
+/** a local static callback, called when a OK message is received. */
+static int resp_on_okay(resp_parser_s *parser) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_true());
+  return 0;
+}
+/** a local static callback, called when NULL is received. */
+static int resp_on_null(resp_parser_s *parser) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_null());
+  return 0;
 }
 
-static void redis_on_pub_connect(intptr_t uuid, void *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  if (r->pub_data.uuid)
-    sock_close(r->pub_data.uuid);
-  r->pub_data.uuid = uuid;
-  facil_attach(uuid, pr);
-  facil_set_timeout(uuid, r->ping_int);
+/**
+ * a local static callback, called when a String should be allocated.
+ *
+ * `str_len` is the expected number of bytes that will fill the final string
+ * object, without any NUL byte marker (the string might be binary).
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_string(resp_parser_s *parser, size_t str_len) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_str_buf(str_len));
+  return 0;
+}
+/** a local static callback, called as String objects are streamed. */
+static int resp_on_string_chunk(resp_parser_s *parser, void *data, size_t len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  fiobj_str_write(i->str, data, len);
+  return 0;
+}
+/** a local static callback, called when a String object had finished
+ * streaming.
+ */
+static int resp_on_end_string(resp_parser_s *parser) {
+  return 0;
+  (void)parser;
+}
 
-  if (!facil_is_running() || !r->flag) {
-    sock_close(uuid);
-    return;
-  }
+/** a local static callback, called an error message is received. */
+static int resp_on_err_msg(resp_parser_s *parser, void *data, size_t len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  resp_add_obj(i, fiobj_str_new(data, len));
+  return 0;
+}
 
-  if (r->auth) {
-    redis_commands_s *cmd = malloc(sizeof(*cmd) + r->auth_len);
-    *cmd =
-        (redis_commands_s){.cmd_len = r->auth_len, .callback = redis_on_auth};
-    memcpy(cmd->cmd, r->auth, r->auth_len);
-    spn_lock(&r->lock);
-    fio_ls_embd_unshift(&r->callbacks, &cmd->node);
-    r->sent = 1;
-    spn_unlock(&r->lock);
-    sock_write2(.uuid = uuid, .buffer = cmd->cmd, .length = cmd->cmd_len,
-                .dealloc = SOCK_DEALLOC_NOOP);
+/**
+ * a local static callback, called when an Array should be allocated.
+ *
+ * `array_len` is the expected number of objects that will fill the Array
+ * object.
+ *
+ * There's no `resp_on_end_array` callback since the RESP protocol assumes the
+ * message is finished along with the Array (`resp_on_message` is called).
+ * However, just in case a non-conforming client/server sends nested Arrays,
+ * the callback should test against possible overflow or nested Array endings.
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_array(resp_parser_s *parser, size_t array_len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  if (i->ary) {
+    ++i->nesting;
+    FIOBJ tmp = fiobj_ary_new2(array_len + 2);
+    fiobj_ary_push(tmp, fiobj_num_new(i->ary_count));
+    fiobj_ary_push(tmp, fiobj_num_new(i->ary));
+    i->ary = tmp;
   } else {
-    r->sent = 0;
-    defer(redis_send_cmd_queue, r, NULL);
-  }
-  fprintf(stderr, "INFO: (redis %d) publishing connection established.\n",
-          (int)getpid());
-}
-static void redis_on_pub_connect_fail(intptr_t uuid, void *pr);
-static void redis_on_sub_connect(intptr_t uuid, void *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  r->sub_data.uuid = uuid;
-  facil_attach(uuid, pr);
-  facil_set_timeout(uuid, r->ping_int);
-
-  if (!facil_is_running() || !r->flag) {
-    sock_close(uuid);
-    return;
+    i->ary = fiobj_ary_new2(array_len + 2);
   }
+  i->ary_count = array_len;
+  return 0;
+}
 
-  if (r->auth)
-    sock_write2(.uuid = uuid, .buffer = r->auth, .length = r->auth_len,
-                .dealloc = SOCK_DEALLOC_NOOP);
-  pubsub_engine_resubscribe(&r->en);
-  if (!r->pub_data.uuid) {
-    spn_add(&r->ref, 1);
-    redis_on_pub_connect_fail(uuid, pr);
-  }
-  fprintf(stderr, "INFO: (redis %d) subscription connection established.\n",
-          (int)getpid());
+/* *****************************************************************************
+Publication and Command Handling
+***************************************************************************** */
+
+/* the deferred callback handler */
+static void redis_perform_callback(void *e, void *cmd_) {
+  redis_commands_s *cmd = cmd_;
+  FIOBJ reply = (FIOBJ)cmd->node.next;
+  if (cmd->callback)
+    cmd->callback(e, reply, cmd->udata);
+  fiobj_free(reply);
+  FIO_LOG_DEBUG("Handled: %s\n", cmd->cmd);
+  fio_free(cmd);
 }
 
-static void redis_deferred_connect(void *r_, void *is_pub);
-static void redis_on_pub_connect_fail(intptr_t uuid, void *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  if ((facil_parent_pid() == getpid() && !r->sub_data.uuid) || r->flag == 0 ||
-      !facil_is_running()) {
-    r->pub_data.uuid = 0;
-    redis_free(r);
-    return;
+/* attach a command to the queue */
+static void redis_attach_cmd(redis_engine_s *r, redis_commands_s *cmd) {
+  fio_lock(&r->lock);
+  fio_ls_embd_push(&r->queue, &cmd->node);
+  if (r->pub_send) {
+    fio_write2(r->pub_data.uuid, .data.buffer = cmd->cmd,
+               .length = cmd->cmd_len, .after.dealloc = FIO_DEALLOC_NOOP);
+    FIO_LOG_DEBUG("(%d) Sending (%zu bytes):\n%s\n", getpid(), cmd->cmd_len,
+                  cmd->cmd);
   }
-  r->pub_data.uuid = 0;
-  /* we defer publishing by a cycle, so subsciptions race a bit faster */
-  defer(redis_deferred_connect, r, (void *)1);
-  (void)uuid;
+  fio_unlock(&r->lock);
 }
-static void redis_on_sub_connect_fail(intptr_t uuid, void *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  if (!facil_is_running() || !r->flag) {
-    redis_free(r);
-    return;
-  }
-  if (facil_parent_pid() != getpid()) {
-    /* respawned as worker */
-    redis_on_pub_connect_fail(uuid, pr);
+
+/** a local static callback, called when the RESP message is complete. */
+static void resp_on_pub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
+  redis_engine_s *r = pub2redis(i);
+#if DEBUG
+  FIOBJ json = fiobj_obj2json(msg, 1);
+  FIO_LOG_DEBUG("Redis reply:\n%s\n", fiobj_obj2cstr(json).data);
+  fiobj_free(json);
+#endif
+  /* publishing / command parser */
+  fio_lock(&r->lock);
+  fio_ls_embd_s *node = fio_ls_embd_shift(&r->queue);
+  fio_unlock(&r->lock);
+  if (!node) {
+    /* TODO: possible ping? from server?! not likely... */
+    FIO_LOG_STATE(
+        "WARNING: (redis %d) received a reply when no command was sent.\n",
+        getpid());
     return;
   }
-  r->sub_data.uuid = 0;
-  defer(redis_deferred_connect, r, (void *)0);
-  (void)uuid;
+  node->next = (void *)fiobj_dup(msg);
+  fio_defer(redis_perform_callback, &r->en,
+            FIO_LS_EMBD_OBJ(redis_commands_s, node, node));
 }
 
-static void redis_deferred_connect(void *r_, void *is_pub) {
-  redis_engine_s *r = r_;
-  if (is_pub) {
-    facil_connect(.address = r->address, .port = r->port,
-                  .on_connect = redis_on_pub_connect,
-                  .udata = &r->pub_data.protocol,
-                  .on_fail = redis_on_pub_connect_fail);
+/* *****************************************************************************
+Subscription Message Handling
+***************************************************************************** */
 
+/** a local static callback, called when the RESP message is complete. */
+static void resp_on_sub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
+  redis_engine_s *r = sub2redis(i);
+  /* subscriotion parser */
+  if (FIOBJ_TYPE(msg) != FIOBJ_T_ARRAY) {
+    if (FIOBJ_TYPE(msg) != FIOBJ_T_STRING || fiobj_obj2cstr(msg).len != 4 ||
+        fiobj_obj2cstr(msg).data[0] != 'P') {
+      FIO_LOG_STATE("WARNING: (redis) unexpected data format in "
+                    "subscription stream:\n");
+      fio_str_info_s tmp = fiobj_obj2cstr(msg);
+      FIO_LOG_STATE("     %s\n", tmp.data);
+    }
   } else {
-    facil_connect(.address = r->address, .port = r->port,
-                  .on_connect = redis_on_sub_connect,
-                  .udata = &r->sub_data.protocol,
-                  .on_fail = redis_on_sub_connect_fail);
+    // FIOBJ *ary = fiobj_ary2ptr(msg);
+    // for (size_t i = 0; i < fiobj_ary_count(msg); ++i) {
+    //   fio_str_info_s tmp = fiobj_obj2cstr(ary[i]);
+    //   fprintf(stderr, "(%lu) %s\n", (unsigned long)i, tmp.data);
+    // }
+    fio_str_info_s tmp = fiobj_obj2cstr(fiobj_ary_index(msg, 0));
+    if (tmp.len == 7) { /* "message"  */
+      fiobj_free(r->last_ch);
+      r->last_ch = fiobj_dup(fiobj_ary_index(msg, 1));
+      fio_publish(.channel = fiobj_obj2cstr(r->last_ch),
+                  .message = fiobj_obj2cstr(fiobj_ary_index(msg, 2)),
+                  .engine = FIO_PUBSUB_CLUSTER);
+    } else if (tmp.len == 8) { /* "pmessage" */
+      if (!fiobj_iseq(r->last_ch, fiobj_ary_index(msg, 2)))
+        fio_publish(.channel = fiobj_obj2cstr(fiobj_ary_index(msg, 2)),
+                    .message = fiobj_obj2cstr(fiobj_ary_index(msg, 3)),
+                    .engine = FIO_PUBSUB_CLUSTER);
+    }
   }
 }
 
 /* *****************************************************************************
-The Protocol layer (connection handling)
+Connection Callbacks (fio_protocol_s) and Engine
 ***************************************************************************** */
 
-static void redis_on_data(intptr_t uuid, protocol_s *pr) {
-  redis_engine_s *r = prot2redis(pr);
+/** defined later - connects to Redis */
+static void redis_connect(void *r, void *i);
+
+/** Called when a data is available, but will not run concurrently */
+static void redis_on_data(intptr_t uuid, fio_protocol_s *pr) {
   struct redis_engine_internal_s *internal =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, protocol, pr);
+      (struct redis_engine_internal_s *)pr;
   uint8_t *buf;
-  if (internal->is_pub) {
-    buf = r->buf + REDIS_READ_BUFFER;
+  if (internal->on_message == resp_on_sub_message) {
+    buf = sub2redis(pr)->buf + REDIS_READ_BUFFER;
   } else {
-    buf = r->buf;
+    buf = pub2redis(pr)->buf;
   }
-  ssize_t i = sock_read(uuid, buf + internal->buf_pos,
-                        REDIS_READ_BUFFER - internal->buf_pos);
+  ssize_t i = fio_read(uuid, buf + internal->buf_pos,
+                       REDIS_READ_BUFFER - internal->buf_pos);
   if (i <= 0)
     return;
   internal->buf_pos += i;
@@ -303,534 +416,515 @@ static void redis_on_data(intptr_t uuid, protocol_s *pr) {
   internal->buf_pos = i;
 }
 
-static void redis_pub_on_close(intptr_t uuid, protocol_s *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  fiobj_free(r->pub_data.ary ? r->pub_data.ary : r->pub_data.str);
-  r->pub_data.ary = r->pub_data.str = FIOBJ_INVALID;
-  r->pub_data.uuid = 0;
-  r->sent = 0;
-  if (r->flag && facil_is_running()) {
-    fprintf(stderr,
-            "WARNING: (redis %d) lost publishing connection to database\n",
-            (int)getpid());
-    redis_on_pub_connect_fail(uuid, &r->pub_data.protocol);
-  } else {
-    redis_free(r);
-  }
-}
-
-static void redis_sub_on_close(intptr_t uuid, protocol_s *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  fiobj_free(r->sub_data.ary ? r->sub_data.ary : r->sub_data.str);
-  r->sub_data.ary = r->sub_data.str = FIOBJ_INVALID;
-  r->sub_data.uuid = 0;
-  if (r->flag && facil_is_running() && facil_parent_pid() == getpid()) {
-    fprintf(stderr,
-            "WARNING: (redis %d) lost subscribing connection to database\n",
-            (int)getpid());
-    redis_on_sub_connect_fail(uuid, &r->sub_data.protocol);
+/** Called when the connection was closed, but will not run concurrently */
+static void redis_on_close(intptr_t uuid, fio_protocol_s *pr) {
+  struct redis_engine_internal_s *internal =
+      (struct redis_engine_internal_s *)pr;
+  redis_internal_reset(internal);
+  redis_engine_s *r;
+  if (internal->on_message == resp_on_sub_message) {
+    r = sub2redis(pr);
+    fiobj_free(r->last_ch);
+    r->last_ch = FIOBJ_INVALID;
+    if (r->flag) {
+      /* reconnection for subscription connection. */
+      if (uuid != -1) {
+        FIO_LOG_STATE("WARNING: (redis %d) subscription connection lost. "
+                      "Reconnecting...\n",
+                      (int)getpid());
+      }
+      fio_atomic_sub(&r->ref, 1);
+      fio_defer(redis_connect, r, internal);
+    } else {
+      redis_free(r);
+    }
   } else {
+    r = pub2redis(pr);
+    if (r->flag && uuid != -1) {
+      FIO_LOG_STATE("WARNING: (redis %d) publication connection lost. "
+                    "Reconnecting...\n",
+                    (int)getpid());
+    }
+    r->pub_send = 0;
+    fio_close(r->sub_data.uuid);
     redis_free(r);
   }
+  (void)uuid;
 }
 
-static void redis_on_shutdown(intptr_t uuid, protocol_s *pr) {
-  sock_write2(.uuid = uuid, .buffer = "*1\r\n$4\r\nQUIT\r\n", .length = 14,
-              .dealloc = SOCK_DEALLOC_NOOP);
+/** Called before the facil.io reactor is shut down. */
+static uint8_t redis_on_shutdown(intptr_t uuid, fio_protocol_s *pr) {
+  fio_write2(uuid, .data.buffer = "*1\r\n$4\r\nQUIT\r\n", .length = 14,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  return 0;
   (void)pr;
 }
 
-static void redis_sub_ping(intptr_t uuid, protocol_s *pr) {
-  sock_write2(.uuid = uuid, .buffer = "*1\r\n$4\r\nPING\r\n", .length = 14,
-              .dealloc = SOCK_DEALLOC_NOOP);
+/** Called on connection timeout. */
+static void redis_sub_ping(intptr_t uuid, fio_protocol_s *pr) {
+  fio_write2(uuid, .data.buffer = "*1\r\n$4\r\nPING\r\n", .length = 14,
+             .after.dealloc = FIO_DEALLOC_NOOP);
   (void)pr;
 }
 
-static void redis_pub_ping(intptr_t uuid, protocol_s *pr) {
-  redis_engine_s *r = prot2redis(pr);
-  if (r->sent) {
-    fprintf(stderr,
-            "WARNING: (redis) Redis server unresponsive, disconnecting.\n");
-    sock_close(uuid);
+/** Called on connection timeout. */
+static void redis_pub_ping(intptr_t uuid, fio_protocol_s *pr) {
+  redis_engine_s *r = pub2redis(pr);
+  if (fio_ls_embd_any(&r->queue)) {
+    FIO_LOG_STATE(
+        "WARNING: (redis) Redis server unresponsive, disconnecting.\n");
+    fio_close(uuid);
     return;
   }
-  redis_commands_s *cmd = malloc(sizeof(*cmd) + 15);
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + 15);
   *cmd = (redis_commands_s){.cmd_len = 14};
   memcpy(cmd->cmd, "*1\r\n$4\r\nPING\r\n\0", 15);
   redis_attach_cmd(r, cmd);
 }
 
 /* *****************************************************************************
-Engine Callbacks
+Connecting to Redis
+***************************************************************************** */
+
+static void redis_on_auth(fio_pubsub_engine_s *e, FIOBJ reply, void *udata) {
+  if (FIOBJ_TYPE_IS(reply, FIOBJ_T_TRUE)) {
+    fio_str_info_s s = fiobj_obj2cstr(reply);
+    FIO_LOG_STATE("WARNING: (redis) Authentication FAILED.\n"
+                  "        %.*s\n",
+                  (int)s.len, s.data);
+  }
+  (void)e;
+  (void)udata;
+}
+
+static void redis_on_connect(intptr_t uuid, void *i_) {
+  struct redis_engine_internal_s *i = i_;
+  redis_engine_s *r;
+  i->uuid = uuid;
+
+  if (i->on_message == resp_on_sub_message) {
+    r = sub2redis(i);
+    if (r->auth_len) {
+      fio_write2(uuid, .data.buffer = r->auth, .length = r->auth_len,
+                 .after.dealloc = FIO_DEALLOC_NOOP);
+    }
+    fio_pubsub_reattach(&r->en);
+    if (r->pub_data.uuid == -1) {
+      fio_defer(redis_connect, r, &r->pub_data);
+    }
+    FIO_LOG_STATE("INFO: (redis %d) subscription connection established.\n",
+                  (int)getpid());
+  } else {
+    r = pub2redis(i);
+    if (r->auth_len) {
+      redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + r->auth_len);
+      *cmd =
+          (redis_commands_s){.cmd_len = r->auth_len, .callback = redis_on_auth};
+      memcpy(cmd->cmd, r->auth, r->auth_len);
+      fio_lock(&r->lock);
+      fio_ls_embd_unshift(&r->queue, &cmd->node);
+      fio_unlock(&r->lock);
+    }
+    fio_lock(&r->lock);
+    FIO_LS_EMBD_FOR(&r->queue, node) {
+      redis_commands_s *cmd = FIO_LS_EMBD_OBJ(redis_commands_s, node, node);
+      FIO_LOG_DEBUG("(%d) Sending (%zu bytes):\n%s\n", getpid(), cmd->cmd_len,
+                    cmd->cmd);
+      fio_write2(uuid, .data.buffer = cmd->cmd, .length = cmd->cmd_len,
+                 .after.dealloc = FIO_DEALLOC_NOOP);
+    }
+    r->pub_send = 1;
+    fio_unlock(&r->lock);
+    FIO_LOG_STATE("INFO: (redis %d) publication connection established.\n",
+                  (int)getpid());
+  }
+
+  i->protocol.rsv = 0;
+  fio_attach(uuid, &i->protocol);
+  fio_timeout_set(uuid, r->ping_int);
+
+  return;
+}
+
+static void redis_on_connect_failed(intptr_t uuid, void *i_) {
+  struct redis_engine_internal_s *i = i_;
+  i->uuid = -1;
+  i->protocol.on_close(-1, &i->protocol);
+  (void)uuid;
+}
+
+static void redis_connect(void *r_, void *i_) {
+  redis_engine_s *r = r_;
+  struct redis_engine_internal_s *i = i_;
+  if (r->flag == 0 || i->uuid != -1 || !fio_is_running())
+    return;
+  fio_atomic_add(&r->ref, 1);
+  i->uuid = fio_connect(.address = r->address, .port = r->port,
+                        .on_connect = redis_on_connect, .udata = i,
+                        .on_fail = redis_on_connect_failed);
+}
+
+/* *****************************************************************************
+Engine / Bridge Callbacks (Root Process)
 ***************************************************************************** */
 
-static void redis_on_subscribe(const pubsub_engine_s *eng, FIOBJ channel,
-                               uint8_t use_pattern) {
-  redis_engine_s *r = en2redis(eng);
-  if (r->sub_data.uuid) {
-    fio_cstr_s ch_str = fiobj_obj2cstr(channel);
-    FIOBJ cmd = fiobj_str_buf(96 + ch_str.len);
-    if (use_pattern)
+static void redis_on_subscribe_root(const fio_pubsub_engine_s *eng,
+                                    fio_str_info_s channel,
+                                    fio_match_fn match) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  if (r->sub_data.uuid != -1) {
+    FIOBJ cmd = fiobj_str_buf(96 + channel.len);
+    if (match == FIO_MATCH_GLOB)
       fiobj_str_write(cmd, "*2\r\n$10\r\nPSUBSCRIBE\r\n$", 22);
     else
       fiobj_str_write(cmd, "*2\r\n$9\r\nSUBSCRIBE\r\n$", 20);
-    fiobj_str_join(cmd, fiobj_num_tmp(ch_str.len));
+    fiobj_str_write_i(cmd, channel.len);
     fiobj_str_write(cmd, "\r\n", 2);
-    fiobj_str_write(cmd, ch_str.data, ch_str.len);
+    fiobj_str_write(cmd, channel.data, channel.len);
     fiobj_str_write(cmd, "\r\n", 2);
     // {
-    //   fio_cstr_s s = fiobj_obj2cstr(cmd);
-    //   fprintf(stderr, "%s\n", s.data);
+    //   fio_str_info_s s = fiobj_obj2cstr(cmd);
+    //   fprintf(stderr, "(%d) Sending Subscription (%p):\n%s\n", getpid(),
+    //           (void *)r->sub_data.uuid, s.data);
     // }
     fiobj_send_free(r->sub_data.uuid, cmd);
   }
 }
-static void redis_on_unsubscribe(const pubsub_engine_s *eng, FIOBJ channel,
-                                 uint8_t use_pattern) {
-  redis_engine_s *r = en2redis(eng);
-  if (r->sub_data.uuid) {
-    fio_cstr_s ch_str = fiobj_obj2cstr(channel);
-    FIOBJ cmd = fiobj_str_buf(96 + ch_str.len);
-    if (use_pattern)
-      fiobj_str_write(cmd, "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$", 24);
+
+static void redis_on_unsubscribe_root(const fio_pubsub_engine_s *eng,
+                                      fio_str_info_s channel,
+                                      fio_match_fn match) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  if (r->sub_data.uuid != -1) {
+    fio_str_s *cmd = fio_str_new2();
+    fio_str_capa_assert(cmd, 96 + channel.len);
+    if (match == FIO_MATCH_GLOB)
+      fio_str_write(cmd, "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$", 24);
     else
-      fiobj_str_write(cmd, "*2\r\n$11\r\nUNSUBSCRIBE\r\n$", 23);
-    fiobj_str_join(cmd, fiobj_num_tmp(ch_str.len));
-    fiobj_str_write(cmd, "\r\n", 2);
-    fiobj_str_write(cmd, ch_str.data, ch_str.len);
-    fiobj_str_write(cmd, "\r\n", 2);
+      fio_str_write(cmd, "*2\r\n$11\r\nUNSUBSCRIBE\r\n$", 23);
+    fio_str_write_i(cmd, channel.len);
+    fio_str_write(cmd, "\r\n", 2);
+    fio_str_write(cmd, channel.data, channel.len);
+    fio_str_write(cmd, "\r\n", 2);
     // {
-    //   fio_cstr_s s = fiobj_obj2cstr(cmd);
-    //   fprintf(stderr, "%s\n", s.data);
+    //   fio_str_info_s s = fio_str_info(cmd);
+    //   fprintf(stderr, "(%d) Cancel Subscription (%p):\n%s\n", getpid(),
+    //           (void *)r->sub_data.uuid, s.data);
     // }
-    fiobj_send_free(r->sub_data.uuid, cmd);
+    fio_str_send_free2(r->sub_data.uuid, cmd);
   }
 }
-static int redis_on_publish(const pubsub_engine_s *eng, FIOBJ channel,
-                            FIOBJ msg) {
-  redis_engine_s *r = en2redis(eng);
-  if (FIOBJ_TYPE(msg) == FIOBJ_T_ARRAY || FIOBJ_TYPE(msg) == FIOBJ_T_HASH)
-    msg = fiobj_obj2json(msg, 0);
-  else
-    msg = fiobj_dup(msg);
 
-  fio_cstr_s msg_str = fiobj_obj2cstr(msg);
-  fio_cstr_s ch_str = fiobj_obj2cstr(channel);
-
-  redis_commands_s *cmd = malloc(sizeof(*cmd) + ch_str.len + msg_str.len + 96);
+static void redis_on_publish_root(const fio_pubsub_engine_s *eng,
+                                  fio_str_info_s channel, fio_str_info_s msg,
+                                  uint8_t is_json) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + channel.len + msg.len + 96);
   *cmd = (redis_commands_s){.cmd_len = 0};
   memcpy(cmd->cmd, "*3\r\n$7\r\nPUBLISH\r\n$", 18);
   char *buf = (char *)cmd->cmd + 18;
-  buf += fio_ltoa((void *)buf, ch_str.len, 10);
+  buf += fio_ltoa((void *)buf, channel.len, 10);
   *buf++ = '\r';
   *buf++ = '\n';
-  memcpy(buf, ch_str.data, ch_str.len);
-  buf += ch_str.len;
+  memcpy(buf, channel.data, channel.len);
+  buf += channel.len;
   *buf++ = '\r';
   *buf++ = '\n';
   *buf++ = '$';
-  msg_str = fiobj_obj2cstr(msg);
-  buf += fio_ltoa(buf, msg_str.len, 10);
+  buf += fio_ltoa(buf, msg.len, 10);
   *buf++ = '\r';
   *buf++ = '\n';
-  memcpy(buf, msg_str.data, msg_str.len);
-  buf += msg_str.len;
+  memcpy(buf, msg.data, msg.len);
+  buf += msg.len;
   *buf++ = '\r';
   *buf++ = '\n';
   *buf = 0;
-  // fprintf(stderr, "%s\n", cmd->cmd);
+  FIO_LOG_DEBUG("(%d) Publishing:\n%s\n", getpid(), cmd->cmd);
   cmd->cmd_len = (uintptr_t)buf - (uintptr_t)(cmd + 1);
   redis_attach_cmd(r, cmd);
-  fiobj_free(msg);
-  return 0;
+  return;
+  (void)is_json;
 }
+
 /* *****************************************************************************
-Object Creation
+Engine / Bridge Stub Callbacks (Child Process)
 ***************************************************************************** */
 
-static void redis_on_startup(const pubsub_engine_s *r_) {
-  redis_engine_s *r = en2redis(r_);
-  /* start adding one connection, so add one reference. */
-  spn_add(&r->ref, 1);
-  if (facil_parent_pid() == getpid()) {
-    defer((void (*)(void *, void *))redis_on_sub_connect_fail, 0,
-          &r->sub_data.protocol);
-  } else {
-    /* workers don't need to subscribe, tha't only on the root process. */
-    defer((void (*)(void *, void *))redis_on_pub_connect_fail, 0,
-          &r->pub_data.protocol);
-  }
+static void redis_on_mock_subscribe_child(const fio_pubsub_engine_s *eng,
+                                          fio_str_info_s channel,
+                                          fio_match_fn match) {
+  /* do nothing, root process is notified about (un)subscriptions by facil.io */
+  (void)eng;
+  (void)channel;
+  (void)match;
+}
+
+static void redis_on_publish_child(const fio_pubsub_engine_s *eng,
+                                   fio_str_info_s channel, fio_str_info_s msg,
+                                   uint8_t is_json) {
+  /* attach engine data to channel (prepend) */
+  fio_str_s tmp = FIO_STR_INIT;
+  /* by using fio_str_s, short names are allocated on the stack */
+  fio_str_info_s tmp_info = fio_str_resize(&tmp, channel.len + 8);
+  fio_u2str64(tmp_info.data, (uint64_t)eng);
+  memcpy(tmp_info.data + 8, channel.data, channel.len);
+  /* forward publication request to Root */
+  fio_publish(.filter = -1, .channel = tmp_info, .message = msg,
+              .engine = FIO_PUBSUB_ROOT, .is_json = is_json);
+  fio_str_free(&tmp);
+  (void)eng;
 }
 
-#undef redis_engine_create
-pubsub_engine_s *redis_engine_create(struct redis_engine_create_args args) {
-  if (!args.address)
-    return NULL;
-  if (!args.port)
-    args.port = "6379";
-  size_t port_len = 0;
-  size_t address_len = 0;
-  if (args.auth && !args.auth_len)
-    args.auth_len = strlen(args.auth);
-  if (args.address)
-    address_len = strlen(args.address);
-  if (args.port)
-    port_len = strlen(args.port);
-  redis_engine_s *r =
-      malloc(sizeof(*r) + REDIS_READ_BUFFER + REDIS_READ_BUFFER + 2 +
-             address_len + port_len + (args.auth_len ? args.auth_len + 32 : 0));
-  *r = (redis_engine_s){
-      .id_protection = 15,
-      .flag = 1,
-      .ping_int = args.ping_interval,
-      .callbacks = FIO_LS_INIT(r->callbacks),
-      .port = (char *)r->buf + (REDIS_READ_BUFFER + REDIS_READ_BUFFER),
-      .address = (char *)r->buf + (REDIS_READ_BUFFER + REDIS_READ_BUFFER) +
-                 port_len + 1,
-      .auth = (char *)r->buf + (REDIS_READ_BUFFER + REDIS_READ_BUFFER) +
-              port_len + address_len + 2,
-      .auth_len = args.auth_len,
-      .en =
-          {
-              .subscribe = redis_on_subscribe,
-              .unsubscribe = redis_on_unsubscribe,
-              .publish = redis_on_publish,
-              .on_startup = redis_on_startup,
-          },
-      .pub_data =
-          {
-              .is_pub = 1,
-              .protocol =
-                  {
-                      .service = "redis engine publishing connection",
-                      .on_data = redis_on_data,
-                      .on_close = redis_pub_on_close,
-                      .ping = redis_pub_ping,
-                      .on_shutdown = redis_on_shutdown,
-                  },
-          },
-      .sub_data =
-          {
-              .protocol =
-                  {
-                      .service = "redis engine subscribing connection",
-                      .on_data = redis_on_data,
-                      .on_close = redis_sub_on_close,
-                      .ping = redis_sub_ping,
-                      .on_shutdown = redis_on_shutdown,
-                  },
-          },
-      .ref = 1, /* starts with only the user handle */
-  };
-  memcpy(r->port, args.port, port_len);
-  r->port[port_len] = 0;
-  memcpy(r->address, args.address, address_len);
-  r->address[address_len] = 0;
-  if (args.auth) {
-    char *pos = r->auth;
-    pos = memcpy(pos, "*2\r\n$4\r\nAUTH\r\n$", 15);
-    pos += fio_ltoa(pos, args.auth_len, 10);
-    *pos++ = '\r';
-    *pos++ = '\n';
-    pos = memcpy(pos, args.auth, args.auth_len);
-    pos[0] = 0;
-    args.auth_len = (uintptr_t)pos - (uintptr_t)r->auth;
-  } else {
-    r->auth = NULL;
-  }
-  pubsub_engine_register(&r->en);
-  if (facil_is_running())
-    redis_on_startup(&r->en);
-  return &r->en;
+/* *****************************************************************************
+Root Publication Handler
+***************************************************************************** */
+
+/* listens to filter -1 and publishes and messages */
+static void redis_on_internal_publish(fio_msg_s *msg) {
+  if (msg->channel.len < 8)
+    return; /* internal error, unexpected data */
+  void *en = (void *)fio_str2u64(msg->channel.data);
+  if (en != msg->udata1)
+    return; /* should be delivered by a different engine */
+  /* step after the engine data */
+  msg->channel.len -= 8;
+  msg->channel.data += 8;
+  /* forward to publishing */
+  FIO_LOG_DEBUG("Forwarding to engine %p, on channel %s\n", msg->udata1,
+                msg->channel.data);
+  redis_on_publish_root(msg->udata1, msg->channel, msg->msg, msg->is_json);
 }
 
-void redis_engine_destroy(pubsub_engine_s *e) {
-  if (e == PUBSUB_CLUSTER_ENGINE || e == PUBSUB_PROCESS_ENGINE) {
-    fprintf(stderr, "WARNING: (redis free) trying to distroy one of the "
-                    "core engines\n");
+/* *****************************************************************************
+Sending commands using the Root connection
+***************************************************************************** */
+
+/* callback from the Redis reply */
+static void redis_forward_reply(fio_pubsub_engine_s *e, FIOBJ reply,
+                                void *udata) {
+  uint8_t *data = udata;
+  fio_pubsub_engine_s *engine = (fio_pubsub_engine_s *)fio_str2u64(data + 0);
+  void *callback = (void *)fio_str2u64(data + 8);
+  if (engine != e || !callback) {
+    FIO_LOG_DEBUG("Redis reply not forwarded (callback: %p)\n", callback);
     return;
   }
-  redis_engine_s *r = en2redis(e);
-  if (r->id_protection != 15) {
-    fprintf(
-        stderr,
-        "FATAL ERROR: (redis) engine pointer incorrect, protection failure.\n");
-    exit(-1);
+  int32_t pid = (int32_t)fio_str2u32(data + 24);
+  FIOBJ rp = fiobj_obj2json(reply, 0);
+  fio_publish(.filter = (-10 - pid), .channel.data = (char *)data,
+              .channel.len = 28, .message = fiobj_obj2cstr(rp), .is_json = 1);
+  fiobj_free(rp);
+}
+
+/* listens to channel -2 for commands that need to be sent (only ROOT) */
+static void redis_on_internal_cmd(fio_msg_s *msg) {
+  // void*(void *)fio_str2u64(msg->msg.data);
+  fio_pubsub_engine_s *engine =
+      (fio_pubsub_engine_s *)fio_str2u64(msg->channel.data + 0);
+  if (engine != msg->udata1) {
+    return;
   }
-  pubsub_engine_deregister(&r->en);
-  r->flag = 0;
-  if (r->pub_data.uuid)
-    sock_close(r->pub_data.uuid);
-  if (r->sub_data.uuid)
-    sock_close(r->sub_data.uuid);
-  redis_free(r);
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + msg->msg.len + 1 + 28);
+  FIO_ASSERT_ALLOC(cmd);
+  *cmd = (redis_commands_s){.callback = redis_forward_reply,
+                            .udata = (cmd->cmd + msg->msg.len + 1),
+                            .cmd_len = msg->msg.len};
+  memcpy(cmd->cmd, msg->msg.data, msg->msg.len);
+  memcpy(cmd->cmd + msg->msg.len + 1, msg->channel.data, 28);
+  redis_attach_cmd((redis_engine_s *)engine, cmd);
+  // fprintf(stderr, " *** Attached CMD (%d) ***\n%s\n", getpid(), cmd->cmd);
+}
+
+/* Listens on filter `-10 -getpid()` for incoming reply data */
+static void redis_on_internal_reply(fio_msg_s *msg) {
+  fio_pubsub_engine_s *engine =
+      (fio_pubsub_engine_s *)fio_str2u64(msg->channel.data + 0);
+  if (engine != msg->udata1) {
+    FIO_LOG_DEBUG("Redis reply not forwarded (engine mismatch: %p != %p)\n",
+                  (void *)engine, msg->udata1);
+    return;
+  }
+  FIOBJ reply;
+  fiobj_json2obj(&reply, msg->msg.data, msg->msg.len);
+  void (*callback)(fio_pubsub_engine_s *, FIOBJ, void *) = (void (*)(
+      fio_pubsub_engine_s *, FIOBJ, void *))fio_str2u64(msg->channel.data + 8);
+  void *udata = (void *)fio_str2u64(msg->channel.data + 16);
+  callback(engine, reply, udata);
+  fiobj_free(reply);
 }
 
-/**
- * Sends a Redis command through the engine's connection.
- *
- * The response will be sent back using the optional callback. `udata` is passed
- * along untouched.
- *
- * The message will be resent on network failures, until a response validates
- * the fact that the command was sent (or the engine is destroyed).
- *
- * Note: NEVER call Pub/Sub commands using this function, as it will violate the
- * Redis connection's protocol (best case scenario, a disconnection will occur
- * before and messages are lost).
- */
-intptr_t redis_engine_send(pubsub_engine_s *engine, FIOBJ command, FIOBJ data,
-                           void (*callback)(pubsub_engine_s *e, FIOBJ reply,
+/* publishes a Redis command to Root's filter -2 */
+intptr_t redis_engine_send(fio_pubsub_engine_s *engine, FIOBJ command,
+                           void (*callback)(fio_pubsub_engine_s *e, FIOBJ reply,
                                             void *udata),
                            void *udata) {
-  if (engine == PUBSUB_CLUSTER_ENGINE || engine == PUBSUB_PROCESS_ENGINE) {
+  if ((uintptr_t)engine < 4) {
     fprintf(stderr, "WARNING: (redis send) trying to use one of the "
                     "core engines\n");
     return -1;
   }
-  redis_engine_s *r = en2redis(engine);
-  if (r->id_protection != 15) {
-    fprintf(stderr,
-            "ERROR: (redis) engine pointer incorrect, protection failure.\n");
-    return -1;
-  }
-  FIOBJ tmp = fiobj2resp_tmp(command, data);
-  fio_cstr_s cmd_str = fiobj_obj2cstr(tmp);
-  redis_commands_s *cmd = malloc(sizeof(*cmd) + cmd_str.len + 1);
-  *cmd = (redis_commands_s){
-      .callback = callback, .udata = udata, .cmd_len = cmd_str.len};
-  memcpy(cmd->cmd, cmd_str.data, cmd_str.len + 1);
-  redis_attach_cmd(r, cmd);
+  // if(fio_is_master()) {
+  // FIOBJ resp = fiobj2resp_tmp(fio_str_info_s obj1, FIOBJ obj2);
+  // TODO...
+  // } else {
+  /* forward publication request to Root */
+  fio_str_s tmp = FIO_STR_INIT;
+  fio_str_info_s ti = fio_str_resize(&tmp, 28);
+  /* combine metadata */
+  fio_u2str64(ti.data + 0, (uint64_t)engine);
+  fio_u2str64(ti.data + 8, (uint64_t)callback);
+  fio_u2str64(ti.data + 16, (uint64_t)udata);
+  fio_u2str32(ti.data + 24, (uint32_t)getpid());
+  FIOBJ cmd = fiobj2resp_tmp(command);
+  fio_publish(.filter = -2, .channel = ti, .message = fiobj_obj2cstr(cmd),
+              .engine = FIO_PUBSUB_ROOT, .is_json = 0);
+  fio_str_free(&tmp);
+  // }
   return 0;
 }
 
 /* *****************************************************************************
-Simple RESP formatting
+Redis Engine Creation
 ***************************************************************************** */
 
-/**
- * Converts FIOBJ objects into a RESP string (client mode).
- *
- * Don't call `fiobj_free`, object will self-destruct.
- */
-static FIOBJ fiobj2resp_tmp(FIOBJ obj1, FIOBJ obj2) {
-  FIOBJ dest = fiobj_str_tmp();
-  if (!obj2 || FIOBJ_IS_NULL(obj2)) {
-    fio_cstr_s s = fiobj_obj2cstr(obj1);
-    fiobj_str_write(dest, "*1\r\n$", 5);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n", 2);
-  } else if (FIOBJ_TYPE(obj2) == FIOBJ_T_ARRAY) {
-    size_t count = fiobj_ary_count(obj2);
-    fiobj_str_write(dest, "*", 1);
-    fiobj_str_join(dest, fiobj_num_tmp(count + 1));
-    fiobj_str_write(dest, "\r\n$", 3);
-
-    fio_cstr_s s = fiobj_obj2cstr(obj1);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n", 2);
-
-    FIOBJ *ary = fiobj_ary2ptr(obj2);
-
-    for (size_t i = 0; i < count; ++i) {
-      s = fiobj_obj2cstr(ary[i]);
-      fiobj_str_write(dest, "$", 1);
-      fiobj_str_join(dest, fiobj_num_tmp(s.len));
-      fiobj_str_write(dest, "\r\n", 2);
-      fiobj_str_write(dest, s.data, s.len);
-      fiobj_str_write(dest, "\r\n", 2);
-    }
-
-  } else if (FIOBJ_TYPE(obj2) == FIOBJ_T_HASH) {
-    FIOBJ json = fiobj_obj2json(obj2, 0);
-    fio_cstr_s s = fiobj_obj2cstr(obj1);
-    fiobj_str_write(dest, "*2\r\n$", 5);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n$", 3);
-    s = fiobj_obj2cstr(json);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_free(json);
-
-  } else {
-    fio_cstr_s s = fiobj_obj2cstr(obj1);
-    fiobj_str_write(dest, "*2\r\n$", 5);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n$", 3);
-    s = fiobj_obj2cstr(obj2);
-    fiobj_str_join(dest, fiobj_num_tmp(s.len));
-    fiobj_str_write(dest, "\r\n", 2);
-    fiobj_str_write(dest, s.data, s.len);
-    fiobj_str_write(dest, "\r\n", 2);
+static void redis_on_facil_start(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 1;
+  if (!fio_is_valid(r->sub_data.uuid)) {
+    fio_defer(redis_connect, r, &r->sub_data);
   }
-  return dest;
 }
-
-/* *****************************************************************************
-RESP parser callbacks
-***************************************************************************** */
-
-/** a local static callback, called when a parser / protocol error occurs. */
-static int resp_on_parser_error(resp_parser_s *parser) {
-  struct redis_engine_internal_s *i =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  fprintf(stderr,
-          "ERROR: (redis) parser error - attempting to restart connection.\n");
-  sock_close(i->uuid);
-  return -1;
+static void redis_on_facil_shutdown(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 0;
 }
 
-/** a local static callback, called when the RESP message is complete. */
-static int resp_on_message(resp_parser_s *parser) {
-  struct redis_engine_internal_s *i =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  FIOBJ msg = i->ary ? i->ary : i->str;
-  if (i->is_pub) {
-    /* publishing / command parser */
-    redis_cmd_reply(FIO_LS_EMBD_OBJ(redis_engine_s, pub_data, i), msg);
-  } else {
-    /* subscriotion parser */
-    if (FIOBJ_TYPE(msg) != FIOBJ_T_ARRAY) {
-      if (FIOBJ_TYPE(msg) != FIOBJ_T_STRING || fiobj_obj2cstr(msg).len != 4 ||
-          fiobj_obj2cstr(msg).data[0] != 'P') {
-        fprintf(stderr, "WARNING: (redis) unexpected data format in "
-                        "subscription stream:\n");
-        fio_cstr_s tmp = fiobj_obj2cstr(msg);
-        fprintf(stderr, "     %s\n", tmp.data);
-      }
-    } else {
-      // FIOBJ *ary = fiobj_ary2ptr(msg);
-      // for (size_t i = 0; i < fiobj_ary_count(msg); ++i) {
-      //   fio_cstr_s tmp = fiobj_obj2cstr(ary[i]);
-      //   fprintf(stderr, "(%lu) %s\n", (unsigned long)i, tmp.data);
-      // }
-      fio_cstr_s tmp = fiobj_obj2cstr(fiobj_ary_index(msg, 0));
-      redis_engine_s *r = parser2redis(parser);
-      if (tmp.len == 7) { /* "message"  */
-        fiobj_free(r->last_ch);
-        r->last_ch = fiobj_dup(fiobj_ary_index(msg, 1));
-        pubsub_publish(.channel = r->last_ch,
-                       .message = fiobj_ary_index(msg, 2),
-                       .engine = PUBSUB_CLUSTER_ENGINE);
-      } else if (tmp.len == 8) { /* "pmessage" */
-        if (!fiobj_iseq(r->last_ch, fiobj_ary_index(msg, 2)))
-          pubsub_publish(.channel = fiobj_ary_index(msg, 2),
-                         .message = fiobj_ary_index(msg, 3),
-                         .engine = PUBSUB_CLUSTER_ENGINE);
-      }
-    }
+static void redis_on_engine_fork(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 0;
+  r->lock = FIO_LOCK_INIT;
+  fio_force_close(r->sub_data.uuid);
+  r->sub_data.uuid = -1;
+  fio_force_close(r->pub_data.uuid);
+  r->pub_data.uuid = -1;
+  while (fio_ls_embd_any(&r->queue)) {
+    redis_commands_s *cmd =
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, fio_ls_embd_pop(&r->queue));
+    fio_free(cmd);
   }
-  /* cleanup */
-  fiobj_free(msg);
-  i->ary = FIOBJ_INVALID;
-  i->str = FIOBJ_INVALID;
-  return 0;
-}
-
-/** a local helper to add parsed objects to the data store. */
-static inline void resp_add_obj(struct redis_engine_internal_s *dest, FIOBJ o) {
-  if (dest->ary) {
-    if (!dest->ary_count)
-      fprintf(stderr,
-              "ERROR: (redis) array overflow indicates a protocol error.\n");
-    fiobj_ary_push(dest->ary, o);
-    --dest->ary_count;
+  r->en = (fio_pubsub_engine_s){
+      .subscribe = redis_on_mock_subscribe_child,
+      .unsubscribe = redis_on_mock_subscribe_child,
+      .publish = redis_on_publish_child,
+  };
+  fio_unsubscribe(r->publication_forwarder);
+  r->publication_forwarder = NULL;
+  fio_unsubscribe(r->cmd_forwarder);
+  r->cmd_forwarder = NULL;
+  fio_unsubscribe(r->cmd_reply);
+  r->cmd_reply =
+      fio_subscribe(.filter = -10 - (int32_t)getpid(),
+                    .on_message = redis_on_internal_reply, .udata1 = r);
+}
+
+fio_pubsub_engine_s *redis_engine_create
+FIO_IGNORE_MACRO(struct redis_engine_create_args args) {
+  if (getpid() != fio_parent_pid()) {
+    fprintf(stderr, "FATAL ERROR: (redis) Redis engine initialization can only "
+                    "be performed in the Root process.\n");
+  }
+  if (!args.address.len && args.address.data)
+    args.address.len = strlen(args.address.data);
+  if (!args.port.len && args.port.data)
+    args.port.len = strlen(args.port.data);
+  if (!args.auth.len && args.auth.data) {
+    args.auth.len = strlen(args.auth.data);
   }
-  dest->str = o;
-}
 
-/** a local static callback, called when a Number object is parsed. */
-static int resp_on_number(resp_parser_s *parser, int64_t num) {
-  struct redis_engine_internal_s *data =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  resp_add_obj(data, fiobj_num_new(num));
-  return 0;
-}
-/** a local static callback, called when a OK message is received. */
-static int resp_on_okay(resp_parser_s *parser) {
-  struct redis_engine_internal_s *data =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  resp_add_obj(data, fiobj_true());
-  return 0;
-}
-/** a local static callback, called when NULL is received. */
-static int resp_on_null(resp_parser_s *parser) {
-  struct redis_engine_internal_s *data =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  resp_add_obj(data, fiobj_null());
-  return 0;
-}
+  if (!args.address.data || !args.address.len) {
+    args.address = (fio_str_info_s){.len = 9, .data = "localhost"};
+  }
+  if (!args.port.data || !args.port.len) {
+    args.port = (fio_str_info_s){.len = 4, .data = "6379"};
+  }
+  redis_engine_s *r =
+      fio_malloc(sizeof(*r) + args.port.len + 1 + args.address.len + 1 +
+                 args.auth.len + 1 + (REDIS_READ_BUFFER * 2));
+  FIO_ASSERT_ALLOC(r);
+  *r = (redis_engine_s){
+      .en =
+          {
+              .subscribe = redis_on_subscribe_root,
+              .unsubscribe = redis_on_unsubscribe_root,
+              .publish = redis_on_publish_root,
+          },
+      .pub_data =
+          {
+              .protocol =
+                  {
+                      .on_data = redis_on_data,
+                      .on_close = redis_on_close,
+                      .on_shutdown = redis_on_shutdown,
+                      .ping = redis_pub_ping,
+                  },
+              .uuid = -1,
+              .on_message = resp_on_pub_message,
+          },
+      .sub_data =
+          {
+              .protocol =
+                  {
+                      .on_data = redis_on_data,
+                      .on_close = redis_on_close,
+                      .on_shutdown = redis_on_shutdown,
+                      .ping = redis_sub_ping,
+                  },
+              .on_message = resp_on_sub_message,
+              .uuid = -1,
+          },
+      .publication_forwarder =
+          fio_subscribe(.filter = -1, .udata1 = r,
+                        .on_message = redis_on_internal_publish),
+      .cmd_forwarder = fio_subscribe(.filter = -2, .udata1 = r,
+                                     .on_message = redis_on_internal_cmd),
+      .cmd_reply =
+          fio_subscribe(.filter = -10 - (uint32_t)getpid(), .udata1 = r,
+                        .on_message = redis_on_internal_reply),
+      .address = ((char *)(r + 1) + 0),
+      .port = ((char *)(r + 1) + args.address.len + 1),
+      .auth = ((char *)(r + 1) + args.address.len + args.port.len + 2),
+      .auth_len = args.auth.len,
+      .ref = 1,
+      .queue = FIO_LS_INIT(r->queue),
+      .lock = FIO_LOCK_INIT,
+      .ping_int = args.ping_interval,
+      .flag = 1,
+  };
+  memcpy(r->address, args.address.data, args.address.len);
+  memcpy(r->port, args.port.data, args.port.len);
+  if (args.auth.len)
+    memcpy(r->auth, args.auth.data, args.auth.len);
+  fio_pubsub_attach(&r->en);
+  redis_on_facil_start(r);
+  fio_state_callback_add(FIO_CALL_IN_CHILD, redis_on_engine_fork, r);
+  fio_state_callback_add(FIO_CALL_ON_SHUTDOWN, redis_on_facil_shutdown, r);
+  /* if restarting */
+  fio_state_callback_add(FIO_CALL_PRE_START, redis_on_facil_start, r);
 
-/**
- * a local static callback, called when a String should be allocated.
- *
- * `str_len` is the expected number of bytes that will fill the final string
- * object, without any NUL byte marker (the string might be binary).
- *
- * If this function returns any value besides 0, parsing is stopped.
- */
-static int resp_on_start_string(resp_parser_s *parser, size_t str_len) {
-  struct redis_engine_internal_s *data =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  resp_add_obj(data, fiobj_str_buf(str_len));
-  return 0;
-}
-/** a local static callback, called as String objects are streamed. */
-static int resp_on_string_chunk(resp_parser_s *parser, void *data, size_t len) {
-  struct redis_engine_internal_s *i =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  fiobj_str_write(i->str, data, len);
-  return 0;
-}
-/** a local static callback, called when a String object had finished
- * streaming.
- */
-static int resp_on_end_string(resp_parser_s *parser) {
-  return 0;
-  (void)parser;
+  return &r->en;
 }
 
-/** a local static callback, called an error message is received. */
-static int resp_on_err_msg(resp_parser_s *parser, void *data, size_t len) {
-  struct redis_engine_internal_s *i =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  resp_add_obj(i, fiobj_str_new(data, len));
-  return 0;
-}
+/* *****************************************************************************
+Redis Engine Destruction
+***************************************************************************** */
 
-/**
- * a local static callback, called when an Array should be allocated.
- *
- * `array_len` is the expected number of objects that will fill the Array
- * object.
- *
- * There's no `resp_on_end_array` callback since the RESP protocol assumes the
- * message is finished along with the Array (`resp_on_message` is called).
- * However, just in case a non-conforming client/server sends nested Arrays,
- * the callback should test against possible overflow or nested Array endings.
- *
- * If this function returns any value besides 0, parsing is stopped.
- */
-static int resp_on_start_array(resp_parser_s *parser, size_t array_len) {
-  struct redis_engine_internal_s *i =
-      FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, parser);
-  if (i->ary) {
-    /* this is an error ... */
-    fprintf(stderr, "ERROR: (redis) RESP protocol violation "
-                    "(array within array).\n");
-    return -1;
-  }
-  i->ary = fiobj_ary_new2(array_len);
-  i->ary_count = array_len;
-  return 0;
+void redis_engine_destroy(fio_pubsub_engine_s *engine) {
+  redis_engine_s *r = (redis_engine_s *)engine;
+  r->flag = 0;
+  fio_pubsub_detach(&r->en);
+  fio_state_callback_remove(FIO_CALL_IN_CHILD, redis_on_engine_fork, r);
+  fio_state_callback_remove(FIO_CALL_ON_SHUTDOWN, redis_on_facil_shutdown, r);
+  fio_state_callback_remove(FIO_CALL_PRE_START, redis_on_facil_start, r);
+  FIO_LOG_DEBUG("Redis engine destroyed %p\n", (void *)r);
+  redis_free(r);
 }