iodine 0.4.19 → 0.5.0

data/ext/iodine/pubsub.h

@@ -6,19 +6,21 @@ Feel free to copy, use and enjoy according to the license provided.
 */
 #ifndef H_FACIL_PUBSUB_H
 /**
-This pub/sub API is designed to unload pub/sub stress from external messanging
-systems onto the local process.
-
-For example, the NULL pub/sub engine, which is routed to the facil_cluster
-engine, will only publish a single message per process instead of a message per
-client, allowing the cluster communication channel to be less crowded when
-possible.
-
-This should allow pub/sub engines, such as Redis, to spread their workload
-between all of an application's processes, enhancing overall performance.
+* This pub/sub API is designed to unload pub/sub stress from external messanging
+* systems onto the local process.
+
+* For example, the NULL pub/sub engine, which is routed to the facil_cluster
+* engine, will only publish a single message per process instead of a message
+per
+* client, allowing the cluster communication channel to be less crowded when
+* possible.
+*
+* This should allow pub/sub engines, such as Redis, to spread their workload
+* between all of an application's processes, enhancing overall performance.
 */
 #define H_FACIL_PUBSUB_H
 #include "facil.h"
+#include "fiobj.h"
 
 /* support C++ */
 #ifdef __cplusplus
@@ -35,22 +37,31 @@ typedef struct pubsub_sub_s *pubsub_sub_pt;
 /** A pub/sub engine data structure. See details later on. */
 typedef struct pubsub_engine_s pubsub_engine_s;
 
-/** The information a "client" (callback) receives. */
+/** The default pub/sub engine.
+ * This engine performs pub/sub within a group of processes (process cluster).
+ *
+ * The process cluser is initialized by the `facil_run` command with `processes`
+ * set to more than 1.
+ */
+extern pubsub_engine_s const *PUBSUB_CLUSTER_ENGINE;
+
+/** An engine that performs pub/sub only within a single process. */
+extern pubsub_engine_s const *PUBSUB_PROCESS_ENGINE;
+
+/** Allows process wide changes to the default Pub/Sub Engine.
+ * Setting a new default before calling `facil_run` will change the default for
+ * the whole process cluster.
+ */
+extern pubsub_engine_s *PUBSUB_DEFAULT_ENGINE;
+
+/** Publishing and on_message callback arguments. */
 typedef struct pubsub_message_s {
-  /** The pub/sub engine farwarding this message. */
+  /** The pub/sub engine that should be used to farward this message. */
   pubsub_engine_s const *engine;
   /** The pub/sub target channnel. */
-  struct {
-    char *name;
-    uint32_t len;
-  } channel;
+  FIOBJ channel;
   /** The pub/sub message. */
-  struct {
-    char *data;
-    uint32_t len;
-  } msg;
-  /** indicates that pattern matching was used. */
-  unsigned use_pattern : 1;
+  FIOBJ message;
   /** The subscription that prompted the message to be routed to the client. */
   pubsub_sub_pt subscription;
   /** Client opaque data pointer (from the `subscribe`) function call. */
@@ -61,13 +72,8 @@ typedef struct pubsub_message_s {
 
 /** The arguments used for `pubsub_subscribe` or `pubsub_find_sub`. */
 struct pubsub_subscribe_args {
-  /** The pub/sub engine to use. NULL defaults to the local cluster engine. */
-  pubsub_engine_s const *engine;
-  /** The channel to subscribe to. */
-  struct {
-    char *name;
-    uint32_t len;
-  } channel;
+  /** The channel namr used for the subscription. */
+  FIOBJ channel;
   /** The on message callback. the `*msg` pointer is to a temporary object. */
   void (*on_message)(pubsub_message_s *msg);
   /** An optional callback for when a subscription is fully canceled. */
@@ -80,34 +86,12 @@ struct pubsub_subscribe_args {
   unsigned use_pattern : 1;
 };
 
-/** The arguments used for `pubsub_publish`. */
-struct pubsub_publish_args {
-  /** The pub/sub engine to use. NULL defaults to the local cluster engine. */
-  pubsub_engine_s const *engine;
-  /** The channel to publish to. */
-  struct {
-    char *name;
-    uint32_t len;
-  } channel;
-  /** The data being pushed. */
-  struct {
-    char *data;
-    uint32_t len;
-  } msg;
-  /** Use pattern matching for channel publication. */
-  unsigned use_pattern : 1;
-  /**
-   * Push the message to the whole cluster, using the cluster engine.
-   * Always TRUE unless an engine was specified.
-   */
-  unsigned push2cluster : 1;
-};
-
 /**
  * Subscribes to a specific channel.
  *
  * Returns a subscription pointer or NULL (failure).
- */ pubsub_sub_pt pubsub_subscribe(struct pubsub_subscribe_args);
+ */
+pubsub_sub_pt pubsub_subscribe(struct pubsub_subscribe_args);
 #define pubsub_subscribe(...)                                                  \
   pubsub_subscribe((struct pubsub_subscribe_args){__VA_ARGS__})
 
@@ -125,20 +109,28 @@ pubsub_sub_pt pubsub_find_sub(struct pubsub_subscribe_args);
   pubsub_find_sub((struct pubsub_subscribe_args){__VA_ARGS__})
 
 /**
- * Unsubscribes from a specific channel.
+ * Unsubscribes from a specific subscription.
+ *
+ * Note: This should be called exactly the same number times as
+ * `pubsub_subscribe`. Any less and a memory leak might occur. Any more, and the
+ * program might crash.
  *
  * Returns 0 on success and -1 on failure.
  */
-void pubsub_unsubscribe(pubsub_sub_pt subscription);
+int pubsub_unsubscribe(pubsub_sub_pt subscription);
 
 /**
  * Publishes a message to a channel belonging to a pub/sub service (engine).
  *
- * Returns 0 on success and -1 on failure.
+ * Returns 0 on success and -1 on failure (i.e., no channel, no message or no
+ * known subscriptions).
+ *
+ * NOTE: Memory ownership is retained by the calling function. Both the channel
+ * and the message should be freed when the caller is done with them.
  */
-int pubsub_publish(struct pubsub_publish_args);
+int pubsub_publish(struct pubsub_message_s);
 #define pubsub_publish(...)                                                    \
-  pubsub_publish((struct pubsub_publish_args){__VA_ARGS__})
+  pubsub_publish((struct pubsub_message_s){__VA_ARGS__})
 
 /**
  * defers message hadling if it can't be performed (i.e., resource is busy) or
@@ -160,59 +152,49 @@ void pubsub_defer(pubsub_message_s *msg);
 /**
  * Pub/Sub services (engines) MUST provide the listed function pointers.
  *
+ * Engines should also register using the `pubsub_engine_register` function.
+ *
+ * Engines should deregister, before being destroyed, by using the
+ * `pubsub_engine_deregister` function.
+ *
  * When an engine received a message to publish, they should call the
- * `pubsub_eng_distribute` function. i.e.:
- *
- *       pubsub_engine_distribute(
- *           .engine = self,
- *           .channel.name = "channel 1",
- *           .channel.len = 9,
- *           .msg.data = "hello",
- *           .msg.len = 5,
- *           .push2cluster = self->push2cluster,
- *           .use_pattern = 0 );
- *
- * Engines MUST survive until the pub/sub service is finished using them and
- * there are no more subscriptions.
+ * `pubsub_publish` function with the engine to which the message is forwarded.
+ * i.e.:
+ *
+ *       pubsub_publish(
+ *           .engine = PUBSUB_PROCESS_ENGINE,
+ *           .channel = channel_name,
+ *           .message = msg_body );
+ *
+ * Engines MUST NOT free any of the FIOBJ objects they receive.
+ *
  */
 struct pubsub_engine_s {
-  /** Should return 0 on success and -1 on failure. */
-  int (*subscribe)(const pubsub_engine_s *eng, const char *ch, size_t ch_len,
-                   uint8_t use_pattern);
-  /** Return value is ignored. */
-  void (*unsubscribe)(const pubsub_engine_s *eng, const char *ch, size_t ch_len,
+  /** Must subscribe channel. Failures are ignored. */
+  void (*subscribe)(const pubsub_engine_s *eng, FIOBJ channel,
+                    uint8_t use_pattern);
+  /** Must unsubscribe channel. Failures are ignored. */
+  void (*unsubscribe)(const pubsub_engine_s *eng, FIOBJ channel,
                       uint8_t use_pattern);
   /** Should return 0 on success and -1 on failure. */
-  int (*publish)(const pubsub_engine_s *eng, const char *ch, size_t ch_len,
-                 const char *msg, size_t msg_len, uint8_t use_pattern);
-  /** Set to TRUE (1) if published messages should propegate to the cluster. */
-  unsigned push2cluster : 1;
+  int (*publish)(const pubsub_engine_s *eng, FIOBJ channel, FIOBJ msg);
+  /**
+   * facil.io will call this callback whenever starting, or restarting, the
+   * reactor.
+   *
+   * This will be called when facil.io starts (the master process).
+   *
+   * This will also be called when forking, after facil.io closes all
+   * connections and claim to shut down (running all deferred event).
+   */
+  void (*on_startup)(const pubsub_engine_s *eng);
 };
 
-/** The default pub/sub engine.
- * This engine performs pub/sub within a group of processes (process cluster).
- *
- * The process cluser is initialized by the `facil_run` command with `processes`
- * set to more than 1.
- */
-extern const pubsub_engine_s *PUBSUB_CLUSTER_ENGINE;
-
-/** An engine that performs pub/sub only within a single process. */
-extern const pubsub_engine_s *PUBSUB_PROCESS_ENGINE;
-
-/** Allows process wide changes to the default Pub/Sub Engine.
- * Setting a new default before calling `facil_run` will change the default for
- * the whole process cluster.
- */
-extern const pubsub_engine_s *PUBSUB_DEFAULT_ENGINE;
+/** Registers an engine, so it's callback can be called. */
+void pubsub_engine_register(pubsub_engine_s *engine);
 
-/**
- * The function used by engines to distribute received messages.
- * The `udata*` and `subscription` fields are ignored.
- */
-void pubsub_engine_distribute(pubsub_message_s msg);
-#define pubsub_engine_distribute(...)                                          \
-  pubsub_engine_distribute((pubsub_message_s){__VA_ARGS__})
+/** Unregisters an engine, so it could be safely destroyed. */
+void pubsub_engine_deregister(pubsub_engine_s *engine);
 
 /**
  * Engines can ask facil.io to resubscribe to all active channels.

data/ext/iodine/rb-call.c

@@ -23,7 +23,6 @@ typedef struct {
   VALUE obj;
   int argc;
   VALUE *argv;
-  VALUE returned;
   ID method;
   int exception;
 } iodine_rb_task_s;
@@ -70,7 +69,7 @@ static void *handle_exception(void *ignr) {
               (int)RSTRING_LEN(msg), RSTRING_PTR(msg));
     }
     rb_backtrace();
-    fprintf(stderr, "\n");
+    fprintf(stderr, "\n\n");
     rb_set_errinfo(Qnil);
   }
   return (void *)Qnil;
@@ -130,8 +129,9 @@ static VALUE iodin_rb_call_arg(VALUE obj, ID method, int argc, VALUE *argv) {
                            .argc = argc,
                            .argv = argv};
   void *ret;
-  if (in_gvl)
-    return (VALUE)rb_funcall2(obj, method, argc, argv);
+  if (in_gvl) {
+    return (VALUE)iodine_protected_call(&task);
+  }
   in_gvl = 1;
   ret = rb_thread_call_with_gvl(iodine_protected_call, &task);
   in_gvl = 0;

data/ext/iodine/rb-defer.c

@@ -6,14 +6,19 @@ Feel free to copy, use and enjoy according to the license provided.
 */
 // clang-format off
 #include "rb-registry.h"
+#include "rb-call.h"
+#include "iodine.h"
 #include <ruby.h>
 #include <ruby/thread.h>
 
 #include <stdint.h>
 // clang-format on
 
+#include <spnlock.inc>
 #include "defer.h"
 
+#include <pthread.h>
+
 /* *****************************************************************************
 Local helpers
 ***************************************************************************** */
@@ -26,33 +31,46 @@ struct CreateThreadArgs {
 /* used here but declared elsewhere */
 void call_async_signal(void *pool) { defer_pool_stop((pool_pt)pool); }
 
-/* the thread's GVL release */
-static VALUE thread_loop(void *args_) {
+static void *defer_thread_start(void *args_) {
   struct CreateThreadArgs *args = args_;
   void *(*thread_func)(void *) = args->thread_func;
   void *arg = args->arg;
-  free(args_);
-  rb_thread_call_without_gvl2(thread_func, arg,
-                              (void (*)(void *))call_async_signal, arg);
+  free(args);
+  RubyCaller.set_gvl_state(0);
+  thread_func(arg);
+  return NULL;
+}
+
+/* the thread's GVL release */
+static VALUE defer_thread_inGVL(void *args_) {
+  struct CreateThreadArgs *args = args_;
+  rb_thread_call_without_gvl(defer_thread_start, args_,
+                             (void (*)(void *))call_async_signal, args->arg);
   return Qnil;
 }
 
 /* Within the GVL, creates a Ruby thread using an API call */
 static void *create_ruby_thread_gvl(void *args) {
-  return (void *)Registry.add(rb_thread_create(thread_loop, args));
-}
-
-/* protect the call to join from any exceptions */
-static void *inner_join_with_rbthread_(void *rbt) {
-  return (void *)rb_funcall((VALUE)rbt, rb_intern("join"), 0);
+  return (void *)Registry.add(rb_thread_create(defer_thread_inGVL, args));
 }
 
 static void *fork_using_ruby(void *ignr) {
+  RubyCaller.call(Iodine, rb_intern("before_fork"));
   const VALUE ProcessClass = rb_const_get(rb_cObject, rb_intern("Process"));
-  const VALUE pid = rb_funcall(ProcessClass, rb_intern("fork"), 0);
-  if (pid == Qnil)
-    return (void *)0;
-  return (void *)(intptr_t)(NUM2INT(pid));
+  const VALUE rb_pid = RubyCaller.call(ProcessClass, rb_intern("fork"));
+  intptr_t pid = 0;
+  if (rb_pid != Qnil) {
+    pid = NUM2INT(rb_pid);
+  } else {
+    pid = 0;
+  }
+  RubyCaller.set_gvl_state(1); /* enforce GVL state in thread storage */
+  if (!pid) {
+    Registry.on_fork();
+    RubyCaller.call(Iodine, rb_intern("after_fork"));
+  }
+  return (void *)pid;
+  (void)ignr;
 }
 
 /* *****************************************************************************
@@ -62,14 +80,17 @@ The Defer library overriding functions
 /**
 OVERRIDE THIS to replace the default pthread implementation.
 */
-void *defer_new_thread(void *(*thread_func)(void *), pool_pt pool) {
+void *defer_new_thread(void *(*thread_func)(void *), void *arg) {
   struct CreateThreadArgs *data = malloc(sizeof(*data));
   if (!data)
     return NULL;
-  *data = (struct CreateThreadArgs){.thread_func = thread_func, .arg = pool};
-  void *thr = rb_thread_call_with_gvl(create_ruby_thread_gvl, data);
-  if (!thr || thr == (void *)Qnil || thr == (void *)Qfalse)
+  *data = (struct CreateThreadArgs){
+      .thread_func = thread_func, .arg = arg,
+  };
+  void *thr = RubyCaller.call_c(create_ruby_thread_gvl, data);
+  if (!thr || thr == (void *)Qnil || thr == (void *)Qfalse) {
     thr = NULL;
+  }
   return thr;
 }
 
@@ -79,18 +100,21 @@ OVERRIDE THIS to replace the default pthread implementation.
 int defer_join_thread(void *thr) {
   if (!thr || (VALUE)thr == Qfalse || (VALUE)thr == Qnil)
     return -1;
-  rb_thread_call_with_gvl(inner_join_with_rbthread_, (void *)thr);
+  RubyCaller.call((VALUE)thr, rb_intern("join"));
   Registry.remove((VALUE)thr);
   return 0;
 }
 
+// void defer_free_thread(void *thr) { (void)thr; }
+void defer_free_thread(void *thr) { Registry.remove((VALUE)thr); }
+
 /**
 OVERRIDE THIS to replace the default `fork` implementation or to inject hooks
 into the forking function.
 
 Behaves like the system's `fork`.
 */
-int defer_new_child(void) {
-  intptr_t pid = (intptr_t)rb_thread_call_with_gvl(fork_using_ruby, NULL);
+int facil_fork(void) {
+  intptr_t pid = (intptr_t)RubyCaller.call_c(fork_using_ruby, NULL);
   return (int)pid;
 }

data/ext/iodine/rb-fiobj2rb.h

@@ -0,0 +1,117 @@
+#ifndef H_RB_FIOBJ2RUBY_H
+/*
+Copyright: Boaz segev, 2016-2017
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#define H_RB_FIOBJ2RUBY_H
+#include <fiobj.h>
+#include <ruby.h>
+
+#include "rb-registry.h"
+
+typedef struct {
+  FIOBJ stack;
+  uintptr_t count;
+  VALUE rb;
+  uint8_t str2sym;
+} fiobj2rb_s;
+
+typedef struct { uint8_t str2sym; } fiobj2rb_settings_s;
+
+static inline VALUE fiobj2rb(FIOBJ o, uint8_t str2sym) {
+  VALUE rb;
+  if (!o)
+    return Qnil;
+  switch (FIOBJ_TYPE(o)) {
+  case FIOBJ_T_NUMBER:
+    rb = LONG2FIX(fiobj_obj2num(o));
+    break;
+  case FIOBJ_T_NULL:
+    rb = Qnil;
+    break;
+  case FIOBJ_T_TRUE:
+    rb = Qtrue;
+    break;
+  case FIOBJ_T_FALSE:
+    rb = Qfalse;
+    break;
+  case FIOBJ_T_FLOAT:
+    rb = DBL2NUM(fiobj_obj2float(o));
+    break;
+  case FIOBJ_T_DATA:    /* fallthrough */
+  case FIOBJ_T_UNKNOWN: /* fallthrough */
+  case FIOBJ_T_STRING: {
+    fio_cstr_s tmp = fiobj_obj2cstr(o);
+    if (str2sym) {
+      rb = rb_intern2(tmp.data, tmp.len);
+      rb = ID2SYM(rb);
+    } else {
+      rb = rb_str_new(tmp.data, tmp.len);
+    }
+
+  } break;
+  case FIOBJ_T_ARRAY:
+    rb = rb_ary_new();
+    break;
+  case FIOBJ_T_HASH:
+    rb = rb_hash_new();
+    break;
+  };
+  return rb;
+}
+
+static int fiobj2rb_task(FIOBJ o, void *data_) {
+  fiobj2rb_s *data = data_;
+  VALUE rb_tmp;
+  rb_tmp = fiobj2rb(o, 0);
+  Registry.add(rb_tmp);
+  if (data->rb) {
+    if (RB_TYPE_P(data->rb, T_HASH)) {
+      rb_hash_aset(data->rb, fiobj2rb(fiobj_hash_key_in_loop(), data->str2sym),
+                   rb_tmp);
+    } else {
+      rb_ary_push(data->rb, rb_tmp);
+    }
+    --(data->count);
+    Registry.remove(rb_tmp);
+  } else {
+    data->rb = rb_tmp;
+    // Registry.add(rb_tmp);
+  }
+  if (FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY)) {
+    fiobj_ary_push(data->stack, (FIOBJ)data->count);
+    fiobj_ary_push(data->stack, (FIOBJ)data->rb);
+    data->count = fiobj_ary_count(o);
+    data->rb = rb_tmp;
+  } else if (FIOBJ_TYPE_IS(o, FIOBJ_T_HASH)) {
+    fiobj_ary_push(data->stack, (FIOBJ)data->count);
+    fiobj_ary_push(data->stack, (FIOBJ)data->rb);
+    data->count = fiobj_hash_count(o);
+    data->rb = rb_tmp;
+  }
+  while (data->count == 0 && fiobj_ary_count(data->stack)) {
+    data->rb = fiobj_ary_pop(data->stack);
+    data->count = fiobj_ary_pop(data->stack);
+  }
+  return 0;
+}
+
+static inline VALUE fiobj2rb_deep(FIOBJ obj, uint8_t str2sym) {
+  fiobj2rb_s data = {.stack = fiobj_ary_new2(4), .str2sym = str2sym};
+
+  /* deep copy */
+  fiobj_each2(obj, fiobj2rb_task, &data);
+  /* cleanup (shouldn't happen, but what the hell)... */
+  while (fiobj_ary_pop(data.stack))
+    ;
+  fiobj_free(data.stack);
+  Registry.remove(data.rb);
+  return data.rb;
+}
+
+// require 'iodine'
+// Iodine::JSON.parse "{\"1\":[1,2,3,4]}"
+// Iodine::JSON.parse IO.binread("")
+#endif /* H_RB_FIOBJ2RUBY_H */