iodine 0.7.14 → 0.7.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d3f8b237b265c8151aa3fe966a9c863fa3c053e4fd97289cfe498b77818e564
-  data.tar.gz: 389b80f9feff3abf97ba7fdaf08d0a2986521093fb0365ab10c5e4b3af3b2de5
+  metadata.gz: f1996d0474dc00ad0614035f36b703154b594415278cd18624cec94c8149150e
+  data.tar.gz: 3060659e07df3fc734594584b407b5bf9156ae77fcdb8709f62a49d0f2f62296
 SHA512:
-  metadata.gz: d8b220410c1ebdecf221bde09d48f64ed0c69dc50a0f0ecba69a3e530dc2b9cace643c9a1d8bc7ce73ecf9f1274ffc1162a21db6db273673de84318f9ea81cb3
-  data.tar.gz: 94f71023813326cb0edfabecbbd862e566ec37882de35ca6b6f14a2759088b54ed52df9bb943888b3c58f3441881c67a4c82197550272593050eaca6fc66b3e8
+  metadata.gz: 2ab1733a97c6617232022405bd25d4f0c46d4a99c6751b5a8467e48fa17efc0dc815dc47ca329c7ca955d135c56e8a06c69d3fd168ead0b3510712e909711468
+  data.tar.gz: 53be4c7971bca24fd484bfa4dd18d67a99ffba196a5908c76bf1a3001a957ef607c51ec8d90b3d4714d6297609bf4dde8077d24bd85eee081b875bf27faa5d8f

data/CHANGELOG.md

@@ -6,6 +6,18 @@ Please notice that this change log contains changes for upcoming releases as wel
 
 ## Changes:
 
+#### Change log v.0.7.15
+
+**Fix**: (`fio`) fixed a minor memory leak in cluster mode, caused by the root process not freeing the hash map used for child process subscription monitoring (only effected hot restarts).
+
+**Fix**: (`fio`) fixed superfluous and potentially erroneous pub/sub engine callback calls to `unsubscribe`, caused by (mistakingly) reporting filter channel closure.
+
+**Fix**: (`http/1.1`) avoid processing further requests if the connection was closed.
+
+**Fix**: (`iodine`) fixed some errors in the documentation and added a missing deprecation notice.
+
+**Update**: (`fio`) updated the automatic concurrency calculations to leave resources for the system when a negative value is provided (was only available for worker count calculations, now available for thread count as well).
+
 #### Change log v.0.7.14
 
 **Fix**: (`facil.io`) fixed superfluous ping event.

data/README.md

@@ -507,25 +507,26 @@ require 'iodine'
 
 # an echo protocol with asynchronous notifications.
 class EchoProtocol
-  # `on_message` is an optional alternative to the `on_data` callback.
-  # `on_message` has a 1Kb buffer that recycles itself for memory optimization.
+  # `on_message` is called when data is available.
   def on_message client, buffer
     # writing will never block and will use a buffer written in C when needed.
     client.write buffer
     # close will be performed only once all the data in the write buffer
     # was sent. use `force_close` to close early.
     client.close if buffer =~ /^bye[\r\n]/i
-    # run asynchronous tasks using the thread pool
-    Iodine.run do
-      sleep 1
-      puts "Echoed data: #{buffer}"
+    # run asynchronous tasks... after a set number of milliseconds
+    Iodine.run_after(1000) do
+      # or schedule the task immediately
+      Iodine.run do
+        puts "Echoed data: #{buffer}"
+      end
     end
   end
 end
 
 # listen on port 3000 for the echo protocol.
 Iodine.listen(port: "3000") { EchoProtocol.new }
-Iodine.threads = 4
+Iodine.threads = 1
 Iodine.workers = 1
 Iodine.start
 ```
@@ -565,7 +566,7 @@ class ChatProtocol
   end
 end
 
-#an initial login protocol
+# an initial login protocol
 class LoginProtocol
   def on_open client
     client.write "Enter nickname to log in to chat room:\n"

data/SPEC-PubSub-Draft.md

@@ -20,7 +20,7 @@ Conforming Pub/Sub implementations **MUST** implement the following pub/sub rela
 
     * `:match` indicates a matching algorithm should be applied to the `to` variable (`to` is a pattern).
     
-        Possible values should include [`:redis`](https://github.com/antirez/redis/blob/398b2084af067ae4d669e0ce5a63d3bc89c639d3/src/util.c#L46-L167), [`:nats`](https://nats.io/documentation/faq/#wildcards) or [`:rabbitmq`](https://www.rabbitmq.com/tutorials/tutorial-five-ruby.html). Pub/Sub implementations *MAY* support none, some or all of these common pattern resolution schemes.
+        Possible (suggested) values should include [`:redis`](https://github.com/antirez/redis/blob/398b2084af067ae4d669e0ce5a63d3bc89c639d3/src/util.c#L46-L167), [`:nats`](https://nats.io/documentation/faq/#wildcards) or [`:rabbitmq`](https://www.rabbitmq.com/tutorials/tutorial-five-ruby.html). Pub/Sub implementations *MAY* support none, some or all of these common pattern resolution schemes.
     
     * `:handler` is an alternative to the optional block. It should accept Proc like objects (objects that answer to `.call(from, msg)`).
 
@@ -31,7 +31,9 @@ Conforming Pub/Sub implementations **MUST** implement the following pub/sub rela
         This option is only valid if the optional `block` is missing and the connection is a WebSocket connection. Note that SSE connections are limited to text data by design.
 
         This will dictate the encoding for outgoing WebSocket message when publications are directly sent to the client (as a text message or a binary blob). `:text` will be the default value for a missing `:as` option.
-    
+
+        Servers *MAY* ignore this value if they set the message type (text/binary) based on UTF-8 validation.
+
     If a subscription to `to` already exists, it should be *replaced* by the new subscription (the old subscription should be canceled / unsubscribed).
 
     When the `subscribe` method is called within a WebSocket / SSE Callback object, the subscription must be closed automatically when the connection is closed.
@@ -50,64 +52,66 @@ Conforming Pub/Sub implementations **MUST** implement the following pub/sub rela
 
     * `message` a String with containing the data to be published.
 
-    * `engine` (optional) routed the publish method to the specified Pub/Sub Engine (see later on). If none is specified, the default engine should be used.
+    * `engine` routes the publish method to the specified Pub/Sub Engine (see later on). If none is specified, the default engine should be used. If `false` is specified, the message should be forwarded to all subscribed clients.
 
     The `publish` method must return `true` if a publication was scheduled (not necessarily performed). If it's already known that the publication would fail, the method should return `false`.
 
     An implementation **MUST** call the relevant PubSubEngine's `publish` method after performing any internal book keeping logic. If `engine` is `nil`, the default PubSubEngine should be called. If `engine` is `false`, the implementation **MUST** forward the published message to the actual clients (if any).
 
-    A global alias for this method (allowing it to be accessed from outside active connections) should be defined as `Rack::PubSub.publish`.
+    A global alias for this method (allowing it to be accessed from outside active connections) **MAY** be defined as `Rack::PubSub.publish`.
 
-Implementations **MUST** implement the following methods:
+Implementations **MUST** implement the following methods in one of their public classes / modules (iodine implements these under `Iodine::PubSub`):
 
-* `Rack::PubSub.register(engine)` where `engine` is a PubSubEngine object as described in this specification.
+* `attach(engine)` where `engine` is a `PubSubEngine` object, as described in this specification.
 
-    When a pub/sub engine is registered, the implementation **MUST** inform the engine of any existing or future subscriptions.
+    When a pub/sub engine is attached, the implementation **MUST** inform the engine of any existing or future subscriptions.
 
     The implementation **MUST** call the engine's `subscribe` callback for each existing (and future) subscription.
 
-* `Rack::PubSub.deregister(engine)` where `engine` is a PubSubEngine object as described in this specification.
+    The implementation **MUST** allow multiple "engines" to be attached when multiple calls to `attach` are made.
+
+* `detach(engine)` where `engine` is a PubSubEngine object as described in this specification.
 
-    Removes an engine from the pub/sub registration. The opposit of 
+    Removes an engine from the pub/sub system. The opposite of `attach`.
 
-* `Rack::PubSub.default_engine = engine` sets a default pub/sub engine, where `engine` is a PubSubEngine object as described in this specification.
+* `default = engine` sets a default pub/sub engine, where `engine` is a PubSubEngine object as described in this specification.
 
-    Implementations **MUST** forward any `publish` method calls to the default pub/sub engine.
+    Implementations **MUST** forward any `publish` method calls to the default pub/sub engine, unless an `engine` is specified in arguments passes to the `publish` method.
 
-* `Rack::PubSub.default_engine` returns the current default pub/sub engine, where the engine is a PubSubEngine object as described in this specification.
+* `default` returns the current default pub/sub engine, where the engine is a PubSubEngine object as described in this specification.
 
-* `Rack::PubSub.reset(engine)` where `engine` is a PubSubEngine object as described in this specification.
+* `reset(engine)` where `engine` is a PubSubEngine object as described in this specification.
 
     Implementations **MUST** behave as if the engine was newly registered and (re)inform the engine of any existing subscriptions by calling engine's `subscribe` callback for each existing subscription.
 
-Implementations **MAY** implement pub/sub internally (in which case the `pubsub_default` engine is the server itself or a server's module).
+Implementations **MAY** implement pub/sub internally (in which case the `default` engine is the server itself or a server's module).
 
 However, servers **MUST** support external pub/sub "engines" as described above, using PubSubEngine objects.
 
-PubSubEngine objects **MUST** implement the following methods:
+`PubSubEngine` objects **MUST** implement the following methods:
 
-* `subscribe(channel, as=nil)` this method performs the subscription to the specified channel.
+* `subscribe(channel, match=nil)` this method performs the subscription to the specified channel.
 
-    If `as` is a Symbol that the engine recognizes (i.e., `:redis`, `:nats`, etc'), the engine should behave accordingly. i.e., the value `:redis` on a Redis engine will invoke the PSUBSCRIBE Redis command.
+    If `match` is a Symbol that the engine recognizes (i.e., `:redis`, `:nats`, etc'), the engine should behave accordingly. i.e., the value `:redis` on a Redis engine will invoke the PSUBSCRIBE Redis command.
 
     The method must return `true` if a subscription was scheduled (or performed) or `false` if the subscription is known to fail.
 
     This method will be called by the server (for each registered engine). The engine may assume that the method would never be called directly by an application.
 
-* `unsubscribe(channel, as=nil)` this method performs closes the subscription to the specified channel.
+* `unsubscribe(channel, match=nil)` this method performs closes the subscription to the specified channel.
 
     The method's semantics are similar to `subscribe`.
 
     This method will be called by the server (for each registered engine). The engine may assume that the method would never be called directly by an application.
 
-* `publish(channel, message)` where both `channel` and `message` are String object.
+* `publish(channel, message)` where both `channel` and `message` are String objects.
 
     This method will be called by the server when a message is published using the engine.
 
-    The engine **MUST** assume that the method might called directly by an application.
+    The engine **MUST** assume that the method might get called directly by an application.
 
-When a PubSubEngine object receives a published message, it should call:
+When a PubSubEngine object receives a published message, it *should* call:
 
 ```ruby
-Rack::PubSub.publish channel: channel, message: message, engine: false
+Foo::PubSub.publish channel, message, false
 ```

data/SPEC-Websocket-Draft.md

@@ -13,9 +13,9 @@ The purpose of these specifications is:
 
     Simply put, when choosing between conforming servers, the application doesn’t need to have any knowledge about the chosen server.
 
-2. To Support “native" (server-side) WebSocket and EventSource (SSE) connections and using application side callbacks.
+2. To support “native" (server-side) WebSocket and EventSource (SSE) connections and using application side callbacks.
 
-    Simply put, to make it easy for applications to accept WebSocket and EventSource (SSE) connections from WebSocket and EventSource clients (commonly browsers).
+    Simply put, to make it easy for applications to accept WebSocket and EventSource (SSE) connections from WebSocket and EventSource clients (commonly browsers) while abstracting away any transport layer details.
 
 3. Allow applications to use WebSocket and EventSource (SSE) on HTTP/2 servers. Note: current `hijack` practices will break network connections when attempting to implement EventSource (SSE).
 
@@ -45,11 +45,11 @@ The Callback Object could be a any object which implements any of the following
 
     Servers **MAY**, optionally, implement a **recyclable buffer** for the `on_message` callback. However, this is optional and it is *not* required.
 
-* `on_drained(client)` **MAY** be called when the the `write` buffer becomes empty. **If** `pending` returns a non-zero value, the `on_drained` callback **MUST** be called once the write buffer becomes empty.
+* `on_drained(client)` **MAY** be called when the the `client.write` buffer becomes empty. **If** `client.pending` returns a non-zero value, the `on_drained` callback **MUST** be called once the write buffer becomes empty.
 
 * `on_shutdown(client)` **MAY** be called during the server's graceful shutdown process, _before_ the connection is closed and in addition to the `on_close` function (which is called _after_ the connection is closed.
 
-* `on_close(client)` **MUST** be called _after_ the connection was closed for whatever reason (socket errors, parsing errors, timeouts, client disconnection, `close` being called, etc').
+* `on_close(client)` **MUST** be called _after_ the connection was closed for whatever reason (socket errors, parsing errors, timeouts, client disconnection, `client.close` being called, etc').
 
 
 The server **MUST** provide the Callback Object with a `client` object, that supports the following methods (this approach promises applications could be server agnostic):

data/examples/sub-protocols.ru

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# This is a WebSocket / SSE notification example application.
+#
+# In this example, WebSocket sub-protocols are explored.
+#
+# Running this application from the command line is easy with:
+#
+#      iodine
+#
+# Or, in a single thread and a single process:
+#
+#      iodine -t 1 -w 1
+#
+# Test using:
+#
+#      var subprotocol = "echo"; // or "chat"
+#      ws = new WebSocket("ws://localhost:3000/Mitchel", subprotocol);
+#      ws.onmessage = function(e) { console.log(e.data); };
+#      ws.onclose = function(e) { console.log("Closed"); };
+#      ws.onopen = function(e) { e.target.send("Yo!"); };
+
+
+# Chat clients connect with the "chat" sub-protocol.
+class ChatClient
+  def on_open client
+    @nickname = client.env['PATH_INFO'].to_s.split('/')[1] || "Guest"
+    client.subscribe :chat
+    client.publish :chat , "#{@nickname} joined the chat."
+  end
+  def on_close client
+    client.publish :chat , "#{@nickname} left the chat."
+  end
+  def on_shutdown client
+    client.write "Server is shutting down... disconnecting all clients. Goodbye."
+  end
+  def on_message client, message
+    client.publish :chat , "#{@nickname}: #{message}"
+  end
+end
+
+# Echo clients connect with the "echo" sub-protocol.
+class EchoClient
+  def on_open client
+    client.write "You established an echo connection."
+  end
+  def on_shutdown client
+    client.write "Server is shutting down... goodbye."
+  end
+  def on_message client, message
+    client.write message
+  end
+end
+
+# Rack application module
+module APP
+  # the allowed protocols
+  CHAT_PROTOCOL_NAME = "chat"
+  ECHO_PROTOCOL_NAME = "echo"
+  PROTOCOLS =[CHAT_PROTOCOL_NAME, ECHO_PROTOCOL_NAME]
+
+  # the Rack application
+  def call env
+    return [200, {}, ["Hello World"]] unless env["rack.upgrade?"]
+    protocol = select_protocol(env)
+    case(protocol)
+    when CHAT_PROTOCOL_NAME
+      env["rack.upgrade"] = ChatClient.new
+      [101, { "Sec-Websocket-Protocol" => protocol }, []]
+    when ECHO_PROTOCOL_NAME
+      env["rack.upgrade"] = EchoClient.new
+      [101, { "Sec-Websocket-Protocol" => protocol }, []]
+    else
+      [400, {}, ["Unsupported protocol specified"]]
+    end
+  end
+
+  def select_protocol(env)
+    request_protocols = env["HTTP_SEC_WEBSOCKET_PROTOCOL"]
+    unless request_protocols.nil?
+      request_protocols = request_protocols.split(/,\s?/) if request_protocols.is_a?(String)
+      request_protocols.detect { |request_protocol| PROTOCOLS.include? request_protocol }
+    end # either `nil` or the result of `request_protocols.detect` are returned
+  end
+
+  # make functions availble as singleton module
+  extend self
+end
+
+run APP

data/exe/iodine

@@ -33,7 +33,9 @@ module Iodine
             puts "         Running only static file service."
             opt = ::Rack::Server::Options.new.parse!([])
           else
-            puts cli_parser
+            puts "\nERROR: Couldn't run Ruby application, check command line arguments."
+            ARGV << "-?"
+            Iodine::Base::CLI.parse
             exit(0);
           end
         end

data/ext/iodine/fio.c

@@ -1456,26 +1456,35 @@ void fio_expected_concurrency(int16_t *threads, int16_t *processes) {
     /* Set any option that is less than 0 be equal to cores/value */
     /* Set any option equal to 0 be equal to the other option in value */
     ssize_t cpu_count = fio_detect_cpu_cores();
-    size_t cpu_adjust = (*processes <= 0 ? 1 : 0);
+    size_t thread_cpu_adjust = (*threads <= 0 ? 1 : 0);
+    size_t worker_cpu_adjust = (*processes <= 0 ? 1 : 0);
 
     if (cpu_count > 0) {
-      int16_t tmp_threads = 0;
+      int16_t tmp = 0;
       if (*threads < 0)
-        tmp_threads = (int16_t)(cpu_count / (*threads * -1));
-      else if (*threads == 0)
-        tmp_threads = -1 * *processes;
-      else
-        tmp_threads = *threads;
+        tmp = (int16_t)(cpu_count / (*threads * -1));
+      else if (*threads == 0) {
+        tmp = -1 * *processes;
+        thread_cpu_adjust = 0;
+      } else
+        tmp = *threads;
       if (*processes < 0)
         *processes = (int16_t)(cpu_count / (*processes * -1));
-      else if (*processes == 0)
+      else if (*processes == 0) {
         *processes = -1 * *threads;
-      *threads = tmp_threads;
-      if (cpu_adjust && (*processes * tmp_threads) >= cpu_count &&
+        worker_cpu_adjust = 0;
+      }
+      *threads = tmp;
+      tmp = *processes;
+      if (worker_cpu_adjust && (*processes * *threads) >= cpu_count &&
           cpu_count > 3) {
-        /* leave a core available for the kernel */
+        /* leave a resources available for the kernel */
         --*processes;
       }
+      if (thread_cpu_adjust && (*threads * tmp) >= cpu_count && cpu_count > 3) {
+        /* leave a resources available for the kernel */
+        --*threads;
+      }
     }
   }
 
@@ -2334,11 +2343,12 @@ static intptr_t fio_unix_socket(const char *address, uint8_t server) {
   if (server) {
     unlink(addr.sun_path);
     if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+      // perror("couldn't bind unix socket");
       close(fd);
       return -1;
     }
     if (listen(fd, SOMAXCONN) < 0) {
-      // perror("couldn't start listening");
+      // perror("couldn't start listening to unix socket");
       close(fd);
       return -1;
     }
@@ -4923,7 +4933,7 @@ void fio_unsubscribe(subscription_s *s) {
     /* test again within lock */
     if (fio_ls_embd_is_empty(&ch->subscriptions)) {
       fio_ch_set_remove(&c->channels, hashed, ch, NULL);
-      removed = 1;
+      removed = (c != &fio_postoffice.filters);
     }
     fio_unlock(&c->lock);
   }
@@ -5220,7 +5230,7 @@ static void fio_publish2process(fio_msg_internal_s *m) {
   if (m->filter) {
     ch = fio_filter_find_dup(m->filter);
     if (!ch) {
-      return;
+      goto finish;
     }
   } else {
     ch = fio_channel_find_dup(m->channel);
@@ -5248,6 +5258,7 @@ static void fio_publish2process(fio_msg_internal_s *m) {
     }
     fio_unlock(&fio_postoffice.patterns.lock);
   }
+finish:
   fio_msg_internal_free(m);
 }
 
@@ -5268,6 +5279,8 @@ static void fio_publish2process(fio_msg_internal_s *m) {
 #define FIO_SET_OBJ_DESTROY(obj) fio_unsubscribe(obj)
 #include <fio.h>
 
+#define FIO_CLUSTER_NAME_LIMIT 255
+
 typedef struct cluster_pr_s {
   fio_protocol_s protocol;
   fio_msg_internal_s *msg;
@@ -5289,7 +5302,7 @@ static struct cluster_data_s {
   intptr_t uuid;
   fio_ls_s clients;
   fio_lock_i lock;
-  char name[128];
+  char name[FIO_CLUSTER_NAME_LIMIT + 1];
 } cluster_data = {.clients = FIO_LS_INIT(cluster_data.clients),
                   .lock = FIO_LOCK_INIT};
 
@@ -5306,10 +5319,9 @@ static void fio_cluster_data_cleanup(int delete_file) {
       fio_close(uuid);
     }
   }
-  cluster_data = (struct cluster_data_s){
-      .lock = FIO_LOCK_INIT,
-      .clients = (fio_ls_s)FIO_LS_INIT(cluster_data.clients),
-  };
+  cluster_data.uuid = 0;
+  cluster_data.lock = FIO_LOCK_INIT;
+  cluster_data.clients = (fio_ls_s)FIO_LS_INIT(cluster_data.clients);
 }
 
 static void fio_cluster_cleanup(void *ignore) {
@@ -5323,7 +5335,8 @@ static void fio_cluster_init(void) {
   /* create a unique socket name */
   char *tmp_folder = getenv("TMPDIR");
   uint32_t tmp_folder_len = 0;
-  if (!tmp_folder || ((tmp_folder_len = (uint32_t)strlen(tmp_folder)) > 100)) {
+  if (!tmp_folder || ((tmp_folder_len = (uint32_t)strlen(tmp_folder)) >
+                      (FIO_CLUSTER_NAME_LIMIT - 28))) {
 #ifdef P_tmpdir
     tmp_folder = (char *)P_tmpdir;
     if (tmp_folder)
@@ -5333,7 +5346,7 @@ static void fio_cluster_init(void) {
     tmp_folder_len = 5;
 #endif
   }
-  if (tmp_folder_len >= 100) {
+  if (tmp_folder_len >= (FIO_CLUSTER_NAME_LIMIT - 28)) {
     tmp_folder_len = 0;
   }
   if (tmp_folder_len) {
@@ -5343,8 +5356,9 @@ static void fio_cluster_init(void) {
   }
   memcpy(cluster_data.name + tmp_folder_len, "facil-io-sock-", 14);
   tmp_folder_len += 14;
-  tmp_folder_len += snprintf(cluster_data.name + tmp_folder_len,
-                             127 - tmp_folder_len, "%d", getpid());
+  tmp_folder_len +=
+      snprintf(cluster_data.name + tmp_folder_len,
+               FIO_CLUSTER_NAME_LIMIT - tmp_folder_len, "%d", getpid());
   cluster_data.name[tmp_folder_len] = 0;
 
   /* remove if existing */
@@ -5505,6 +5519,7 @@ static void fio_cluster_on_close(intptr_t uuid, fio_protocol_s *pr_) {
   if (c->msg)
     fio_msg_internal_free(c->msg);
   c->msg = NULL;
+  fio_sub_hash_free(&c->pubsub);
   fio_cluster_protocol_free(c);
   (void)uuid;
 }
@@ -5659,7 +5674,6 @@ static void fio_cluster_listen_on_close(intptr_t uuid,
 static void fio_listen2cluster(void *ignore) {
   /* this is called for each `fork`, but we only need this to run once. */
   fio_lock(&cluster_data.lock);
-  fio_cluster_init();
   cluster_data.uuid = fio_socket(cluster_data.name, NULL, 1);
   fio_unlock(&cluster_data.lock);
   if (cluster_data.uuid < 0) {
@@ -6671,8 +6685,10 @@ Allocator Initialization (initialize arenas and allocate a block for each CPU)
 #if DEBUG
 void fio_memory_dump_missing(void) {
   fprintf(stderr, "\n ==== Attempting Memory Dump (will crash) ====\n");
-  if (fio_ls_embd_any(&memory.available))
+  if (fio_ls_embd_is_empty(&memory.available)) {
+    fprintf(stderr, "- Memory dump attempt canceled\n");
     return;
+  }
   block_node_s *smallest =
       FIO_LS_EMBD_OBJ(block_node_s, node, memory.available.next);
   FIO_LS_EMBD_FOR(&memory.available, node) {
@@ -6732,8 +6748,11 @@ static void fio_mem_destroy(void) {
     FIO_MEMORY_PRINT_BLOCK_STAT_END();
     size_t count = 0;
     FIO_LS_EMBD_FOR(&memory.available, node) { ++count; }
-    FIO_LOG_DEBUG("Memory pool size: %zu (%zu blocks per allocation).", count,
-                  (size_t)FIO_MEMORY_BLOCKS_PER_ALLOCATION);
+    FIO_LOG_DEBUG("Memory blocks in pool: %zu (%zu blocks per allocation).",
+                  count, (size_t)FIO_MEMORY_BLOCKS_PER_ALLOCATION);
+#if FIO_MEM_DUMP
+    fio_memory_dump_missing();
+#endif
   }
   big_free(arenas);
   arenas = NULL;
@@ -8545,7 +8564,7 @@ Testing Memory Allocator
 #define fio_malloc_test()                                                      \
   fprintf(stderr, "\n=== SKIPPED facil.io memory allocator (bypassed)\n");
 #else
-void fio_malloc_test(void) {
+FIO_FUNC void fio_malloc_test(void) {
   fprintf(stderr, "\n=== Testing facil.io memory allocator's system calls\n");
   char *mem = sys_alloc(FIO_MEMORY_BLOCK_SIZE, 0);
   FIO_ASSERT(mem, "sys_alloc failed to allocate memory!\n");
@@ -8558,7 +8577,7 @@ void fio_malloc_test(void) {
       sys_realloc(mem, FIO_MEMORY_BLOCK_SIZE, FIO_MEMORY_BLOCK_SIZE * 2);
   if (mem == mem2)
     fprintf(stderr, "* Performed system realloc without copy :-)\n");
-  FIO_ASSERT(mem2[0] = 'a' && mem2[FIO_MEMORY_BLOCK_SIZE - 1] == 'z',
+  FIO_ASSERT(mem2[0] == 'a' && mem2[FIO_MEMORY_BLOCK_SIZE - 1] == 'z',
              "Reaclloc data was lost!");
   sys_free(mem2, FIO_MEMORY_BLOCK_SIZE * 2);
   fprintf(stderr, "=== Testing facil.io memory allocator's internal data.\n");
@@ -8572,8 +8591,10 @@ void fio_malloc_test(void) {
   mem[0] = 'a';
   FIO_ASSERT(mem[0] == 'a', "allocate memory wasn't written to!\n");
   mem = fio_realloc(mem, 1);
+  FIO_ASSERT(mem, "fio_realloc failed!\n");
   FIO_ASSERT(mem[0] == 'a', "fio_realloc memory wasn't copied!\n");
   FIO_ASSERT(arena_last_used, "arena_last_used wasn't initialized!\n");
+  fio_free(mem);
   block_s *b = arena_last_used->block;
 
   /* move arena to block's start */
@@ -8639,7 +8660,9 @@ void fio_malloc_test(void) {
     ++count;
   } while (arena_last_used->block == b);
 
+  mem2 = mem;
   mem = fio_calloc(FIO_MEMORY_BLOCK_ALLOC_LIMIT - 64, 1);
+  fio_free(mem2);
   FIO_ASSERT(mem,
              "failed to allocate FIO_MEMORY_BLOCK_ALLOC_LIMIT - 64 bytes!\n");
   FIO_ASSERT(((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) != 16,
@@ -8669,11 +8692,18 @@ void fio_malloc_test(void) {
     void *m0 = fio_malloc(0);
     void *rm0 = fio_realloc(m0, 16);
     FIO_ASSERT(m0 != rm0, "fio_realloc(fio_malloc(0), 16) failed!\n");
+    fio_free(rm0);
   }
   {
+    size_t pool_size = 0;
+    FIO_LS_EMBD_FOR(&memory.available, node) { ++pool_size; }
     mem = fio_mmap(512);
     FIO_ASSERT(mem, "fio_mmap allocation failed!\n");
     fio_free(mem);
+    size_t new_pool_size = 0;
+    FIO_LS_EMBD_FOR(&memory.available, node) { ++new_pool_size; }
+    FIO_ASSERT(new_pool_size == pool_size,
+               "fio_free of fio_mmap went to memory pool!\n");
   }
 
   fprintf(stderr, "* passed.\n");
@@ -8684,12 +8714,12 @@ void fio_malloc_test(void) {
 Testing Core Callback add / remove / ensure
 ***************************************************************************** */
 
-static void fio_state_callback_test_task(void *pi) {
+FIO_FUNC void fio_state_callback_test_task(void *pi) {
   ((uintptr_t *)pi)[0] += 1;
 }
 
 #define FIO_STATE_TEST_COUNT 10
-static void fio_state_callback_order_test_task(void *pi) {
+FIO_FUNC void fio_state_callback_order_test_task(void *pi) {
   static uintptr_t start = FIO_STATE_TEST_COUNT;
   --start;
   FIO_ASSERT((uintptr_t)pi == start,
@@ -8697,7 +8727,7 @@ static void fio_state_callback_order_test_task(void *pi) {
              (size_t)pi);
 }
 
-static void fio_state_callback_test(void) {
+FIO_FUNC void fio_state_callback_test(void) {
   fprintf(stderr, "=== Testing facil.io workflow state callback system\n");
   uintptr_t result = 0;
   uintptr_t other = 0;
@@ -8730,9 +8760,9 @@ static void fio_state_callback_test(void) {
 Testing fio_timers
 ***************************************************************************** */
 
-static void fio_timer_test_task(void *arg) { ++(((size_t *)arg)[0]); }
+FIO_FUNC void fio_timer_test_task(void *arg) { ++(((size_t *)arg)[0]); }
 
-static void fio_timer_test(void) {
+FIO_FUNC void fio_timer_test(void) {
   fprintf(stderr, "=== Testing facil.io timer system\n");
   size_t result = 0;
   const size_t total = 5;
@@ -8796,7 +8826,7 @@ static void fio_timer_test(void) {
 Testing listening socket
 ***************************************************************************** */
 
-static void fio_socket_test(void) {
+FIO_FUNC void fio_socket_test(void) {
   /* initialize unix socket name */
   fio_str_s sock_name = FIO_STR_INIT;
 #ifdef P_tmpdir
@@ -8896,17 +8926,17 @@ static void fio_socket_test(void) {
 Testing listening socket
 ***************************************************************************** */
 
-static void fio_cycle_test_task(void *arg) {
+FIO_FUNC void fio_cycle_test_task(void *arg) {
   fio_stop();
   (void)arg;
 }
-static void fio_cycle_test_task2(void *arg) {
+FIO_FUNC void fio_cycle_test_task2(void *arg) {
   fprintf(stderr, "* facil.io cycling test fatal error!\n");
   exit(-1);
   (void)arg;
 }
 
-static void fio_cycle_test(void) {
+FIO_FUNC void fio_cycle_test(void) {
   fprintf(stderr,
           "=== Testing facil.io cycling logic (partial - only tests timers)\n");
   fio_mark_time();
@@ -8931,18 +8961,18 @@ Testing fio_defer task system
 #define FIO_DEFER_TEST_PRINT 0
 #endif
 
-static void sample_task(void *i_count, void *unused2) {
+FIO_FUNC void sample_task(void *i_count, void *unused2) {
   (void)(unused2);
   fio_atomic_add((uintptr_t *)i_count, 1);
 }
 
-static void sched_sample_task(void *count, void *i_count) {
+FIO_FUNC void sched_sample_task(void *count, void *i_count) {
   for (size_t i = 0; i < (uintptr_t)count; i++) {
     fio_defer(sample_task, i_count, NULL);
   }
 }
 
-static void fio_defer_test(void) {
+FIO_FUNC void fio_defer_test(void) {
   const size_t cpu_cores = fio_detect_cpu_cores();
   FIO_ASSERT(cpu_cores, "couldn't detect CPU cores!");
   uintptr_t i_count;
@@ -9015,8 +9045,8 @@ typedef struct {
 #define FIO_ARY_TYPE uintptr_t
 #include "fio.h"
 
-static intptr_t ary_alloc_counter = 0;
-static void copy_s(fio_ary_test_type_s *d, fio_ary_test_type_s *s) {
+FIO_FUNC intptr_t ary_alloc_counter = 0;
+FIO_FUNC void copy_s(fio_ary_test_type_s *d, fio_ary_test_type_s *s) {
   ++ary_alloc_counter;
   *d = *s;
 }
@@ -9028,7 +9058,7 @@ static void copy_s(fio_ary_test_type_s *d, fio_ary_test_type_s *s) {
 #define FIO_ARY_DESTROY(obj) (--ary_alloc_counter)
 #include "fio.h"
 
-void fio_ary_test(void) {
+FIO_FUNC void fio_ary_test(void) {
   /* code */
   fio_i_ary__test();
   fio_s_ary__test();
@@ -9233,7 +9263,7 @@ FIO_FUNC void fio_set_test(void) {
 SipHash tests
 ***************************************************************************** */
 
-static void fio_siphash_speed_test(void) {
+FIO_FUNC void fio_siphash_speed_test(void) {
   /* test based on code from BearSSL with credit to Thomas Pornin */
   uint8_t buffer[8192];
   memset(buffer, 'T', sizeof(buffer));
@@ -9283,7 +9313,7 @@ static void fio_siphash_speed_test(void) {
   }
 }
 
-void fio_siphash_test(void) {
+FIO_FUNC void fio_siphash_test(void) {
   fprintf(stderr, "===================================\n");
 #if NODEBUG
   fio_siphash_speed_test();
@@ -9297,7 +9327,7 @@ void fio_siphash_test(void) {
 SHA-1 tests
 ***************************************************************************** */
 
-static void fio_sha1_speed_test(void) {
+FIO_FUNC void fio_sha1_speed_test(void) {
   /* test based on code from BearSSL with credit to Thomas Pornin */
   uint8_t buffer[8192];
   uint8_t result[21];
@@ -9331,7 +9361,7 @@ static void fio_sha1_speed_test(void) {
 }
 
 #ifdef HAVE_OPENSSL
-static void fio_sha1_open_ssl_speed_test(void) {
+FIO_FUNC void fio_sha1_open_ssl_speed_test(void) {
   /* test based on code from BearSSL with credit to Thomas Pornin */
   uint8_t buffer[8192];
   uint8_t result[21];
@@ -9365,7 +9395,7 @@ static void fio_sha1_open_ssl_speed_test(void) {
 }
 #endif
 
-void fio_sha1_test(void) {
+FIO_FUNC void fio_sha1_test(void) {
   // clang-format off
   struct {
     char *str;
@@ -9436,12 +9466,13 @@ void fio_sha1_test(void) {
 SHA-2 tests
 ***************************************************************************** */
 
-static char *sha2_variant_names[] = {
+FIO_FUNC char *sha2_variant_names[] = {
     "unknown", "SHA_512",     "SHA_256", "SHA_512_256",
     "SHA_224", "SHA_512_224", "none",    "SHA_384",
 };
 
-static void fio_sha2_speed_test(fio_sha2_variant_e var, const char *var_name) {
+FIO_FUNC void fio_sha2_speed_test(fio_sha2_variant_e var,
+                                  const char *var_name) {
   /* test based on code from BearSSL with credit to Thomas Pornin */
   uint8_t buffer[8192];
   uint8_t result[65];
@@ -9474,9 +9505,9 @@ static void fio_sha2_speed_test(fio_sha2_variant_e var, const char *var_name) {
   }
 }
 
-static void fio_sha2_openssl_speed_test(const char *var_name, int (*init)(),
-                                        int (*update)(), int (*final)(),
-                                        void *sha) {
+FIO_FUNC void fio_sha2_openssl_speed_test(const char *var_name, int (*init)(),
+                                          int (*update)(), int (*final)(),
+                                          void *sha) {
   /* test adapted from BearSSL code with credit to Thomas Pornin */
   uint8_t buffer[8192];
   uint8_t result[1024];
@@ -9507,7 +9538,7 @@ static void fio_sha2_openssl_speed_test(const char *var_name, int (*init)(),
     cycles <<= 1;
   }
 }
-void fio_sha2_test(void) {
+FIO_FUNC void fio_sha2_test(void) {
   fio_sha2_s s;
   char *expect;
   char *got;
@@ -9651,7 +9682,7 @@ error:
 Base64 tests
 ***************************************************************************** */
 
-static void fio_base64_speed_test(void) {
+FIO_FUNC void fio_base64_speed_test(void) {
   /* test based on code from BearSSL with credit to Thomas Pornin */
   char buffer[8192];
   char result[8192 * 2];
@@ -9706,7 +9737,7 @@ static void fio_base64_speed_test(void) {
   }
 }
 
-void fio_base64_test(void) {
+FIO_FUNC void fio_base64_test(void) {
   struct {
     char *str;
     char *base64;
@@ -9780,7 +9811,7 @@ void fio_base64_test(void) {
 Random Testing
 ***************************************************************************** */
 
-void fio_test_random(void) {
+FIO_FUNC void fio_test_random(void) {
   fprintf(stderr, "=== Testing random generator\n");
   uint64_t rnd = fio_rand64();
   FIO_ASSERT((rnd != fio_rand64() && rnd != fio_rand64()),
@@ -9818,7 +9849,7 @@ void fio_test_random(void) {
 Poll (not kqueue or epoll) tests
 ***************************************************************************** */
 #if FIO_ENGINE_POLL
-static void fio_poll_test(void) {
+FIO_FUNC void fio_poll_test(void) {
   fprintf(stderr, "=== Testing poll add / remove fd\n");
   fio_poll_add(5);
   FIO_ASSERT(fio_data->start == 5,
@@ -9869,11 +9900,11 @@ static void fio_poll_test(void) {
 Test UUID Linking
 ***************************************************************************** */
 
-static void fio_uuid_link_test_on_close(void *obj) {
+FIO_FUNC void fio_uuid_link_test_on_close(void *obj) {
   fio_atomic_add((uintptr_t *)obj, 1);
 }
 
-static void fio_uuid_link_test(void) {
+FIO_FUNC void fio_uuid_link_test(void) {
   fprintf(stderr, "=== Testing fio_uuid_link\n");
   uintptr_t called = 0;
   uintptr_t removed = 0;
@@ -9896,7 +9927,7 @@ static void fio_uuid_link_test(void) {
 Byte Order Testing
 ***************************************************************************** */
 
-static void fio_str2u_test(void) {
+FIO_FUNC void fio_str2u_test(void) {
   fprintf(stderr, "=== Testing fio_u2strX and fio_u2strX functions.\n");
   char buffer[32];
   for (int64_t i = -1024; i < 1024; ++i) {
@@ -9929,15 +9960,15 @@ Pub/Sub partial tests
 
 #if FIO_PUBSUB_SUPPORT
 
-static void fio_pubsub_test_on_message(fio_msg_s *msg) {
+FIO_FUNC void fio_pubsub_test_on_message(fio_msg_s *msg) {
   fio_atomic_add((uintptr_t *)msg->udata1, 1);
 }
-static void fio_pubsub_test_on_unsubscribe(void *udata1, void *udata2) {
+FIO_FUNC void fio_pubsub_test_on_unsubscribe(void *udata1, void *udata2) {
   fio_atomic_add((uintptr_t *)udata1, 1);
   (void)udata2;
 }
 
-static void fio_pubsub_test(void) {
+FIO_FUNC void fio_pubsub_test(void) {
   fprintf(stderr, "=== Testing pub/sub (partial)\n");
   fio_data->active = 1;
   fio_data->is_worker = 1;
@@ -9953,10 +9984,11 @@ static void fio_pubsub_test(void) {
   fio_u2str32((uint8_t *)buffer, 4);
   FIO_ASSERT(fio_str2u32((uint8_t *)buffer) == 4,
              "fio_u2str32 / fio_str2u32 not reversible (4)!");
-  s = fio_subscribe(.filter = 1, .udata1 = &counter,
+  subscription_s *s2 =
+      fio_subscribe(.filter = 1, .udata1 = &counter,
                     .on_message = fio_pubsub_test_on_message,
                     .on_unsubscribe = fio_pubsub_test_on_unsubscribe);
-  FIO_ASSERT(s, "fio_subscribe FAILED on filtered subscription.");
+  FIO_ASSERT(s2, "fio_subscribe FAILED on filtered subscription.");
   fio_publish(.filter = 1);
   ++expect;
   fio_defer_perform();
@@ -9965,6 +9997,7 @@ static void fio_pubsub_test(void) {
   fio_defer_perform();
   FIO_ASSERT(counter == expect, "publishing to filter 2 arrived at filter 1!");
   fio_unsubscribe(s);
+  fio_unsubscribe(s2);
   ++expect;
   fio_defer_perform();
   FIO_ASSERT(counter == expect, "unsubscribe wasn't called for filter 1!");
@@ -9987,6 +10020,7 @@ static void fio_pubsub_test(void) {
   fio_data->is_worker = 0;
   fio_data->active = 0;
   fio_data->workers = 0;
+  fio_defer_perform();
   (void)fio_pubsub_test_on_message;
   (void)fio_pubsub_test_on_unsubscribe;
   fprintf(stderr, "* passed.\n");
@@ -10004,7 +10038,7 @@ String 2 Number and Number 2 String (partial) testing
 #else
 #define FIO_ATOL_TEST_MAX_CYCLES 4096
 #endif
-static void fio_atol_test(void) {
+FIO_FUNC void fio_atol_test(void) {
   fprintf(stderr, "=== Testing fio_ltoa and fio_atol (partial)\n");
 #ifndef NODEBUG
   fprintf(stderr,
@@ -10159,7 +10193,7 @@ static void fio_atol_test(void) {
 String 2 Float and Float 2 String (partial) testing
 ***************************************************************************** */
 
-static void fio_atof_test(void) {
+FIO_FUNC void fio_atof_test(void) {
   fprintf(stderr, "=== Testing fio_ftoa and fio_ftoa (partial)\n");
 #define TEST_DOUBLE(s, d, must)                                                \
   do {                                                                         \