iodine 0.7.9 → 0.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abc090d98d50e754078aabdc8e46065b7e56b5d79f563866e563365c133ba088
-  data.tar.gz: 03e59675043f7b7f7e31aa640bff7f8ef01912cc074bc0d576461dc31f3a38ca
+  metadata.gz: bd4ea42a7f008886bcadee4c829499686a1fbceeb2738daf56e4155063d9054c
+  data.tar.gz: aeb66d448ad9cd09e55f7f15c79bcb1b3e8f0a41f3e2adfc9e2c894c20d7c80f
 SHA512:
-  metadata.gz: 513c570195901322763ec71076bc9697a8d365885db8b4b10fbd072aee9f36500cecaf33b5b9cd0df198558a512fdef318b83c72c7e1d452fece7c2ca70bf22d
-  data.tar.gz: a6ab97b226cca87200be69507910951ba7f0036e2283fbc8b4b9d67926e4c4ddaa1b87e193aa7776e87d411b7ea371987a1e434c1017c804f17cd9eba6872a92
+  metadata.gz: ecdde8977f2b48089f370356f3bbd272054df193f9be220ac28ad8491c59f749a1c21e1506bb45c9f2532e9bf072b9e7bf365c92ccb4b3a651b52bacbdfb21ad
+  data.tar.gz: 5268a0318e3484a0a703fa68e52f057c6cab513066243ff37e052c58669d8babe5c53650082349bb9b2610f80afd0f5e551f530a3e6d662ad49dfa4609f3d6b1

data/.travis.yml

@@ -22,10 +22,16 @@ addons:
       - gcc-4.9
       - gcc-5
 script:
+  - echo CFLAGS = $CFLAGS
+  - echo cflags = $cflags
   - gem uninstall -x iodine
   - rake build
   - find pkg/iodine-*.gem -exec gem install -V {} +
   - gem uninstall -x iodine
-  - CC=gcc find pkg/iodine-*.gem -exec gem install -V {} +
+  - export CFLAGS="-std=c99"
+  - export CC=gcc
+  - find pkg/iodine-*.gem -exec gem install -V {} +
   - gem uninstall -x iodine
-  - CC=gcc-5 find pkg/iodine-*.gem -exec gem install -V {} +
+  - export CFLAGS="-Wall"
+  - export CC=gcc-5
+  - find pkg/iodine-*.gem -exec gem install -V {} +

data/CHANGELOG.md

@@ -6,6 +6,20 @@ Please notice that this change log contains changes for upcoming releases as wel
 
 ## Changes:
 
+#### Change log v.0.7.10
+
+**Fix**: (pub/sub) fixed connection lock for pub/sub tasks. Now pub/sub Ruby tasks will lock the connection, protecting the user's code against concurrent access to the connection's data.
+
+**Fix**: (installation) fixed `CFLAGS` compilation value to allow for pre-existing values set by Ruby.
+
+**Fix**: (installation) fixed possible issues than could occur when installing iodine with  `FIO_FORCE_MALLOC`.
+
+**Optimization**: (pub/sub) leverages facil.io broadcasting optimizations, minimizing memory allocations when broadcasting pub/sub messages directly to multiple WebSocket clients.
+
+**Update**: (fio) updated the facil.io code to leverage it's urgent task queue for outbound IO, which minimizes reliance on the IO backup thread.
+
+**Update**: (IO) minor tweaks to the IO backup thread and CLI output format.
+
 #### Change log v.0.7.9
 
 **Fix**: fixed the background IO backup thread initialization and sleep interval. This thread isn't critical. It's only used to (slowly) flush sockets when all the actual threads are blocked by long running Ruby application code.

data/README.md

@@ -60,6 +60,16 @@ During development, it's more common to use a single process and a few threads:
 bundler exec iodine -p $PORT -t 16 -w 1
 ```
 
+### Heap Fragmentation Protection
+
+Iodine includes a network oriented custom memory allocator, with very high performance.
+
+This allows the heap to be divided, naturally, into long-living objects (allocated normally) and short living objects (allocated using the iodine allocator).
+
+This approach helps to minimize heap fragmentation for long running processes.
+
+It's still recommended to consider [jemalloc](http://jemalloc.net) or other allocators to mitigate the heap fragmentation that would be caused by Ruby's internal memory management.
+
 ### Static file serving support
 
 Iodine supports an internal static file service that bypasses the Ruby layer  and serves static files directly from "C-land".
@@ -95,11 +105,9 @@ bundler exec iodine -p $PORT -t 16 -w 4 -www /my/public/folder -v
 
 #### X-Sendfile
 
-Ruby can leverage static file support (if enabled) by using the `X-Sendfile` header in the Ruby application response.
-
-To enable iodine's native X-Sendfile support, a static file service (a public folder) needs to be assigned (this informs iodine that static files aren't sent using a different layer, such as nginx).
+When a public folder is assigned (the static file server is active), iodine automatically adds support for the `X-Sendfile` header in any Ruby application response.
 
-This allows Ruby to send very large files using a very small memory footprint, as well as (when possible) leveraging the `sendfile` system call.
+This allows Ruby to send very large files using a very small memory footprint and usually leverages the `sendfile` system call.
 
 i.e. (example `config.ru` for iodine):
 
@@ -121,19 +129,21 @@ end
 run app
 ```
 
-Go to [localhost:3000/source](http://localhost:3000/source) to experience the `X-Sendfile` extension at work.
+Benchmark [localhost:3000/source](http://localhost:3000/source) to experience the `X-Sendfile` extension at work.
 
 #### Pre-Compressed assets / files
 
-Rails does this automatically when compiling assets - simply `gzip` your static files.
+Rails does this automatically when compiling assets, which is: `gzip` your static files.
 
 Iodine will automatically recognize and send the `gz` version if the client (browser) supports the `gzip` transfer-encoding.
 
 For example, to offer a compressed version of `style.css`, run (in the terminal):
 
-      $  gzip -k -9 style.css
+```bash
+$  gzip -k -9 style.css
+```
 
-Now, you will have two files in your folder, `style.css` and `style.css.gz`.
+This results in both files, `style.css` (the original) and `style.css.gz` (the compressed).
 
 When a browser that supports compressed encoding (which is most browsers) requests the file, iodine will recognize that a pre-compressed option exists and will prefer the `gzip` compressed version.
 
@@ -203,11 +213,19 @@ Iodine.start
 run APP
 ```
 
-#### Native Pub/Sub with *optional* Redis scaling
+### Native Pub/Sub with *optional* Redis scaling
+
+Iodine's core, `facil.io` offers a native Pub/Sub implementation that can be scaled across machine boundaries using Redis.
 
-Iodine's core, `facil.io` offers a native Pub/Sub implementation. The implementation is totally native to iodine, it covers the whole process cluster and it can be easily scaled by using Redis (which isn't required except for horizontal scaling).
+The default implementation covers the whole process cluster, so a single cluster doesn't need Redis
+
+Once a single iodine process cluster isn't enough, horizontal scaling for the Pub/Sub layer is as simple as connecting iodine to Redis using the `-r <url>` from the command line. i.e.:
+
+```bash
+$ iodine -w -1 -t 8 -r redis://localhost
+```
 
-Here's an example that adds horizontal scaling to the chat application in the previous example, so that Pub/Sub messages are published across many machines at once:
+It's also possible to initialize the iodine<=>Redis link using Ruby, directly from the application's code:
 
 ```ruby
 # initialize the Redis engine for each iodine process.
@@ -219,7 +237,7 @@ end
 # ... the rest of the application remains unchanged.
 ```
 
-The new Redis client can also be used for asynchronous Redis command execution. i.e.:
+Iodine's Redis client can also be used for asynchronous Redis command execution. i.e.:
 
 ```ruby
 if(Iodine::PubSub.default.is_a? Iodine::PubSub::Redis)
@@ -230,11 +248,13 @@ end
 
 **Pub/Sub Details and Limitations:**
 
-* Iodine's Redis client does *not* support multiple databases. This is both because [database scoping is ignored by Redis during pub/sub](https://redis.io/topics/pubsub#database-amp-scoping) and because [Redis Cluster doesn't support multiple databases](https://redis.io/topics/cluster-spec). This indicated that multiple database support just isn't worth the extra effort.
+* Iodine's Redis client does *not* support multiple databases. This is both because [database scoping is ignored by Redis during pub/sub](https://redis.io/topics/pubsub#database-amp-scoping) and because [Redis Cluster doesn't support multiple databases](https://redis.io/topics/cluster-spec). This indicated that multiple database support just isn't worth the extra effort and performance hit.
 
-* The iodine Redis client will use a single Redis connection per process (for publishing data) and an extra Redis connection for subscriptions (owned by the master process). Connections will be automatically re-established if timeouts or errors occur.
+* The iodine Redis client will use two Redis connections for the whole process cluster (a single publishing connection and a single subscription connection), minimizing the Redis load and network bandwidth.
+* 
+Connections will be automatically re-established if timeouts or errors occur.
 
-#### Hot Restart
+### Hot Restart
 
 Iodine will "hot-restart" the application by shutting down and re-spawning the worker processes.
 
@@ -254,7 +274,7 @@ Since the master / root process doesn't handle any requests (it only handles pub
 
 **Note**: This will **not** re-load the application (any changes to the Ruby code require an actual restart).
 
-#### Optimized HTTP logging
+### Optimized HTTP logging
 
 By default, iodine is pretty quite. Some messages are logged to `stderr`, but not many.
 
@@ -278,7 +298,7 @@ The log output can also be redirected to a `stdout`:
 bundler exec iodine -p $PORT -v  2>&1
 ```
 
-#### Built-in support for Sequel and ActiveRecord
+### Built-in support for Sequel and ActiveRecord
 
 It's a well known fact that Database connections require special attention when using `fork`-ing servers (multi-process servers) such as Puma, Passenger and iodine.
 
@@ -288,7 +308,7 @@ With iodine, there's no need to worry.
 
 Iodine provides built-in `fork` handling for both ActiveRecord and Sequel, in order to protect against these possible errors.
 
-#### TCP/IP (raw) sockets
+### TCP/IP (raw) sockets
 
 Upgrading to a custom protocol (i.e., in order to implement your own WebSocket protocol with special extensions) is available when neither WebSockets nor SSE connection upgrades were requested. In the following (terminal) example, we'll use an echo server without direct socket echo:
 
@@ -317,14 +337,6 @@ Iodine.start
 run APP
 ```
 
-#### A few notes
-
-This design has a number of benefits, some of them related to better IO handling, resource optimization (no need for two IO polling systems), etc. This also allows us to use middleware without interfering with connection upgrades and provides backwards compatibility.
-
-Iodine's HTTP server imposes a few restrictions for performance and security reasons, such as limiting each header line to 8Kb. These restrictions shouldn't be an issue and are similar to limitations imposed by Apache or Nginx.
-
-If you still want to use Rack's `hijack` API, iodine will support you - but be aware that you will need to implement your own reactor and thread pool for any sockets you hijack, as well as a socket buffer for non-blocking `write` operations (why do that when you can write a protocol object and have the main reactor manage the socket?).
-
 ### How does it compare to other servers?
 
 The honest answer is "I don't know". I recommend that you perform your own tests.
@@ -384,7 +396,46 @@ $ RACK_ENV=production puma -p 3000 -t 16 -w 4
 
 It's recommended that the servers (Iodine/Puma) and the client (`wrk`/`ab`) run on separate machines.
 
-### Performance oriented design - but safety first
+### A few notes
+
+Iodine's upgrade / callback design has a number of benefits, some of them related to better IO handling, resource optimization (no need for two IO polling systems), etc. This also allows us to use middleware without interfering with connection upgrades and provides backwards compatibility.
+
+Iodine's HTTP server imposes a few restrictions for performance and security reasons, such as limiting each header line to 8Kb. These restrictions shouldn't be an issue and are similar to limitations imposed by Apache or Nginx.
+
+If you still want to use Rack's `hijack` API, iodine will support you - but be aware that you will need to implement your own reactor and thread pool for any sockets you hijack, as well as a socket buffer for non-blocking `write` operations (why do that when you can write a protocol object and have the main reactor manage the socket?).
+
+## Installation
+
+To install iodine, simply install the the `iodine` gem:
+
+```bash
+$ gem install iodine
+```
+
+Iodine is written in C and allows some compile-time customizations, such as:
+
+* `FIO_FORCE_MALLOC` - avoids iodine's custom memory allocator and use `malloc` instead (mostly used when debugging iodine or when using a different memory allocator).
+
+* `FIO_MAX_SOCK_CAPACITY` - limits iodine's maximum client capacity. Defaults to 131,072 clients.
+
+* `HTTP_MAX_HEADER_COUNT` - limits the number of headers the HTTP server will accept before disconnecting a client (security). Defaults to 128 headers (permissive).
+
+* `HTTP_MAX_HEADER_LENGTH` - limits the number of bytes allowed for a single header (pre-allocated memory per connection + security). Defaults to 8Kb per header line (normal).
+
+* `HTTP_BUSY_UNLESS_HAS_FDS` - requires at least X number of free file descriptors (for new database connections, etc') before accepting a new HTTP client.
+
+* `FIO_ENGINE_POLL` - prefer the `poll` system call over `epoll` or `kqueue` (not recommended).
+
+These options can be used, for example, like so:
+
+```bash
+$ CFLAGS="-DFIO_FORCE_MALLOC=1 -DHTTP_MAX_HEADER_COUNT=64" \
+  gem install iodine
+```
+
+More possible compile time options can be found in the [facil.io documentation](http://facil.io).
+
+## Evented oriented design with extra safety
 
 Iodine is an evened server, similar in it's architecture to `nginx` and `puma`. It's different than the simple "thread-per-client" design that is often taught when we begin to learn about network programming.
 
@@ -417,12 +468,17 @@ The server events are fairly fast and fragmented (longer code is fragmented acro
 
 ...but single threaded mode should probably be avoided.
 
-The thread pool is there to help slow user code.
 
 It's very common that the application's code will run slower and require external resources (i.e., databases, a custom pub/sub service, etc'). This slow code could "starve" the server, which is patiently waiting to run it's tasks on the same thread.
 
+The thread pool is there to help slow user code.
+
 The slower your application code, the more threads you will need to keep the server running in a responsive manner (note that responsiveness and speed aren't always the same).
 
+To make a thread pool easier and safer to use, iodine makes sure that no connection task / callback is called concurrently for the same connection.
+
+For example, a is a WebSocket connection is already busy in it's `on_message` callback, no other messages will be forwarded to the callback until the current callback returns.
+
 ## Free, as in freedom (BYO beer)
 
 Iodine is **free** and **open source**, so why not take it out for a spin?
@@ -475,7 +531,7 @@ Iodine.start
 
 ```
 
-## Why not EventMachine?
+### Why not EventMachine?
 
 You can go ahead and use EventMachine if you like. They're doing amazing work on that one and it's been used a lot in Ruby-land... really, tons of good developers and people on that project.
 

data/examples/shootout.ru

@@ -15,10 +15,10 @@ module ShootoutApp
     env.each { |k, v| out << "#{k}: #{v}\n" ; len += out[-1].length }
     request = Rack::Request.new(env)
     out << "\nRequest Path: #{request.path_info}\n"
-    len += out[-1].length 
+    len += out[-1].length
     unless request.params.empty?
       out << "Params:\n"
-      len += out[-1].length 
+      len += out[-1].length
       request.params.each { |k,v| out << "#{k}: #{v}\n" ; len += out[-1].length }
     end
     [200, { 'Content-Length' => len.to_s, 'Content-Type' => 'text/plain; charset=UTF-8;' }, out]
@@ -27,8 +27,8 @@ module ShootoutApp
   # It's slower than writing to every socket a pre-parsed message, but it's closer
   # to real-life implementations.
   def self.on_open client
-      client.subscribe :shootout_b, as: :binary
-      client.subscribe :shootout
+      client.subscribe(:shootout_b, as: :binary) # { |ch, msg| client.write(msg)}
+      client.subscribe(:shootout) # { |ch, msg| client.write(msg)}
   end
   def self.on_message client, data
     if data[0] == 'b' # binary

data/ext/iodine/extconf.rb

@@ -29,7 +29,7 @@ else
   puts 'using an unknown (old?) compiler... who knows if this will work out... we hope.'
 end
 
-$CFLAGS = "-std=c11 -O2 -Wall -DFIO_PRINT_STATE=0 #{ENV['CFLAGS']}"
+RbConfig::MAKEFILE_CONFIG['CFLAGS'] = $CFLAGS = "-std=c11 -DFIO_PRINT_STATE=0 #{$CFLAGS} #{$CFLAGS == ENV['CFLAGS'] ? "" : ENV['CFLAGS']}"
 RbConfig::MAKEFILE_CONFIG['CC'] = $CC = ENV['CC'] if ENV['CC']
 RbConfig::MAKEFILE_CONFIG['CPP'] = $CPP = ENV['CPP'] if ENV['CPP']
 

data/ext/iodine/fio.c

@@ -65,6 +65,10 @@ Feel free to copy, use and enjoy according to the license provided.
 #define FIO_POLL_TICK 1000
 #endif
 
+#ifndef FIO_USE_URGENT_QUEUE
+#define FIO_USE_URGENT_QUEUE 1
+#endif
+
 #ifndef DEBUG_SPINLOCK
 #define DEBUG_SPINLOCK 0
 #endif
@@ -510,191 +514,6 @@ Section Start Marker
 
 
 
-                                     Timers
-
-
-
-
-
-
-
-
-
-
-***************************************************************************** */
-
-typedef struct {
-  fio_ls_embd_s node;
-  struct timespec due;
-  size_t interval; /*in ms */
-  size_t repetitions;
-  void (*task)(void *);
-  void *arg;
-  void (*on_finish)(void *);
-} fio_timer_s;
-
-static fio_ls_embd_s fio_timers = FIO_LS_INIT(fio_timers);
-
-static fio_lock_i fio_timer_lock = FIO_LOCK_INIT;
-
-/** Marks the current time as facil.io's cycle time */
-static inline void fio_mark_time(void) {
-  clock_gettime(CLOCK_REALTIME, &fio_data->last_cycle);
-}
-
-/** Calculates the due time for a task, given it's interval */
-static struct timespec fio_timer_calc_due(size_t interval) {
-  struct timespec now = fio_last_tick();
-  if (interval > 1000) {
-    now.tv_sec += interval / 1000;
-    interval -= interval / 1000;
-  }
-  now.tv_nsec += (interval * 1000000UL);
-  if (now.tv_nsec > 1000000000L) {
-    now.tv_nsec -= 1000000000L;
-    now.tv_sec += 1;
-  }
-  return now;
-}
-
-/** Returns the number of miliseconds until the next event, up to FIO_POLL_TICK
- */
-static size_t fio_timer_calc_first_interval(void) {
-  if (fio_defer_has_queue())
-    return 0;
-  if (fio_ls_embd_is_empty(&fio_timers)) {
-    return FIO_POLL_TICK;
-  }
-  struct timespec now = fio_last_tick();
-  struct timespec due =
-      FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_timers.next)->due;
-  if (due.tv_sec < now.tv_sec ||
-      (due.tv_sec == now.tv_sec && due.tv_nsec <= now.tv_nsec))
-    return 0;
-  size_t interval = 1000L * (due.tv_sec - now.tv_sec);
-  if (due.tv_nsec >= now.tv_nsec) {
-    interval += (due.tv_nsec - now.tv_nsec) / 1000000L;
-  } else {
-    interval -= (now.tv_nsec - due.tv_nsec) / 1000000L;
-  }
-  if (interval > FIO_POLL_TICK)
-    interval = FIO_POLL_TICK;
-  return interval;
-}
-
-/* simple a<=>b if "a" is bigger a negative result is returned, eq == 0. */
-static int fio_timer_compare(struct timespec a, struct timespec b) {
-  if (a.tv_sec == b.tv_sec) {
-    if (a.tv_nsec < b.tv_nsec)
-      return 1;
-    if (a.tv_nsec > b.tv_nsec)
-      return -1;
-    return 0;
-  }
-  if (a.tv_sec < b.tv_sec)
-    return 1;
-  return -1;
-}
-
-/** Places a timer in an ordered linked list. */
-static void fio_timer_add_order(fio_timer_s *timer) {
-  timer->due = fio_timer_calc_due(timer->interval);
-  // fio_ls_embd_s *pos = &fio_timers;
-  fio_lock(&fio_timer_lock);
-  FIO_LS_EMBD_FOR(&fio_timers, node) {
-    fio_timer_s *t2 = FIO_LS_EMBD_OBJ(fio_timer_s, node, node);
-    if (fio_timer_compare(timer->due, t2->due) >= 0) {
-      fio_ls_embd_push(node, &timer->node);
-      goto finish;
-    }
-  }
-  fio_ls_embd_push(&fio_timers, &timer->node);
-finish:
-  fio_unlock(&fio_timer_lock);
-}
-
-/** Performs a timer task and re-adds it to the queue (or cleans it up) */
-static void fio_timer_perform_single(void *timer_, void *ignr) {
-  fio_timer_s *timer = timer_;
-  timer->task(timer->arg);
-  if (!timer->repetitions || fio_atomic_sub(&timer->repetitions, 1))
-    goto reschedule;
-  if (timer->on_finish)
-    timer->on_finish(timer->arg);
-  free(timer);
-  return;
-  (void)ignr;
-reschedule:
-  fio_timer_add_order(timer);
-}
-
-/** schedules all timers that are due to be performed. */
-static void fio_timer_schedule(void) {
-  struct timespec now = fio_last_tick();
-  fio_lock(&fio_timer_lock);
-  while (fio_ls_embd_any(&fio_timers) &&
-         fio_timer_compare(
-             FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_timers.next)->due, now) >=
-             0) {
-    fio_ls_embd_s *tmp = fio_ls_embd_remove(fio_timers.next);
-    fio_defer(fio_timer_perform_single, FIO_LS_EMBD_OBJ(fio_timer_s, node, tmp),
-              NULL);
-  }
-  fio_unlock(&fio_timer_lock);
-}
-
-static void fio_timer_clear_all(void) {
-  fio_lock(&fio_timer_lock);
-  while (fio_ls_embd_any(&fio_timers)) {
-    fio_timer_s *timer =
-        FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_ls_embd_pop(&fio_timers));
-    if (timer->on_finish)
-      timer->on_finish(timer->arg);
-    free(timer);
-  }
-  fio_unlock(&fio_timer_lock);
-}
-
-/**
- * Creates a timer to run a task at the specified interval.
- *
- * The task will repeat `repetitions` times. If `repetitions` is set to 0, task
- * will repeat forever.
- *
- * Returns -1 on error.
- *
- * The `on_finish` handler is always called (even on error).
- */
-int fio_run_every(size_t milliseconds, size_t repetitions, void (*task)(void *),
-                  void *arg, void (*on_finish)(void *)) {
-  if (!task || (milliseconds == 0 && !repetitions))
-    return -1;
-  fio_timer_s *timer = malloc(sizeof(*timer));
-  FIO_ASSERT_ALLOC(timer);
-  fio_mark_time();
-  *timer = (fio_timer_s){
-      .due = fio_timer_calc_due(milliseconds),
-      .interval = milliseconds,
-      .repetitions = repetitions,
-      .task = task,
-      .arg = arg,
-      .on_finish = on_finish,
-  };
-  fio_timer_add_order(timer);
-  return 0;
-}
-
-/* *****************************************************************************
-Section Start Marker
-
-
-
-
-
-
-
-
-
 
 
 
@@ -757,7 +576,6 @@ typedef struct {
 
 /* task queue block */
 typedef struct fio_defer_queue_block_s fio_defer_queue_block_s;
-
 struct fio_defer_queue_block_s {
   fio_defer_task_s tasks[DEFER_QUEUE_BLOCK_COUNT];
   fio_defer_queue_block_s *next;
@@ -766,18 +584,26 @@ struct fio_defer_queue_block_s {
   unsigned char state;
 };
 
-static fio_defer_queue_block_s fio_defer_static_queue;
-
-/* the state machine - this holds all the data about the task queue and pool */
-static struct {
-  /* a lock for the state machine, used for multi-threading support */
+/* task queue object */
+typedef struct { /* a lock for the state machine, used for multi-threading
+                    support */
   fio_lock_i lock;
   /* current active block to pop tasks */
   fio_defer_queue_block_s *reader;
   /* current active block to push tasks */
   fio_defer_queue_block_s *writer;
-} deferred = {.reader = &fio_defer_static_queue,
-              .writer = &fio_defer_static_queue};
+  /* static, built-in, queue */
+  fio_defer_queue_block_s static_queue;
+} fio_task_queue_s;
+
+/* the state machine - this holds all the data about the task queue and pool */
+static fio_task_queue_s task_queue_normal = {
+    .reader = &task_queue_normal.static_queue,
+    .writer = &task_queue_normal.static_queue};
+
+static fio_task_queue_s task_queue_urgent = {
+    .reader = &task_queue_urgent.static_queue,
+    .writer = &task_queue_urgent.static_queue};
 
 /* *****************************************************************************
 Internal Task API
@@ -797,85 +623,98 @@ static size_t fio_defer_count_alloc, fio_defer_count_dealloc;
 #define COUNT_RESET
 #endif
 
-static inline void fio_defer_push_task(fio_defer_task_s task) {
-  fio_lock(&deferred.lock);
+static inline void fio_defer_push_task_fn(fio_defer_task_s task,
+                                          fio_task_queue_s *queue) {
+  fio_lock(&queue->lock);
 
   /* test if full */
-  if (deferred.writer->state &&
-      deferred.writer->write == deferred.writer->read) {
+  if (queue->writer->state && queue->writer->write == queue->writer->read) {
     /* return to static buffer or allocate new buffer */
-    if (fio_defer_static_queue.state == 2) {
-      deferred.writer->next = &fio_defer_static_queue;
+    if (queue->static_queue.state == 2) {
+      queue->writer->next = &queue->static_queue;
     } else {
-      deferred.writer->next = fio_malloc(sizeof(*deferred.writer->next));
+      queue->writer->next = fio_malloc(sizeof(*queue->writer->next));
       COUNT_ALLOC;
-      if (!deferred.writer->next)
+      if (!queue->writer->next)
         goto critical_error;
     }
-    deferred.writer = deferred.writer->next;
-    deferred.writer->write = 0;
-    deferred.writer->read = 0;
-    deferred.writer->state = 0;
-    deferred.writer->next = NULL;
+    queue->writer = queue->writer->next;
+    queue->writer->write = 0;
+    queue->writer->read = 0;
+    queue->writer->state = 0;
+    queue->writer->next = NULL;
   }
 
   /* place task and finish */
-  deferred.writer->tasks[deferred.writer->write++] = task;
+  queue->writer->tasks[queue->writer->write++] = task;
   /* cycle buffer */
-  if (deferred.writer->write == DEFER_QUEUE_BLOCK_COUNT) {
-    deferred.writer->write = 0;
-    deferred.writer->state = 1;
+  if (queue->writer->write == DEFER_QUEUE_BLOCK_COUNT) {
+    queue->writer->write = 0;
+    queue->writer->state = 1;
   }
-  fio_unlock(&deferred.lock);
+  fio_unlock(&queue->lock);
   return;
 
 critical_error:
-  fio_unlock(&deferred.lock);
+  fio_unlock(&queue->lock);
   FIO_ASSERT_ALLOC(NULL)
 }
 
-static inline fio_defer_task_s fio_defer_pop_task(void) {
+#define fio_defer_push_task(func_, arg1_, arg2_)                               \
+  fio_defer_push_task_fn(                                                      \
+      (fio_defer_task_s){.func = func_, .arg1 = arg1_, .arg2 = arg2_},         \
+      &task_queue_normal)
+
+#if FIO_USE_URGENT_QUEUE
+#define fio_defer_push_urgent(func_, arg1_, arg2_)                             \
+  fio_defer_push_task_fn(                                                      \
+      (fio_defer_task_s){.func = func_, .arg1 = arg1_, .arg2 = arg2_},         \
+      &task_queue_urgent)
+#else
+#define fio_defer_push_urgent(func_, arg1_, arg2_)                             \
+  fio_defer_push_task(func_, arg1_, arg2_)
+#endif
+
+static inline fio_defer_task_s fio_defer_pop_task(fio_task_queue_s *queue) {
   fio_defer_task_s ret = (fio_defer_task_s){.func = NULL};
   fio_defer_queue_block_s *to_free = NULL;
   /* lock the state machine, grab/create a task and place it at the tail */
-  fio_lock(&deferred.lock);
+  fio_lock(&queue->lock);
 
   /* empty? */
-  if (deferred.reader->write == deferred.reader->read &&
-      !deferred.reader->state)
+  if (queue->reader->write == queue->reader->read && !queue->reader->state)
     goto finish;
   /* collect task */
-  ret = deferred.reader->tasks[deferred.reader->read++];
+  ret = queue->reader->tasks[queue->reader->read++];
   /* cycle */
-  if (deferred.reader->read == DEFER_QUEUE_BLOCK_COUNT) {
-    deferred.reader->read = 0;
-    deferred.reader->state = 0;
+  if (queue->reader->read == DEFER_QUEUE_BLOCK_COUNT) {
+    queue->reader->read = 0;
+    queue->reader->state = 0;
   }
   /* did we finish the queue in the buffer? */
-  if (deferred.reader->write == deferred.reader->read) {
-    if (deferred.reader->next) {
-      to_free = deferred.reader;
-      deferred.reader = deferred.reader->next;
+  if (queue->reader->write == queue->reader->read) {
+    if (queue->reader->next) {
+      to_free = queue->reader;
+      queue->reader = queue->reader->next;
     } else {
-      if (deferred.reader != &fio_defer_static_queue &&
-          fio_defer_static_queue.state == 2) {
-        to_free = deferred.reader;
-        deferred.writer = &fio_defer_static_queue;
-        deferred.reader = &fio_defer_static_queue;
+      if (queue->reader != &queue->static_queue &&
+          queue->static_queue.state == 2) {
+        to_free = queue->reader;
+        queue->writer = &queue->static_queue;
+        queue->reader = &queue->static_queue;
       }
-      deferred.reader->write = deferred.reader->read = deferred.reader->state =
-          0;
+      queue->reader->write = queue->reader->read = queue->reader->state = 0;
     }
   }
 
 finish:
-  if (to_free == &fio_defer_static_queue) {
-    fio_defer_static_queue.state = 2;
-    fio_defer_static_queue.next = NULL;
+  if (to_free == &queue->static_queue) {
+    queue->static_queue.state = 2;
+    queue->static_queue.next = NULL;
   }
-  fio_unlock(&deferred.lock);
+  fio_unlock(&queue->lock);
 
-  if (to_free && to_free != &fio_defer_static_queue) {
+  if (to_free && to_free != &queue->static_queue) {
     fio_free(to_free);
     COUNT_DEALLOC;
   }
@@ -883,25 +722,30 @@ finish:
 }
 
 /* same as fio_defer_clear_queue , just inlined */
-static inline void fio_defer_clear_tasks(void) {
-  fio_lock(&deferred.lock);
-  while (deferred.reader) {
-    fio_defer_queue_block_s *tmp = deferred.reader;
-    deferred.reader = deferred.reader->next;
-    if (tmp != &fio_defer_static_queue) {
+static inline void fio_defer_clear_tasks_for_queue(fio_task_queue_s *queue) {
+  fio_lock(&queue->lock);
+  while (queue->reader) {
+    fio_defer_queue_block_s *tmp = queue->reader;
+    queue->reader = queue->reader->next;
+    if (tmp != &queue->static_queue) {
       COUNT_DEALLOC;
       free(tmp);
     }
   }
-  fio_defer_static_queue = (fio_defer_queue_block_s){.next = NULL};
-  deferred.reader = deferred.writer = &fio_defer_static_queue;
-  fio_unlock(&deferred.lock);
+  queue->static_queue = (fio_defer_queue_block_s){.next = NULL};
+  queue->reader = queue->writer = &queue->static_queue;
+  fio_unlock(&queue->lock);
 }
 
-static void fio_defer_on_fork(void) { deferred.lock = FIO_LOCK_INIT; }
+static inline void fio_defer_clear_tasks(void) {
+  fio_defer_clear_tasks_for_queue(&task_queue_normal);
+  fio_defer_clear_tasks_for_queue(&task_queue_urgent);
+}
 
-#define fio_defer_push_task(...)                                               \
-  fio_defer_push_task((fio_defer_task_s){__VA_ARGS__})
+static void fio_defer_on_fork(void) {
+  task_queue_normal.lock = FIO_LOCK_INIT;
+  task_queue_urgent.lock = FIO_LOCK_INIT;
+}
 
 /* *****************************************************************************
 External Task API
@@ -912,7 +756,7 @@ int fio_defer(void (*func)(void *, void *), void *arg1, void *arg2) {
   /* must have a task to defer */
   if (!func)
     goto call_error;
-  fio_defer_push_task(.func = func, .arg1 = arg1, .arg2 = arg2);
+  fio_defer_push_task(func, arg1, arg2);
   return 0;
 
 call_error:
@@ -922,67 +766,261 @@ call_error:
 /** Performs all deferred functions until the queue had been depleted. */
 void fio_defer_perform(void) {
   for (;;) {
-    fio_defer_task_s task = fio_defer_pop_task();
+#if FIO_USE_URGENT_QUEUE
+    fio_defer_task_s task = fio_defer_pop_task(&task_queue_urgent);
+    if (!task.func)
+      task = fio_defer_pop_task(&task_queue_normal);
+#else
+    fio_defer_task_s task = fio_defer_pop_task(&task_queue_normal);
+#endif
     if (!task.func)
       return;
     task.func(task.arg1, task.arg2);
   }
 }
 
-/** Returns true if there are deferred functions waiting for execution. */
-int fio_defer_has_queue(void) {
-  return deferred.reader != deferred.writer ||
-         deferred.reader->write != deferred.reader->read;
+/** Returns true if there are deferred functions waiting for execution. */
+int fio_defer_has_queue(void) {
+  return task_queue_urgent.reader != task_queue_urgent.writer ||
+         task_queue_urgent.reader->write != task_queue_urgent.reader->read ||
+         task_queue_normal.reader != task_queue_normal.writer ||
+         task_queue_normal.reader->write != task_queue_normal.reader->read;
+}
+
+/** Clears the queue. */
+void fio_defer_clear_queue(void) { fio_defer_clear_tasks(); }
+
+static void fio_defer_thread_wait(void);
+static void *fio_defer_cycle(void *ignr) {
+  do {
+    fio_defer_perform();
+    fio_defer_thread_wait();
+  } while (fio_is_running());
+  return ignr;
+}
+
+/* thread pool type */
+typedef struct {
+  size_t thread_count;
+  void *threads[];
+} fio_defer_thread_pool_s;
+
+/* joins a thread pool */
+static void fio_defer_thread_pool_join(fio_defer_thread_pool_s *pool) {
+  for (size_t i = 0; i < pool->thread_count; ++i) {
+    fio_thread_join(pool->threads[i]);
+  }
+  free(pool);
+}
+
+/* creates a thread pool */
+static fio_defer_thread_pool_s *fio_defer_thread_pool_new(size_t count) {
+  if (!count)
+    count = 1;
+  fio_defer_thread_pool_s *pool =
+      malloc(sizeof(*pool) + (count * sizeof(void *)));
+  FIO_ASSERT_ALLOC(pool);
+  pool->thread_count = count;
+  for (size_t i = 0; i < count; ++i) {
+    pool->threads[i] = fio_thread_new(fio_defer_cycle, NULL);
+    if (!pool->threads[i]) {
+      pool->thread_count = i;
+      goto error;
+    }
+  }
+  return pool;
+error:
+  FIO_LOG_FATAL("couldn't spawn threads for thread pool, attempting shutdown.");
+  fio_stop();
+  fio_defer_thread_pool_join(pool);
+  return NULL;
+}
+
+/* *****************************************************************************
+Section Start Marker
+
+
+
+
+
+
+
+
+
+                                     Timers
+
+
+
+
+
+
+
+
+
+
+***************************************************************************** */
+
+typedef struct {
+  fio_ls_embd_s node;
+  struct timespec due;
+  size_t interval; /*in ms */
+  size_t repetitions;
+  void (*task)(void *);
+  void *arg;
+  void (*on_finish)(void *);
+} fio_timer_s;
+
+static fio_ls_embd_s fio_timers = FIO_LS_INIT(fio_timers);
+
+static fio_lock_i fio_timer_lock = FIO_LOCK_INIT;
+
+/** Marks the current time as facil.io's cycle time */
+static inline void fio_mark_time(void) {
+  clock_gettime(CLOCK_REALTIME, &fio_data->last_cycle);
+}
+
+/** Calculates the due time for a task, given it's interval */
+static struct timespec fio_timer_calc_due(size_t interval) {
+  struct timespec now = fio_last_tick();
+  if (interval > 1000) {
+    now.tv_sec += interval / 1000;
+    interval -= interval / 1000;
+  }
+  now.tv_nsec += (interval * 1000000UL);
+  if (now.tv_nsec > 1000000000L) {
+    now.tv_nsec -= 1000000000L;
+    now.tv_sec += 1;
+  }
+  return now;
+}
+
+/** Returns the number of miliseconds until the next event, up to FIO_POLL_TICK
+ */
+static size_t fio_timer_calc_first_interval(void) {
+  if (fio_defer_has_queue())
+    return 0;
+  if (fio_ls_embd_is_empty(&fio_timers)) {
+    return FIO_POLL_TICK;
+  }
+  struct timespec now = fio_last_tick();
+  struct timespec due =
+      FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_timers.next)->due;
+  if (due.tv_sec < now.tv_sec ||
+      (due.tv_sec == now.tv_sec && due.tv_nsec <= now.tv_nsec))
+    return 0;
+  size_t interval = 1000L * (due.tv_sec - now.tv_sec);
+  if (due.tv_nsec >= now.tv_nsec) {
+    interval += (due.tv_nsec - now.tv_nsec) / 1000000L;
+  } else {
+    interval -= (now.tv_nsec - due.tv_nsec) / 1000000L;
+  }
+  if (interval > FIO_POLL_TICK)
+    interval = FIO_POLL_TICK;
+  return interval;
+}
+
+/* simple a<=>b if "a" is bigger a negative result is returned, eq == 0. */
+static int fio_timer_compare(struct timespec a, struct timespec b) {
+  if (a.tv_sec == b.tv_sec) {
+    if (a.tv_nsec < b.tv_nsec)
+      return 1;
+    if (a.tv_nsec > b.tv_nsec)
+      return -1;
+    return 0;
+  }
+  if (a.tv_sec < b.tv_sec)
+    return 1;
+  return -1;
+}
+
+/** Places a timer in an ordered linked list. */
+static void fio_timer_add_order(fio_timer_s *timer) {
+  timer->due = fio_timer_calc_due(timer->interval);
+  // fio_ls_embd_s *pos = &fio_timers;
+  fio_lock(&fio_timer_lock);
+  FIO_LS_EMBD_FOR(&fio_timers, node) {
+    fio_timer_s *t2 = FIO_LS_EMBD_OBJ(fio_timer_s, node, node);
+    if (fio_timer_compare(timer->due, t2->due) >= 0) {
+      fio_ls_embd_push(node, &timer->node);
+      goto finish;
+    }
+  }
+  fio_ls_embd_push(&fio_timers, &timer->node);
+finish:
+  fio_unlock(&fio_timer_lock);
+}
+
+/** Performs a timer task and re-adds it to the queue (or cleans it up) */
+static void fio_timer_perform_single(void *timer_, void *ignr) {
+  fio_timer_s *timer = timer_;
+  timer->task(timer->arg);
+  if (!timer->repetitions || fio_atomic_sub(&timer->repetitions, 1))
+    goto reschedule;
+  if (timer->on_finish)
+    timer->on_finish(timer->arg);
+  free(timer);
+  return;
+  (void)ignr;
+reschedule:
+  fio_timer_add_order(timer);
 }
 
-/** Clears the queue. */
-void fio_defer_clear_queue(void) { fio_defer_clear_tasks(); }
-
-static void fio_defer_thread_wait(void);
-static void *fio_defer_cycle(void *ignr) {
-  do {
-    fio_defer_perform();
-    fio_defer_thread_wait();
-  } while (fio_is_running());
-  return ignr;
+/** schedules all timers that are due to be performed. */
+static void fio_timer_schedule(void) {
+  struct timespec now = fio_last_tick();
+  fio_lock(&fio_timer_lock);
+  while (fio_ls_embd_any(&fio_timers) &&
+         fio_timer_compare(
+             FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_timers.next)->due, now) >=
+             0) {
+    fio_ls_embd_s *tmp = fio_ls_embd_remove(fio_timers.next);
+    fio_defer(fio_timer_perform_single, FIO_LS_EMBD_OBJ(fio_timer_s, node, tmp),
+              NULL);
+  }
+  fio_unlock(&fio_timer_lock);
 }
 
-/* thread pool type */
-typedef struct {
-  size_t thread_count;
-  void *threads[];
-} fio_defer_thread_pool_s;
-
-/* joins a thread pool */
-static void fio_defer_thread_pool_join(fio_defer_thread_pool_s *pool) {
-  for (size_t i = 0; i < pool->thread_count; ++i) {
-    fio_thread_join(pool->threads[i]);
+static void fio_timer_clear_all(void) {
+  fio_lock(&fio_timer_lock);
+  while (fio_ls_embd_any(&fio_timers)) {
+    fio_timer_s *timer =
+        FIO_LS_EMBD_OBJ(fio_timer_s, node, fio_ls_embd_pop(&fio_timers));
+    if (timer->on_finish)
+      timer->on_finish(timer->arg);
+    free(timer);
   }
-  free(pool);
+  fio_unlock(&fio_timer_lock);
 }
 
-/* creates a thread pool */
-static fio_defer_thread_pool_s *fio_defer_thread_pool_new(size_t count) {
-  if (!count)
-    count = 1;
-  fio_defer_thread_pool_s *pool =
-      malloc(sizeof(*pool) + (count * sizeof(void *)));
-  FIO_ASSERT_ALLOC(pool);
-  pool->thread_count = count;
-  for (size_t i = 0; i < count; ++i) {
-    pool->threads[i] = fio_thread_new(fio_defer_cycle, NULL);
-    if (!pool->threads[i]) {
-      pool->thread_count = i;
-      goto error;
-    }
-  }
-  return pool;
-error:
-  FIO_LOG_FATAL("couldn't spawn threads for thread pool, attempting shutdown.");
-  fio_stop();
-  fio_defer_thread_pool_join(pool);
-  return NULL;
+/**
+ * Creates a timer to run a task at the specified interval.
+ *
+ * The task will repeat `repetitions` times. If `repetitions` is set to 0, task
+ * will repeat forever.
+ *
+ * Returns -1 on error.
+ *
+ * The `on_finish` handler is always called (even on error).
+ */
+int fio_run_every(size_t milliseconds, size_t repetitions, void (*task)(void *),
+                  void *arg, void (*on_finish)(void *)) {
+  if (!task || (milliseconds == 0 && !repetitions))
+    return -1;
+  fio_timer_s *timer = malloc(sizeof(*timer));
+  FIO_ASSERT_ALLOC(timer);
+  fio_mark_time();
+  *timer = (fio_timer_s){
+      .due = fio_timer_calc_due(milliseconds),
+      .interval = milliseconds,
+      .repetitions = repetitions,
+      .task = task,
+      .arg = arg,
+      .on_finish = on_finish,
+  };
+  fio_timer_add_order(timer);
+  return 0;
 }
+
 /* *****************************************************************************
 Section Start Marker
 
@@ -1451,12 +1489,12 @@ static size_t fio_poll(void) {
         } else {
           // no error, then it's an active event(s)
           if (events[i].events & EPOLLOUT) {
-            fio_defer(deferred_on_ready, (void *)fd2uuid(events[i].data.fd),
-                      NULL);
+            fio_defer_push_urgent(deferred_on_ready,
+                                  (void *)fd2uuid(events[i].data.fd), NULL);
           }
           if (events[i].events & EPOLLIN)
-            fio_defer(deferred_on_data, (void *)fd2uuid(events[i].data.fd),
-                      NULL);
+            fio_defer_push_task(deferred_on_data,
+                                (void *)fd2uuid(events[i].data.fd), NULL);
         }
       } // end for loop
       total += active_count;
@@ -1584,7 +1622,8 @@ static size_t fio_poll(void) {
     for (int i = 0; i < active_count; i++) {
       // test for event(s) type
       if (events[i].filter == EVFILT_READ) {
-        fio_defer(deferred_on_data, (void *)fd2uuid(events[i].udata), NULL);
+        fio_defer_push_task(deferred_on_data, (void *)fd2uuid(events[i].udata),
+                            NULL);
       }
       // connection errors should be reported after `read` in case there's data
       // left in the buffer... not that the edge case matters.
@@ -1598,7 +1637,8 @@ static size_t fio_poll(void) {
         fio_force_close(fd2uuid(events[i].udata));
       } else if (events[i].filter == EVFILT_WRITE) {
         // we can only write if there's no error in the socket
-        fio_defer(deferred_on_ready, (void *)fd2uuid(events[i].udata), NULL);
+        fio_defer_push_urgent(deferred_on_ready,
+                              ((void *)fd2uuid(events[i].udata)), NULL);
       }
     }
   } else if (active_count < 0) {
@@ -1732,12 +1772,12 @@ static size_t fio_poll(void) {
       if (list[i].revents & FIO_POLL_READ_EVENTS) {
         // FIO_LOG_DEBUG("Poll Read %zu => %p", i, (void *)fd2uuid(i));
         fio_poll_remove_read(i);
-        fio_defer(deferred_on_data, (void *)fd2uuid(i), NULL);
+        fio_defer_push_task(deferred_on_data, (void *)fd2uuid(i), NULL);
       }
       if (list[i].revents & FIO_POLL_WRITE_EVENTS) {
         // FIO_LOG_DEBUG("Poll Write %zu => %p", i, (void *)fd2uuid(i));
         fio_poll_remove_write(i);
-        fio_defer(deferred_on_ready, (void *)fd2uuid(i), NULL);
+        fio_defer_push_urgent(deferred_on_ready, (void *)fd2uuid(i), NULL);
       }
       if (list[i].revents & (POLLHUP | POLLERR)) {
         // FIO_LOG_DEBUG("Poll Hangup %zu => %p", i, (void *)fd2uuid(i));
@@ -1847,7 +1887,7 @@ static void deferred_on_close(void *uuid_, void *pr_) {
   pr->on_close((intptr_t)uuid_, pr);
   return;
 postpone:
-  fio_defer(deferred_on_close, uuid_, pr_);
+  fio_defer_push_task(deferred_on_close, uuid_, pr_);
 }
 
 static void deferred_on_shutdown(void *arg, void *arg2) {
@@ -1880,19 +1920,12 @@ static void deferred_on_shutdown(void *arg, void *arg2) {
   }
   return;
 postpone:
-  fio_defer(deferred_on_shutdown, arg, NULL);
+  fio_defer_push_task(deferred_on_shutdown, arg, NULL);
   (void)arg2;
 }
 
-static void deferred_on_ready(void *arg, void *arg2) {
+static void deferred_on_ready_usr(void *arg, void *arg2) {
   errno = 0;
-  if (fio_flush((intptr_t)arg) > 0 || errno == EWOULDBLOCK) {
-    fio_poll_add_write(fio_uuid2fd(arg));
-    return;
-  }
-  if (!uuid_data(arg).protocol) {
-    return;
-  }
   fio_protocol_s *pr = protocol_try_lock(fio_uuid2fd(arg), FIO_PR_LOCK_WRITE);
   if (!pr) {
     if (errno == EBADF)
@@ -1903,7 +1936,20 @@ static void deferred_on_ready(void *arg, void *arg2) {
   protocol_unlock(pr, FIO_PR_LOCK_WRITE);
   return;
 postpone:
-  fio_defer(deferred_on_ready, arg, NULL);
+  fio_defer_push_task(deferred_on_ready, arg, NULL);
+  (void)arg2;
+}
+
+static void deferred_on_ready(void *arg, void *arg2) {
+  errno = 0;
+  if (fio_flush((intptr_t)arg) > 0 || errno == EWOULDBLOCK) {
+    fio_poll_add_write(fio_uuid2fd(arg));
+    return;
+  }
+  if (!uuid_data(arg).protocol) {
+    return;
+  }
+  fio_defer_push_task(deferred_on_ready_usr, arg, NULL);
   (void)arg2;
 }
 
@@ -1928,7 +1974,7 @@ static void deferred_on_data(void *uuid, void *arg2) {
 postpone:
   if (arg2) {
     /* the event is being forced, so force rescheduling */
-    fio_defer(deferred_on_data, (void *)uuid, (void *)1);
+    fio_defer_push_task(deferred_on_data, (void *)uuid, (void *)1);
   } else {
     /* the protocol was locked, so there might not be any need for the event */
     fio_poll_add_read(fio_uuid2fd((intptr_t)uuid));
@@ -1950,7 +1996,7 @@ static void deferred_ping(void *arg, void *arg2) {
   protocol_unlock(pr, FIO_PR_LOCK_WRITE);
   return;
 postpone:
-  fio_defer(deferred_ping, arg, NULL);
+  fio_defer_push_task(deferred_ping, arg, NULL);
   (void)arg2;
 }
 
@@ -1962,13 +2008,13 @@ void fio_force_event(intptr_t uuid, enum fio_io_event ev) {
   switch (ev) {
   case FIO_EVENT_ON_DATA:
     fio_trylock(&uuid_data(uuid).scheduled);
-    fio_defer(deferred_on_data, (void *)uuid, (void *)1);
+    fio_defer_push_task(deferred_on_data, (void *)uuid, (void *)1);
     break;
   case FIO_EVENT_ON_TIMEOUT:
-    fio_defer(deferred_ping, (void *)uuid, NULL);
+    fio_defer_push_task(deferred_ping, (void *)uuid, NULL);
     break;
   case FIO_EVENT_ON_READY:
-    fio_defer(deferred_on_ready, (void *)uuid, NULL);
+    fio_defer_push_urgent(deferred_on_ready, (void *)uuid, NULL);
     break;
   }
 }
@@ -2501,10 +2547,13 @@ ssize_t fio_write2_fn(intptr_t uuid, fio_write_args_s options) {
     packet->dealloc = (options.after.dealloc ? options.after.dealloc : free);
   }
   /* add packet to outgoing list */
+  uint8_t was_empty = 1;
   fio_lock(&uuid_data(uuid).sock_lock);
   if (!uuid_is_valid(uuid)) {
     goto locked_error;
   }
+  if (uuid_data(uuid).packet)
+    was_empty = 0;
   if (options.urgent == 0) {
     *uuid_data(uuid).packet_last = packet;
     uuid_data(uuid).packet_last = &packet->next;
@@ -2518,10 +2567,12 @@ ssize_t fio_write2_fn(intptr_t uuid, fio_write_args_s options) {
       uuid_data(uuid).packet_last = &packet->next;
     }
   }
+  fio_atomic_add(&uuid_data(uuid).packet_count, 1);
   fio_unlock(&uuid_data(uuid).sock_lock);
 
-  fio_atomic_add(&uuid_data(uuid).packet_count, 1);
-  fio_poll_add_write(fio_uuid2fd(uuid));
+  if (was_empty) {
+    fio_defer_push_urgent(deferred_on_ready, (void *)uuid, NULL);
+  }
   return 0;
 locked_error:
   fio_unlock(&uuid_data(uuid).sock_lock);
@@ -2835,7 +2886,7 @@ static int fio_attach__internal(void *uuid_, void *protocol_) {
   uuid_data(uuid).active = fio_data->last_cycle.tv_sec;
   fio_unlock(&uuid_data(uuid).protocol_lock);
   if (old_pr) {
-    fio_defer(deferred_on_close, (void *)uuid, old_pr);
+    fio_defer_push_task(deferred_on_close, (void *)uuid, old_pr);
   } else if (protocol) {
     fio_poll_add(fio_uuid2fd(uuid));
   }
@@ -2844,7 +2895,7 @@ static int fio_attach__internal(void *uuid_, void *protocol_) {
 invalid_uuid:
   fio_unlock(&uuid_data(uuid).protocol_lock);
   if (protocol)
-    fio_defer(deferred_on_close, (void *)uuid, protocol);
+    fio_defer_push_task(deferred_on_close, (void *)uuid, protocol);
   if (uuid == -1)
     errno = EBADF;
   else
@@ -2852,7 +2903,8 @@ invalid_uuid:
   return -1;
 }
 
-/** Attaches (or updates) a protocol object to a socket UUID.
+/**
+ * Attaches (or updates) a protocol object to a socket UUID.
  * Returns -1 on error and 0 on success.
  */
 void fio_attach(intptr_t uuid, fio_protocol_s *protocol) {
@@ -2971,8 +3023,8 @@ void fio_state_callback_force(callback_type_e c_type) {
   case FIO_CALL_ON_IDLE: /* idle callbacks are orderless and evented */
     FIO_LS_EMBD_FOR(&callback_collection[c_type].callbacks, pos) {
       callback_data_s *tmp = FIO_LS_EMBD_OBJ(callback_data_s, node, pos);
-      fio_defer(fio_state_on_idle_perform, (void *)(uintptr_t)tmp->func,
-                tmp->arg);
+      fio_defer_push_task(fio_state_on_idle_perform,
+                          (void *)(uintptr_t)tmp->func, tmp->arg);
     }
     break;
 
@@ -3058,7 +3110,7 @@ postpone:
     fio_free(args);
     return;
   }
-  fio_defer(fio_io_task_perform, uuid_, args_);
+  fio_defer_push_task(fio_io_task_perform, uuid_, args_);
 }
 /**
  * Schedules a protected connection task. The task will run within the
@@ -3071,14 +3123,14 @@ void fio_defer_io_task FIO_IGNORE_MACRO(intptr_t uuid,
                                         fio_defer_iotask_args_s args) {
   if (!args.task) {
     if (args.fallback)
-      fio_defer((void (*)(void *, void *))args.fallback, (void *)uuid,
-                args.udata);
+      fio_defer_push_task((void (*)(void *, void *))args.fallback, (void *)uuid,
+                          args.udata);
     return;
   }
   fio_defer_iotask_args_s *cpy = fio_malloc(sizeof(*cpy));
   FIO_ASSERT_ALLOC(cpy);
   *cpy = args;
-  fio_defer(fio_io_task_perform, (void *)uuid, cpy);
+  fio_defer_push_task(fio_io_task_perform, (void *)uuid, cpy);
 }
 
 /* *****************************************************************************
@@ -3264,12 +3316,15 @@ static void fio_review_timeout(void *arg, void *ignr) {
   if (!fd_data(fd).protocol || (fd_data(fd).active + timeout >= review))
     goto finish;
   tmp = protocol_try_lock(fd, FIO_PR_LOCK_STATE);
-  if (!tmp)
+  if (!tmp) {
+    if (errno == EBADF)
+      goto finish;
     goto reschedule;
+  }
   if (prt_meta(tmp).locks[FIO_PR_LOCK_TASK] ||
       prt_meta(tmp).locks[FIO_PR_LOCK_WRITE])
     goto unlock;
-  fio_defer(deferred_ping, (void *)fio_fd2uuid((int)fd), NULL);
+  fio_defer_push_task(deferred_ping, (void *)fio_fd2uuid((int)fd), NULL);
 unlock:
   protocol_unlock(tmp, FIO_PR_LOCK_STATE);
 finish:
@@ -3282,7 +3337,7 @@ finish:
     return;
   }
 reschedule:
-  fio_defer(fio_review_timeout, (void *)fd, NULL);
+  fio_defer_push_task(fio_review_timeout, (void *)fd, NULL);
 }
 
 /* reactor pattern cycling - common actions */
@@ -3312,7 +3367,7 @@ static void fio_cycle_schedule_events(void) {
   if (fio_data->need_review && fio_data->last_cycle.tv_sec != last_to_review) {
     last_to_review = fio_data->last_cycle.tv_sec;
     fio_data->need_review = 0;
-    fio_defer(fio_review_timeout, (void *)0, NULL);
+    fio_defer_push_task(fio_review_timeout, (void *)0, NULL);
   }
 }
 
@@ -3320,7 +3375,7 @@ static void fio_cycle_schedule_events(void) {
 static void fio_cycle_unwind(void *ignr, void *ignr2) {
   if (fio_data->connection_count) {
     fio_cycle_schedule_events();
-    fio_defer(fio_cycle_unwind, ignr, ignr2);
+    fio_defer_push_task(fio_cycle_unwind, ignr, ignr2);
     return;
   }
   fio_stop();
@@ -3331,7 +3386,7 @@ static void fio_cycle_unwind(void *ignr, void *ignr2) {
 static void fio_cycle(void *ignr, void *ignr2) {
   fio_cycle_schedule_events();
   if (fio_data->active) {
-    fio_defer(fio_cycle, ignr, ignr2);
+    fio_defer_push_task(fio_cycle, ignr, ignr2);
     return;
   }
   return;
@@ -3360,7 +3415,7 @@ static void fio_worker_startup(void) {
   fio_data->need_review = 1;
 
   /* the cycle task will loop by re-scheduling until it's time to finish */
-  fio_defer(fio_cycle, NULL, NULL);
+  fio_defer_push_task(fio_cycle, NULL, NULL);
 
   /* A single thread doesn't need a pool. */
   if (fio_data->threads > 1) {
@@ -3380,10 +3435,10 @@ static void fio_worker_cleanup(void) {
   fio_state_callback_force(FIO_CALL_ON_SHUTDOWN);
   for (size_t i = 0; i <= fio_data->max_protocol_fd; ++i) {
     if (fd_data(i).protocol) {
-      fio_defer(deferred_on_shutdown, (void *)fd2uuid(i), NULL);
+      fio_defer_push_task(deferred_on_shutdown, (void *)fd2uuid(i), NULL);
     }
   }
-  fio_defer(fio_cycle_unwind, NULL, NULL);
+  fio_defer_push_task(fio_cycle_unwind, NULL, NULL);
   fio_defer_perform();
   for (size_t i = 0; i <= fio_data->max_protocol_fd; ++i) {
     if (fd_data(i).protocol || fd_data(i).open) {
@@ -3443,7 +3498,7 @@ static void *fio_sentinel_worker_thread(void *arg) {
         FIO_LOG_WARNING("Child worker (%d) shutdown. Respawning worker.",
                         child);
       }
-      fio_defer(fio_sentinel_task, NULL, NULL);
+      fio_defer_push_task(fio_sentinel_task, NULL, NULL);
       fio_unlock(&fio_fork_lock);
     }
 #endif
@@ -4246,17 +4301,17 @@ typedef enum fio_cluster_message_type_e {
 
 typedef struct fio_collection_s fio_collection_s;
 
+#pragma pack(1)
 typedef struct {
-  fio_str_s id; /* MUST be on top, so the *channel == *string */
+  size_t name_len;
+  char *name;
+  size_t ref;
   fio_ls_embd_s subscriptions;
   fio_collection_s *parent;
+  fio_match_fn match;
   fio_lock_i lock;
 } channel_s;
-
-typedef struct {
-  channel_s ch; /* MUST be on top, so *channel == *pattern */
-  fio_match_fn match;
-} pattern_s;
+#pragma pack()
 
 struct subscription_s {
   fio_ls_embd_s node;
@@ -4272,15 +4327,58 @@ struct subscription_s {
   fio_lock_i unsubscribed;
 };
 
+/* Use `malloc` / `free`, because channles might have a long life. */
+
+/** Used internally by the Set object to create a new channel. */
+static channel_s *fio_channel_copy(channel_s *src) {
+  channel_s *dest = malloc(sizeof(*dest) + src->name_len + 1);
+  FIO_ASSERT_ALLOC(dest);
+  dest->name_len = src->name_len;
+  dest->match = src->match;
+  dest->parent = src->parent;
+  dest->name = (char *)(dest + 1);
+  if (src->name_len)
+    memcpy(dest->name, src->name, src->name_len);
+  dest->name[src->name_len] = 0;
+  dest->subscriptions = (fio_ls_embd_s)FIO_LS_INIT(dest->subscriptions);
+  dest->ref = 1;
+  dest->lock = FIO_LOCK_INIT;
+  return dest;
+}
+/** Frees a channel (reference counting). */
+static void fio_channel_free(channel_s *ch) {
+  if (!ch)
+    return;
+  if (fio_atomic_sub(&ch->ref, 1))
+    return;
+  free(ch);
+}
+/** Increases a channel's reference count. */
+static void fio_channel_dup(channel_s *ch) {
+  if (!ch)
+    return;
+  fio_atomic_add(&ch->ref, 1);
+}
+/** Tests if two channels are equal. */
+static int fio_channel_cmp(channel_s *ch1, channel_s *ch2) {
+  return ch1->name_len == ch2->name_len && ch1->match == ch2->match &&
+         !memcmp(ch1->name, ch2->name, ch1->name_len);
+}
+/* pub/sub channels and core data sets have a long life, so avoid fio_malloc */
+#if !FIO_FORCE_MALLOC
+#define FIO_FORCE_MALLOC 1
+#define FIO_FORCE_MALLOC_IS_TMP 1
+#endif
+
 #define FIO_SET_NAME fio_ch_set
 #define FIO_SET_OBJ_TYPE channel_s *
-#define FIO_SET_OBJ_COMPARE(o1, o2) fio_str_iseq(&(o1)->id, &(o2)->id)
-#define FIO_SET_OBJ_DESTROY(obj) fio_str_free2(&(obj)->id)
+#define FIO_SET_OBJ_COMPARE(o1, o2) fio_channel_cmp((o1), (o2))
+#define FIO_SET_OBJ_DESTROY(obj) fio_channel_free((obj))
+#define FIO_SET_OBJ_COPY(dest, src) ((dest) = fio_channel_copy((src)))
 #include <fio.h>
 
-#define FIO_SET_NAME fio_meta_set
-#define FIO_SET_OBJ_TYPE fio_msg_metadata_fn
-#define FIO_SET_OBJ_COMPARE(k1, k2) ((k1) == (k2))
+#define FIO_ARY_NAME fio_meta_ary
+#define FIO_ARY_TYPE fio_msg_metadata_fn
 #include <fio.h>
 
 #define FIO_SET_NAME fio_engine_set
@@ -4288,6 +4386,10 @@ struct subscription_s {
 #define FIO_SET_OBJ_COMPARE(k1, k2) ((k1) == (k2))
 #include <fio.h>
 
+#if FIO_FORCE_MALLOC_IS_TMP
+#undef FIO_FORCE_MALLOC
+#endif
+
 struct fio_collection_s {
   fio_ch_set_s channels;
   fio_lock_i lock;
@@ -4305,7 +4407,7 @@ static struct {
     fio_lock_i lock;
   } engines;
   struct {
-    fio_meta_set_s set;
+    fio_meta_ary_s ary;
     fio_lock_i lock;
   } meta;
 } fio_postoffice = {
@@ -4319,23 +4421,169 @@ static struct {
 /** used to contain the message before it's passed to the handler */
 typedef struct {
   fio_msg_s msg;
-  fio_msg_metadata_s *meta;
   size_t marker;
+  size_t meta_len;
+  fio_msg_metadata_s *meta;
 } fio_msg_client_s;
 
 /** used to contain the message internally while publishing */
 typedef struct {
-  fio_msg_metadata_s *meta;
   fio_str_info_s channel;
   fio_str_info_s data;
   uintptr_t ref; /* internal reference counter */
   int32_t filter;
   int8_t is_json;
+  size_t meta_len;
+  fio_msg_metadata_s meta[];
 } fio_msg_internal_s;
 
 /** The default engine (settable). */
 fio_pubsub_engine_s *FIO_PUBSUB_DEFAULT = FIO_PUBSUB_CLUSTER;
 
+/* *****************************************************************************
+Internal message object creation
+***************************************************************************** */
+
+#if 1 /* Copy Meta-Data Array vs. lock and unlock per callback */
+
+/** returns a temporary fio_meta_ary_s with a copy of the metadata array */
+static fio_meta_ary_s fio_postoffice_meta_copy_new(void) {
+  fio_meta_ary_s t = FIO_ARY_INIT;
+  if (!fio_meta_ary_count(&fio_postoffice.meta.ary)) {
+    return t;
+  }
+  fio_lock(&fio_postoffice.meta.lock);
+  size_t len = fio_meta_ary_count(&fio_postoffice.meta.ary);
+  if (len) {
+    t.end = t.capa = len;
+    t.arry = fio_malloc(sizeof(*t.arry) * len);
+    FIO_ASSERT_ALLOC(t.arry);
+    memcpy(t.arry, fio_meta_ary_to_a(&fio_postoffice.meta.ary),
+           sizeof(*t.arry) * len);
+  }
+  fio_unlock(&fio_postoffice.meta.lock);
+  return t;
+}
+
+/** frees a temporary copy created by postoffice_meta_copy_new */
+static void fio_postoffice_meta_copy_free(fio_meta_ary_s cpy) {
+  fio_free(cpy.arry);
+}
+
+static fio_msg_internal_s *
+fio_pubsub_create_message(int32_t filter, fio_str_info_s ch,
+                          fio_str_info_s data, int8_t is_json, int8_t cpy) {
+  fio_meta_ary_s t = FIO_ARY_INIT;
+  if (!filter)
+    t = fio_postoffice_meta_copy_new();
+  fio_msg_internal_s *m = fio_malloc(sizeof(*m) + (sizeof(*m->meta) * t.end) +
+                                     (ch.len + 1) + (data.len + 1));
+  *m = (fio_msg_internal_s){
+      .filter = filter,
+      .channel =
+          (fio_str_info_s){.data = (char *)(m->meta + t.end), .len = ch.len},
+      .data = (fio_str_info_s){.data = ((char *)(m->meta + t.end) + ch.len + 1),
+                               .len = data.len},
+      .is_json = is_json,
+      .ref = 1,
+      .meta_len = t.end,
+  };
+  // m->channel.data[ch.len] = 0; /* redundant, fio_malloc is all zero */
+  // m->data.data[data.len] = 0; /* redundant, fio_malloc is all zero */
+  if (cpy) {
+    memcpy(m->channel.data, ch.data, ch.len);
+    memcpy(m->data.data, data.data, data.len);
+    while (t.end) {
+      --t.end;
+      m->meta[t.end] = t.arry[t.end](m->channel, m->data, is_json);
+    }
+  }
+  fio_postoffice_meta_copy_free(t);
+  return m;
+}
+
+static void fio_pubsub_create_message_update_meta(fio_msg_internal_s *m) {
+  if (m->filter || !m->meta_len)
+    return;
+  fio_meta_ary_s t = fio_postoffice_meta_copy_new();
+  if (t.end > m->meta_len)
+    t.end = m->meta_len;
+  m->meta_len = t.end;
+  while (t.end) {
+    --t.end;
+    m->meta[t.end] = t.arry[t.end](m->channel, m->data, m->is_json);
+  }
+  fio_postoffice_meta_copy_free(t);
+}
+
+#else
+
+/** returns the pub/sub metadata count safely (locks) */
+static size_t fio_postoffice_meta_count(void) {
+  size_t count;
+  fio_lock(&fio_postoffice.meta.lock);
+  count = fio_meta_ary_count(&fio_postoffice.meta.ary);
+  fio_unlock(&fio_postoffice.meta.lock);
+  return count;
+}
+/** collects a callback by index, from within a loop */
+static fio_msg_metadata_fn fio_postoffice_meta_index(intptr_t index) {
+  fio_msg_metadata_fn cb;
+  fio_lock(&fio_postoffice.meta.lock);
+  cb = fio_meta_ary_get(&fio_postoffice.meta.ary, index);
+  fio_unlock(&fio_postoffice.meta.lock);
+  return cb;
+}
+
+static fio_msg_internal_s *
+fio_pubsub_create_message(int32_t filter, fio_str_info_s ch,
+                          fio_str_info_s data, int8_t is_json, int8_t cpy) {
+  size_t meta_len = 0;
+  if (!filter)
+    meta_len = fio_postoffice_meta_count();
+  fio_msg_internal_s *m =
+      fio_malloc(sizeof(*m) + (sizeof(*m->meta) * meta_len) + (ch.len + 1) +
+                 (data.len + 1));
+  *m = (fio_msg_internal_s){
+      .filter = filter,
+      .channel =
+          (fio_str_info_s){.data = (char *)(m->meta + meta_len), .len = ch.len},
+      .data =
+          (fio_str_info_s){.data = ((char *)(m->meta + meta_len) + ch.len + 1),
+                           .len = data.len},
+      .is_json = is_json,
+      .ref = 1,
+      .meta_len = meta_len,
+  };
+  // m->channel.data[ch.len] = 0; /* redundant, fio_malloc is all zero */
+  // m->data.data[data.len] = 0; /* redundant, fio_malloc is all zero */
+  if (cpy) {
+    memcpy(m->channel.data, ch.data, ch.len);
+    memcpy(m->data.data, data.data, data.len);
+    while (meta_len) {
+      --meta_len;
+      fio_msg_metadata_fn cb = fio_postoffice_meta_index(meta_len);
+      if (cb)
+        m->meta[meta_len] = cb(m->channel, m->data, is_json);
+    }
+  }
+  return m;
+}
+
+static void fio_pubsub_create_message_update_meta(fio_msg_internal_s *m) {
+  if (m->filter || !m->meta_len)
+    return;
+  size_t meta_len = m->meta_len;
+  while (meta_len) {
+    --meta_len;
+    fio_msg_metadata_fn cb = fio_postoffice_meta_index(meta_len);
+    if (cb)
+      m->meta[meta_len] = cb(m->channel, m->data, m->is_json);
+  }
+}
+
+#endif
+
 /* *****************************************************************************
 Cluster forking handler
 ***************************************************************************** */
@@ -4376,8 +4624,8 @@ static void fio_pubsub_on_fork(void) {
 Channel Subscription Management
 ***************************************************************************** */
 
-static void pubsub_on_channel_create(channel_s *ch, fio_match_fn match);
-static void pubsub_on_channel_destroy(channel_s *ch, fio_match_fn match);
+static void pubsub_on_channel_create(channel_s *ch);
+static void pubsub_on_channel_destroy(channel_s *ch);
 
 /* some comon tasks extracted */
 static inline channel_s *fio_filter_dup_lock_internal(channel_s *ch,
@@ -4385,7 +4633,7 @@ static inline channel_s *fio_filter_dup_lock_internal(channel_s *ch,
                                                       fio_collection_s *c) {
   fio_lock(&c->lock);
   ch = fio_ch_set_insert(&c->channels, hashed, ch);
-  fio_str_dup(&ch->id);
+  fio_channel_dup(ch);
   fio_lock(&ch->lock);
   fio_unlock(&c->lock);
   return ch;
@@ -4393,63 +4641,49 @@ static inline channel_s *fio_filter_dup_lock_internal(channel_s *ch,
 
 /** Creates / finds a filter channel, adds a reference count and locks it. */
 static channel_s *fio_filter_dup_lock(uint32_t filter) {
-  channel_s *ch = fio_malloc(sizeof(*ch));
-  FIO_ASSERT_ALLOC(ch);
-  *ch = (channel_s){
-      .id = FIO_STR_INIT,
-      .subscriptions = FIO_LS_INIT(ch->subscriptions),
+  channel_s ch = (channel_s){
+      .name = (char *)&filter,
+      .name_len = (sizeof(filter)),
       .parent = &fio_postoffice.filters,
-      .lock = FIO_LOCK_INIT,
+      .ref = 8, /* avoid freeing stack memory */
   };
-  fio_str_write(&ch->id, &filter, sizeof(filter));
-  fio_str_freeze(&ch->id);
-  return fio_filter_dup_lock_internal(ch, filter, &fio_postoffice.filters);
+  return fio_filter_dup_lock_internal(&ch, filter, &fio_postoffice.filters);
 }
 
 /** Creates / finds a pubsub channel, adds a reference count and locks it. */
 static channel_s *fio_channel_dup_lock(fio_str_info_s name) {
-  channel_s *ch = fio_malloc(sizeof(*ch));
-  FIO_ASSERT_ALLOC(ch);
-  *ch = (channel_s){
-      .id = FIO_STR_INIT,
-      .subscriptions = FIO_LS_INIT(ch->subscriptions),
+  channel_s ch = (channel_s){
+      .name = name.data,
+      .name_len = name.len,
       .parent = &fio_postoffice.pubsub,
-      .lock = FIO_LOCK_INIT,
+      .ref = 8, /* avoid freeing stack memory */
   };
-  fio_str_write(&ch->id, name.data, name.len);
-  fio_str_freeze(&ch->id);
-  uint64_t hashed_name = fio_str_hash(&ch->id);
-  ch = fio_filter_dup_lock_internal(ch, hashed_name, &fio_postoffice.pubsub);
-  if (fio_ls_embd_is_empty(&ch->subscriptions)) {
-    pubsub_on_channel_create(ch, NULL);
+  uint64_t hashed_name = fio_siphash(name.data, name.len);
+  channel_s *ch_p =
+      fio_filter_dup_lock_internal(&ch, hashed_name, &fio_postoffice.pubsub);
+  if (fio_ls_embd_is_empty(&ch_p->subscriptions)) {
+    pubsub_on_channel_create(ch_p);
   }
-  return ch;
+  return ch_p;
 }
 
 /** Creates / finds a pattern channel, adds a reference count and locks it. */
 static channel_s *fio_channel_match_dup_lock(fio_str_info_s name,
                                              fio_match_fn match) {
-  pattern_s *ch = fio_malloc(sizeof(*ch));
-  FIO_ASSERT_ALLOC(ch);
-  *ch = (pattern_s){
-      .ch =
-          {
-              .id = FIO_STR_INIT,
-              .subscriptions = FIO_LS_INIT(ch->ch.subscriptions),
-              .parent = &fio_postoffice.patterns,
-              .lock = FIO_LOCK_INIT,
-          },
+  channel_s ch = (channel_s){
+      .name = name.data,
+      .name_len = name.len,
+      .parent = &fio_postoffice.patterns,
       .match = match,
+      .ref = 8, /* avoid freeing stack memory */
   };
-  fio_str_write(&ch->ch.id, name.data, name.len);
-  fio_str_freeze(&ch->ch.id);
-  uint64_t hashed_name = fio_str_hash(&ch->ch.id) ^ ((uintptr_t)match);
-  ch = (pattern_s *)fio_filter_dup_lock_internal(&ch->ch, hashed_name,
-                                                 &fio_postoffice.patterns);
-  if (fio_ls_embd_is_empty(&ch->ch.subscriptions)) {
-    pubsub_on_channel_create(&ch->ch, match);
+  uint64_t hashed_name = fio_siphash(name.data, name.len);
+  channel_s *ch_p =
+      fio_filter_dup_lock_internal(&ch, hashed_name, &fio_postoffice.patterns);
+  if (fio_ls_embd_is_empty(&ch_p->subscriptions)) {
+    pubsub_on_channel_create(ch_p);
   }
-  return &ch->ch;
+  return ch_p;
 }
 
 /* to be used for reference counting (subtructing) */
@@ -4460,7 +4694,7 @@ static inline void fio_subscription_free(subscription_s *s) {
   if (s->on_unsubscribe) {
     s->on_unsubscribe(s->udata1, s->udata2);
   }
-  fio_str_free2((fio_str_s *)s->parent);
+  fio_channel_free(s->parent);
   fio_free(s);
 }
 
@@ -4504,19 +4738,13 @@ void fio_unsubscribe(subscription_s *s) {
     goto finish;
   fio_lock(&s->lock);
   channel_s *ch = s->parent;
-  fio_match_fn match = NULL;
   uint8_t removed = 0;
   fio_lock(&ch->lock);
   fio_ls_embd_remove(&s->node);
   /* check if channel is done for */
   if (fio_ls_embd_is_empty(&ch->subscriptions)) {
     fio_collection_s *c = ch->parent;
-    uint64_t hashed = fio_str_hash(&ch->id);
-    if (c == &fio_postoffice.patterns) {
-      pattern_s *pat = (pattern_s *)ch;
-      hashed ^= ((uintptr_t)pat->match);
-      match = ((pattern_s *)(ch))->match;
-    }
+    uint64_t hashed = fio_siphash(ch->name, ch->name_len);
     /* lock collection */
     fio_lock(&c->lock);
     /* test again within lock */
@@ -4528,7 +4756,7 @@ void fio_unsubscribe(subscription_s *s) {
   }
   fio_unlock(&ch->lock);
   if (removed) {
-    pubsub_on_channel_destroy(ch, match);
+    pubsub_on_channel_destroy(ch);
   }
 
   /* promise the subscription will be inactive */
@@ -4545,7 +4773,8 @@ finish:
  * To keep the string beyond the lifetime of the subscription, copy the string.
  */
 fio_str_info_s fio_subscription_channel(subscription_s *subscription) {
-  return fio_str_info(&subscription->parent->id);
+  return (fio_str_info_s){.data = subscription->parent->name,
+                          .len = subscription->parent->name_len};
 }
 
 /* *****************************************************************************
@@ -4554,31 +4783,34 @@ Engine handling and Management
 
 /* implemented later, informs root process about pub/sub subscriptions */
 static inline void fio_cluster_inform_root_about_channel(channel_s *ch,
-                                                         fio_match_fn match,
                                                          int add);
 
 /* runs in lock(!) let'm all know */
-static void pubsub_on_channel_create(channel_s *ch, fio_match_fn match) {
+static void pubsub_on_channel_create(channel_s *ch) {
   fio_lock(&fio_postoffice.engines.lock);
   FIO_SET_FOR_LOOP(&fio_postoffice.engines.set, pos) {
     if (!pos->hash)
       continue;
-    pos->obj->subscribe(pos->obj, fio_str_info(&ch->id), match);
+    pos->obj->subscribe(pos->obj,
+                        (fio_str_info_s){.data = ch->name, .len = ch->name_len},
+                        ch->match);
   }
   fio_unlock(&fio_postoffice.engines.lock);
-  fio_cluster_inform_root_about_channel(ch, match, 1);
+  fio_cluster_inform_root_about_channel(ch, 1);
 }
 
 /* runs in lock(!) let'm all know */
-static void pubsub_on_channel_destroy(channel_s *ch, fio_match_fn match) {
+static void pubsub_on_channel_destroy(channel_s *ch) {
   fio_lock(&fio_postoffice.engines.lock);
   FIO_SET_FOR_LOOP(&fio_postoffice.engines.set, pos) {
     if (!pos->hash)
       continue;
-    pos->obj->unsubscribe(pos->obj, fio_str_info(&ch->id), match);
+    pos->obj->unsubscribe(
+        pos->obj, (fio_str_info_s){.data = ch->name, .len = ch->name_len},
+        ch->match);
   }
   fio_unlock(&fio_postoffice.engines.lock);
-  fio_cluster_inform_root_about_channel(ch, match, 0);
+  fio_cluster_inform_root_about_channel(ch, 0);
 }
 
 /**
@@ -4636,15 +4868,20 @@ void fio_pubsub_reattach(fio_pubsub_engine_s *eng) {
   FIO_SET_FOR_LOOP(&fio_postoffice.pubsub.channels, pos) {
     if (!pos->hash)
       continue;
-    eng->subscribe(eng, fio_str_info(&pos->obj->id), NULL);
+    eng->subscribe(
+        eng,
+        (fio_str_info_s){.data = pos->obj->name, .len = pos->obj->name_len},
+        NULL);
   }
   fio_unlock(&fio_postoffice.pubsub.lock);
   fio_lock(&fio_postoffice.patterns.lock);
   FIO_SET_FOR_LOOP(&fio_postoffice.patterns.channels, pos) {
     if (!pos->hash)
       continue;
-    eng->subscribe(eng, fio_str_info(&pos->obj->id),
-                   ((pattern_s *)pos->obj)->match);
+    eng->subscribe(
+        eng,
+        (fio_str_info_s){.data = pos->obj->name, .len = pos->obj->name_len},
+        pos->obj->match);
   }
   fio_unlock(&fio_postoffice.patterns.lock);
 }
@@ -4653,60 +4890,25 @@ void fio_pubsub_reattach(fio_pubsub_engine_s *eng) {
  * Message Metadata handling
  **************************************************************************** */
 
-static inline void fio_call_meta_callbacks(fio_msg_internal_s *m,
-                                           fio_str_info_s ch,
-                                           fio_str_info_s msg,
-                                           uint8_t is_json) {
-  uintptr_t len;
-  fio_meta_set__ordered_s_ *cpy = NULL;
-  fio_lock(&fio_postoffice.meta.lock);
-  /* don't call user code within a lock - copy the array :-( */
-  len = fio_postoffice.meta.set.pos;
-  if (len && fio_meta_set_count(&fio_postoffice.meta.set)) {
-    cpy = fio_malloc(sizeof(*cpy) * len);
-    FIO_ASSERT_ALLOC(cpy);
-    memcpy(cpy, fio_postoffice.meta.set.ordered, sizeof(*cpy) * len);
-  }
-  fio_unlock(&fio_postoffice.meta.lock);
-  if (!cpy) {
-    return;
-  }
-  for (uintptr_t i = 0; i < len; ++i) {
-    if (!cpy[i].hash)
-      continue;
-    fio_msg_metadata_s *meta = fio_malloc(sizeof(*meta));
-    FIO_ASSERT_ALLOC(meta);
-    *meta = cpy[i].obj(ch, msg, is_json);
-    if (meta->metadata) {
-      meta->next = m->meta;
-      m->meta = meta;
-    } else {
-      fio_free(meta);
-    }
-  }
-  fio_free(cpy);
-}
-
 void fio_message_metadata_callback_set(fio_msg_metadata_fn callback,
                                        int enable) {
+  if (!callback)
+    return;
   fio_lock(&fio_postoffice.meta.lock);
+  fio_meta_ary_remove2(&fio_postoffice.meta.ary, callback, NULL);
   if (enable)
-    fio_meta_set_insert(&fio_postoffice.meta.set, (uintptr_t)callback,
-                        callback);
-  else
-    fio_meta_set_remove(&fio_postoffice.meta.set, (uintptr_t)callback, callback,
-                        NULL);
+    fio_meta_ary_push(&fio_postoffice.meta.ary, callback);
   fio_unlock(&fio_postoffice.meta.lock);
 }
 
 /** Finds the message's metadata by it's type ID. */
 void *fio_message_metadata(fio_msg_s *msg, intptr_t type_id) {
-  fio_msg_metadata_s *exists = ((fio_msg_client_s *)msg)->meta;
-  while (exists && exists->type_id != type_id) {
-    exists = exists->next;
-  }
-  if (exists) {
-    return exists->metadata;
+  fio_msg_metadata_s *meta = ((fio_msg_client_s *)msg)->meta;
+  size_t len = ((fio_msg_client_s *)msg)->meta_len;
+  while (len) {
+    --len;
+    if (meta[len].type_id == type_id)
+      return meta[len].metadata;
   }
   return NULL;
 }
@@ -4716,23 +4918,23 @@ void *fio_message_metadata(fio_msg_s *msg, intptr_t type_id) {
  **************************************************************************** */
 
 /* common internal tasks */
-static channel_s *fio_channel_find_dup_internal(fio_str_s *s, uint64_t hashed,
+static channel_s *fio_channel_find_dup_internal(channel_s *ch_tmp,
+                                                uint64_t hashed,
                                                 fio_collection_s *c) {
   fio_lock(&c->lock);
-  channel_s *ch = fio_ch_set_find(&c->channels, hashed, (channel_s *)s);
+  channel_s *ch = fio_ch_set_find(&c->channels, hashed, ch_tmp);
   if (!ch) {
     fio_unlock(&c->lock);
     return NULL;
   }
-  fio_str_dup((fio_str_s *)ch);
+  fio_channel_dup(ch);
   fio_unlock(&c->lock);
   return ch;
 }
 
 /** Finds a filter channel, increasing it's reference count if it exists. */
 static channel_s *fio_filter_find_dup(uint32_t filter) {
-  fio_str_s tmp = FIO_STR_INIT_EXISTING((char *)&filter, sizeof(filter),
-                                        0); /* don't free */
+  channel_s tmp = {.name = (char *)(&filter), .name_len = sizeof(filter)};
   channel_s *ch =
       fio_channel_find_dup_internal(&tmp, filter, &fio_postoffice.filters);
   return ch;
@@ -4740,9 +4942,8 @@ static channel_s *fio_filter_find_dup(uint32_t filter) {
 
 /** Finds a pubsub channel, increasing it's reference count if it exists. */
 static channel_s *fio_channel_find_dup(fio_str_info_s name) {
-  fio_str_s tmp =
-      FIO_STR_INIT_EXISTING(name.data, name.len, 0); /* don't free */
-  uint64_t hashed_name = fio_str_hash(&tmp);
+  channel_s tmp = {.name = name.data, .name_len = name.len};
+  uint64_t hashed_name = fio_siphash(name.data, name.len);
   channel_s *ch =
       fio_channel_find_dup_internal(&tmp, hashed_name, &fio_postoffice.pubsub);
   return ch;
@@ -4752,20 +4953,16 @@ static channel_s *fio_channel_find_dup(fio_str_info_s name) {
 static inline void fio_msg_internal_free(fio_msg_internal_s *msg) {
   if (fio_atomic_sub(&msg->ref, 1))
     return;
-  if (msg->meta) {
-    fio_msg_metadata_s *meta = msg->meta;
-    do {
-      fio_msg_metadata_s *tmp = meta;
-      meta = meta->next;
-      if (tmp->on_finish) {
-        fio_msg_s tmp_msg = {
-            .channel = msg->channel,
-            .msg = msg->data,
-        };
-        tmp->on_finish(&tmp_msg, tmp->metadata);
-      }
-      fio_free(tmp);
-    } while (meta);
+  while (msg->meta_len) {
+    --msg->meta_len;
+    if (msg->meta[msg->meta_len].on_finish) {
+      fio_msg_s tmp_msg = {
+          .channel = msg->channel,
+          .msg = msg->data,
+      };
+      msg->meta[msg->meta_len].on_finish(&tmp_msg,
+                                         msg->meta[msg->meta_len].metadata);
+    }
   }
   fio_free(msg);
 }
@@ -4785,7 +4982,7 @@ void fio_message_defer(fio_msg_s *msg_) {
 static void fio_perform_subscription_callback(void *s_, void *msg_) {
   subscription_s *s = s_;
   if (fio_trylock(&s->lock)) {
-    fio_defer(fio_perform_subscription_callback, s_, msg_);
+    fio_defer_push_task(fio_perform_subscription_callback, s_, msg_);
     return;
   }
   fio_msg_internal_s *msg = (fio_msg_internal_s *)msg_;
@@ -4798,6 +4995,7 @@ static void fio_perform_subscription_callback(void *s_, void *msg_) {
               .udata1 = s->udata1,
               .udata2 = s->udata2,
           },
+      .meta_len = msg->meta_len,
       .meta = msg->meta,
       .marker = 0,
   };
@@ -4807,7 +5005,7 @@ static void fio_perform_subscription_callback(void *s_, void *msg_) {
   }
   fio_unlock(&s->lock);
   if (m.marker) {
-    fio_defer(fio_perform_subscription_callback, s_, msg_);
+    fio_defer_push_task(fio_perform_subscription_callback, s_, msg_);
     return;
   }
   fio_msg_internal_free(msg);
@@ -4823,7 +5021,7 @@ static void fio_publish2channel(channel_s *ch, fio_msg_internal_s *msg) {
     }
     fio_atomic_add(&s->ref, 1);
     fio_atomic_add(&msg->ref, 1);
-    fio_defer(fio_perform_subscription_callback, s, msg);
+    fio_defer_push_task(fio_perform_subscription_callback, s, msg);
   }
   fio_msg_internal_free(msg);
 }
@@ -4834,13 +5032,13 @@ static void fio_publish2channel_task(void *ch_, void *msg) {
   if (!msg)
     goto finish;
   if (fio_trylock(&ch->lock)) {
-    fio_defer(fio_publish2channel_task, ch, msg);
+    fio_defer_push_urgent(fio_publish2channel_task, ch, msg);
     return;
   }
   fio_publish2channel(ch, msg);
   fio_unlock(&ch->lock);
 finish:
-  fio_str_free(&ch->id);
+  fio_channel_free(ch);
 }
 
 /** Publishes the message to the current process and frees the strings. */
@@ -4854,12 +5052,10 @@ static void fio_publish2process(fio_msg_internal_s *m) {
   } else {
     ch = fio_channel_find_dup(m->channel);
   }
-  if (m->filter == 0) {
-    fio_call_meta_callbacks(m, m->channel, m->data, m->is_json);
-  }
   /* exact match */
   if (ch) {
-    fio_defer(fio_publish2channel_task, &ch->id, fio_msg_internal_dup(m));
+    fio_defer_push_urgent(fio_publish2channel_task, ch,
+                          fio_msg_internal_dup(m));
   }
   if (m->filter == 0) {
     /* pattern matching match */
@@ -4868,10 +5064,13 @@ static void fio_publish2process(fio_msg_internal_s *m) {
       if (!p->hash) {
         continue;
       }
-      pattern_s *pattern = (pattern_s *)p->obj;
-      if (pattern->match(fio_str_info(&pattern->ch.id), m->channel)) {
-        fio_defer(fio_publish2channel_task, fio_str_dup(&pattern->ch.id),
-                  fio_msg_internal_dup(m));
+
+      if (p->obj->match(
+              (fio_str_info_s){.data = p->obj->name, .len = p->obj->name_len},
+              m->channel)) {
+        fio_channel_dup(p->obj);
+        fio_defer_push_urgent(fio_publish2channel_task, p->obj,
+                              fio_msg_internal_dup(m));
       }
     }
     fio_unlock(&fio_postoffice.patterns.lock);
@@ -4953,7 +5152,7 @@ static void fio_cluster_init(void) {
   uint32_t tmp_folder_len = 0;
   if (!tmp_folder || ((tmp_folder_len = (uint32_t)strlen(tmp_folder)) > 100)) {
 #ifdef P_tmpdir
-    tmp_folder = P_tmpdir;
+    tmp_folder = (char *)P_tmpdir;
     if (tmp_folder)
       tmp_folder_len = (uint32_t)strlen(tmp_folder);
 #else
@@ -5052,18 +5251,14 @@ static void fio_cluster_on_data(intptr_t uuid, fio_protocol_s *pr_) {
           return;
         }
       }
-      c->msg =
-          fio_malloc(sizeof(*c->msg) + c->exp_msg + 1 + c->exp_channel + 1);
-      FIO_ASSERT_ALLOC(c->msg);
-      *c->msg = (fio_msg_internal_s){
-          .filter = c->filter,
-          .channel = {.data = (char *)(c->msg + 1), .len = c->exp_channel},
-          .data = {.data = ((char *)(c->msg + 1) + c->exp_channel + 1),
-                   .len = c->exp_msg},
-          .is_json = c->type == FIO_CLUSTER_MSG_JSON ||
-                     c->type == FIO_CLUSTER_MSG_ROOT_JSON,
-          .ref = 1,
-      };
+      c->msg = fio_pubsub_create_message(
+          c->filter,
+          (fio_str_info_s){.data = (char *)(c->msg + 1), .len = c->exp_channel},
+          (fio_str_info_s){.data = ((char *)(c->msg + 1) + c->exp_channel + 1),
+                           .len = c->exp_msg},
+          (int8_t)(c->type == FIO_CLUSTER_MSG_JSON ||
+                   c->type == FIO_CLUSTER_MSG_ROOT_JSON),
+          0);
       i += 16;
     }
     if (c->exp_channel) {
@@ -5094,6 +5289,7 @@ static void fio_cluster_on_data(intptr_t uuid, fio_protocol_s *pr_) {
         c->exp_msg = 0;
       }
     }
+    fio_pubsub_create_message_update_meta(c->msg);
     c->handler(c);
     fio_msg_internal_free(c->msg);
     c->msg = NULL;
@@ -5343,8 +5539,8 @@ static void fio_cluster_client_handler(struct cluster_pr_s *pr) {
 static void fio_cluster_client_sender(fio_str_s *data, intptr_t ignr_) {
   if (!uuid_is_valid(cluster_data.uuid) && fio_data->active) {
     /* delay message delivery until we have a vaild uuid */
-    fio_defer((void (*)(void *, void *))fio_cluster_client_sender, data,
-              (void *)ignr_);
+    fio_defer_push_task((void (*)(void *, void *))fio_cluster_client_sender,
+                        data, (void *)ignr_);
     return;
   }
   fio_str_send_free2(cluster_data.uuid, data);
@@ -5370,7 +5566,7 @@ static void fio_cluster_on_connect(intptr_t uuid, void *udata) {
     if (!pos->hash) {
       continue;
     }
-    fio_cluster_inform_root_about_channel(pos->obj, NULL, 1);
+    fio_cluster_inform_root_about_channel(pos->obj, 1);
   }
   fio_unlock(&fio_postoffice.pubsub.lock);
   fio_lock(&fio_postoffice.patterns.lock);
@@ -5378,8 +5574,7 @@ static void fio_cluster_on_connect(intptr_t uuid, void *udata) {
     if (!pos->hash) {
       continue;
     }
-    fio_cluster_inform_root_about_channel(pos->obj,
-                                          ((pattern_s *)pos->obj)->match, 1);
+    fio_cluster_inform_root_about_channel(pos->obj, 1);
   }
   fio_unlock(&fio_postoffice.patterns.lock);
 
@@ -5441,34 +5636,34 @@ static void fio_send2cluster(int32_t filter, fio_str_info_s ch,
  **************************************************************************** */
 
 static inline void fio_cluster_inform_root_about_channel(channel_s *ch,
-                                                         fio_match_fn match,
                                                          int add) {
   if (!fio_data->is_worker || fio_data->workers == 1 || !cluster_data.uuid ||
       !ch)
     return;
-  fio_str_info_s ch_name = fio_str_info(&ch->id);
+  fio_str_info_s ch_name = {.data = ch->name, .len = ch->name_len};
   fio_str_info_s msg = {.data = NULL, .len = 0};
 #if DEBUG
   FIO_LOG_DEBUG("(%d) informing root about: %s (%zu) msg type %d", getpid(),
                 ch_name.data, ch_name.len,
-                (match ? (add ? FIO_CLUSTER_MSG_PATTERN_SUB
-                              : FIO_CLUSTER_MSG_PATTERN_UNSUB)
-                       : (add ? FIO_CLUSTER_MSG_PUBSUB_SUB
-                              : FIO_CLUSTER_MSG_PUBSUB_UNSUB)));
+                (ch->match ? (add ? FIO_CLUSTER_MSG_PATTERN_SUB
+                                  : FIO_CLUSTER_MSG_PATTERN_UNSUB)
+                           : (add ? FIO_CLUSTER_MSG_PUBSUB_SUB
+                                  : FIO_CLUSTER_MSG_PUBSUB_UNSUB)));
 #endif
   char buf[8] = {0};
-  if (match) {
-    fio_u2str64(buf, (uint64_t)match);
+  if (ch->match) {
+    fio_u2str64(buf, (uint64_t)ch->match);
     msg.data = buf;
-    msg.len = sizeof(match);
+    msg.len = sizeof(ch->match);
   }
 
   fio_cluster_client_sender(
       fio_cluster_wrap_message(ch_name.len, msg.len,
-                               (match ? (add ? FIO_CLUSTER_MSG_PATTERN_SUB
-                                             : FIO_CLUSTER_MSG_PATTERN_UNSUB)
-                                      : (add ? FIO_CLUSTER_MSG_PUBSUB_SUB
-                                             : FIO_CLUSTER_MSG_PUBSUB_UNSUB)),
+                               (ch->match
+                                    ? (add ? FIO_CLUSTER_MSG_PATTERN_SUB
+                                           : FIO_CLUSTER_MSG_PATTERN_UNSUB)
+                                    : (add ? FIO_CLUSTER_MSG_PUBSUB_SUB
+                                           : FIO_CLUSTER_MSG_PUBSUB_UNSUB)),
                                0, ch_name.data, msg.data),
       -1);
 }
@@ -5533,7 +5728,7 @@ static void fio_cluster_at_exit(void *ignore) {
   fio_engine_set_free(&fio_postoffice.engines.set);
 
   /* clear meta hooks */
-  fio_meta_set_free(&fio_postoffice.meta.set);
+  fio_meta_ary_free(&fio_postoffice.meta.ary);
   /* perform newly created tasks */
   fio_defer_perform();
   (void)ignore;
@@ -5563,21 +5758,10 @@ static void fio_cluster_signal_children(void) {
       -1);
 }
 
-static void fio_publish2process2(int32_t filter, fio_str_info_s ch_name,
-                                 fio_str_info_s msg, uint8_t is_json) {
-  fio_msg_internal_s *m =
-      fio_malloc(sizeof(*m) + ch_name.len + 1 + msg.len + 1);
-  FIO_ASSERT_ALLOC(m);
-  *m = (fio_msg_internal_s){
-      .filter = filter,
-      .channel = {.data = (char *)(m + 1), .len = ch_name.len},
-      .data = {.data = ((char *)(m + 1) + ch_name.len + 1), .len = msg.len},
-      .is_json = is_json,
-      .ref = 1,
-  };
-  memcpy(m->channel.data, ch_name.data, ch_name.len);
-  memcpy(m->data.data, msg.data, msg.len);
-  fio_publish2process(m);
+static inline void fio_publish2process2(int32_t filter, fio_str_info_s ch_name,
+                                        fio_str_info_s msg, uint8_t is_json) {
+  fio_publish2process(
+      fio_pubsub_create_message(filter, ch_name, msg, is_json, 1));
 }
 
 /**
@@ -6217,14 +6401,16 @@ static void fio_mem_init(void) {
   if (arenas)
     return;
 
+  ssize_t cpu_count = 0;
 #ifdef _SC_NPROCESSORS_ONLN
-  ssize_t cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
+  cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
 #else
 #warning Dynamic CPU core count is unavailable - assuming 8 cores for memory allocation pools.
-  ssize_t cpu_count = 8; /* fallback */
 #endif
+  if (cpu_count <= 0)
+    cpu_count = 8;
   memory.cores = cpu_count;
-  memory.count = 0 - (intptr_t)cpu_count;
+  memory.count = 0 - cpu_count;
   arenas = big_alloc(sizeof(*arenas) * cpu_count);
   FIO_ASSERT_ALLOC(arenas);
   size_t pre_pool = cpu_count > 32 ? 32 : cpu_count;
@@ -6271,6 +6457,7 @@ void *fio_malloc(size_t size) {
   }
   if (size >= FIO_MEMORY_BLOCK_ALLOC_LIMIT) {
     /* system allocation - must be block aligned */
+    // FIO_LOG_WARNING("fio_malloc re-routed to mmap - big allocation");
     return big_alloc(size);
   }
   /* ceiling for 16 byte alignement, translated to 16 byte units */
@@ -8443,7 +8630,9 @@ Testing fio_defer task system
 
 #define FIO_DEFER_TOTAL_COUNT (512 * 1024)
 
+#ifndef FIO_DEFER_TEST_PRINT
 #define FIO_DEFER_TEST_PRINT 0
+#endif
 
 static void sample_task(void *i_count, void *unused2) {
   (void)(unused2);
@@ -8511,7 +8700,7 @@ static void fio_defer_test(void) {
                "defer deallocation vs. allocation error, %zu != %zu",
                fio_defer_count_dealloc, fio_defer_count_alloc);
   }
-  FIO_ASSERT(deferred.writer == &fio_defer_static_queue,
+  FIO_ASSERT(task_queue_normal.writer == &task_queue_normal.static_queue,
              "defer library didn't release dynamic queue (should be static)");
   fprintf(stderr, "\n* passed.\n");
 }