iodine 0.7.1 → 0.7.2

data/ext/iodine/fio.h

@@ -11,82 +11,84 @@ Feel free to copy, use and enjoy according to the license provided.
 #define H_FACIL_IO_H
 
 /* *****************************************************************************
-Table of contents:
-=================
-* Version and helper macros
-* Helper String Information Type
-* Memory pool / custom allocator for short lived objects
-*
-* Connection Callback (Protocol) Management
-* Listening to Incoming Connections
-* Connecting to remote servers as a client
-* Starting the IO reactor and reviewing it's state
-* Socket / Connection Functions
-* Connection Read / Write Hooks, for overriding the system calls
-* Concurrency overridable functions
-* Connection Task scheduling
-* Event / Task scheduling
-* Startup / State Callbacks (fork, start up, idle, etc')
-* Lower Level API - for special circumstances, use with care under
-*
-* Pub/Sub / Cluster Messages API
-* Cluster Messages and Pub/Sub
-* Cluster / Pub/Sub Middleware and Extensions ("Engines")
-*
-* Atomic Operations and Spin Locking Helper Functions
-* Byte Swapping and Network Order
-*
-* Converting Numbers to Strings (and back)
-* Strings to Numbers
-* Numbers to Strings* Random Generator Functions
-*
-* SipHash
-* SHA-1
-* SHA-2
-* Base64 (URL) encoding
-*
-* Memory Allocator Details
-*
-* Spin locking Implementation
-*
-******** facil.io Data Types (String, Set / Hash Map, Linked Lists, etc')
-*
-* These types can be included by defining the macros and (re)including fio.h.
-*
-*
-*
-*                      #ifdef FIO_INCLUDE_LINKED_LIST
-*
-* Linked List Helpers
-* Independent Linked List API
-* Embedded Linked List API* Independent Linked List Implementation
-* Embeded Linked List Implementation
-*
-*
-*
-*                      #ifdef FIO_INCLUDE_STR
-*
-* String Helpers
-* String API - Initialization and Destruction
-* String API - String state (data pointers, length, capacity, etc')
-* String API - Memory management
-* String API - UTF-8 State
-* String Implementation - state (data pointers, length, capacity, etc')
-* String Implementation - Memory management
-* String Implementation - UTF-8 State
-* String Implementation - Content Manipulation and Review
-*
-*
-*
-*            #ifdef FIO_SET_NAME - can be included more than once
-*
-* Set / Hash Map Data-Store
-* Set / Hash Map API
-* Set / Hash Map Internal Data Structures
-* Set / Hash Map Internal Helpers
-* Set / Hash Map Implementation
-*
-***************************************************************************** */
+ * Table of contents (find by subject):
+ * =================
+ * Version and helper macros
+ * Helper String Information Type
+ * Memory pool / custom allocator for short lived objects
+ * Logging and testing helpers
+ *
+ * Connection Callback (Protocol) Management
+ * Listening to Incoming Connections
+ * Connecting to remote servers as a client
+ * Starting the IO reactor and reviewing it's state
+ * Socket / Connection Functions
+ * Connection Read / Write Hooks, for overriding the system calls
+ * Concurrency overridable functions
+ * Connection Task scheduling
+ * Event / Task scheduling
+ * Startup / State Callbacks (fork, start up, idle, etc')
+ * Lower Level API - for special circumstances, use with care under
+ *
+ * Pub/Sub / Cluster Messages API
+ * Cluster Messages and Pub/Sub
+ * Cluster / Pub/Sub Middleware and Extensions ("Engines")
+ *
+ * Atomic Operations and Spin Locking Helper Functions
+ * Byte Swapping and Network Order
+ *
+ * Converting Numbers to Strings (and back)
+ * Strings to Numbers
+ * Numbers to Strings* Random Generator Functions
+ *
+ * SipHash
+ * SHA-1
+ * SHA-2
+ * Base64 (URL) encoding
+ *
+ * Memory Allocator Details
+ *
+ * Spin locking Implementation
+ *
+ ******** facil.io Data Types (String, Set / Hash Map, Linked Lists, etc')
+ *
+ * These types can be included by defining the macros and (re)including fio.h.
+ *
+ *
+ *
+ *                      #ifdef FIO_INCLUDE_LINKED_LIST
+ *
+ * Linked List Helpers
+ * Independent Linked List API
+ * Embedded Linked List API* Independent Linked List Implementation
+ * Embeded Linked List Implementation
+ *
+ *
+ *
+ *                      #ifdef FIO_INCLUDE_STR
+ *
+ * String Helpers
+ * String API - Initialization and Destruction
+ * String API - String state (data pointers, length, capacity, etc')
+ * String API - Memory management
+ * String API - UTF-8 State
+ * String Implementation - state (data pointers, length, capacity, etc')
+ * String Implementation - Memory management
+ * String Implementation - UTF-8 State
+ * String Implementation - Content Manipulation and Review
+ *
+ *
+ *
+ *            #ifdef FIO_SET_NAME - can be included more than once
+ *
+ * Set / Hash Map Data-Store
+ * Set / Hash Map API
+ * Set / Hash Map Internal Data Structures
+ * Set / Hash Map Internal Helpers
+ * Set / Hash Map Implementation
+ *
+ *****************************************************************************
+ */
 
 /* *****************************************************************************
 Version and helper macros
@@ -199,44 +201,11 @@ Version and helper macros
 #define FIO_FUNC static __attribute__((unused))
 #endif
 
-#ifndef FIO_ASSERT_ALLOC
-/** Tests for an allocation failure. The behavior can be overridden. */
-#define FIO_ASSERT_ALLOC(ptr)                                                  \
-  if (!(ptr)) {                                                                \
-    fprintf(stderr,                                                            \
-            "FATAL ERROR: memory allocation error "__FILE__                    \
-            ":%d\n",                                                           \
-            __LINE__);                                                         \
-    perror("             Error details (errno)");                              \
-    kill(0, SIGINT);                                                           \
-    exit(errno);                                                               \
-  }
-#endif
-
 #if defined(__FreeBSD__)
 #include <netinet/in.h>
 #include <sys/socket.h>
 #endif
 
-#if FIO_PRINT_STATE
-#define FIO_LOG_STATE(...) fprintf(stderr, __VA_ARGS__)
-#else
-#define FIO_LOG_STATE(...)
-#endif
-
-#if DEBUG
-#define FIO_LOG_DEBUG(...) FIO_LOG_STATE("INFO [DEBUG]: " __VA_ARGS__)
-#define FIO_ASSERT(cond, ...)                                                  \
-  if (!(cond)) {                                                               \
-    fprintf(stderr, "FATAL [DEBUG] (" __FILE__                                 \
-                    ":" FIO_MACRO2STR(__LINE__) "): " __VA_ARGS__);            \
-    exit(-1);                                                                  \
-  }
-#else
-#define FIO_LOG_DEBUG(...)
-#define FIO_ASSERT(...)
-#endif
-
 /* *****************************************************************************
 C++ extern start
 ***************************************************************************** */
@@ -260,7 +229,31 @@ typedef struct fio_str_info_s {
 #endif
 
 /* *****************************************************************************
+
+
+
+
+
+
+
+
+
+
+
+
 Memory pool / custom allocator for short lived objects
+
+
+
+
+
+
+
+
+
+
+
+
 ***************************************************************************** */
 
 /**
@@ -308,10 +301,16 @@ void *fio_realloc2(void *ptr, size_t new_size, size_t copy_length);
  */
 void *fio_mmap(size_t size);
 
+/**
+ * When forking is called manually, call this function to reset the facil.io
+ * memory allocator's locks.
+ */
+void fio_malloc_after_fork(void);
+
 #if FIO_FORCE_MALLOC
-#define fio_malloc malloc
+#define fio_malloc(size) calloc(size, 1)
 #define fio_calloc calloc
-#define fio_mmap malloc
+#define fio_mmap(size) calloc(size, 1)
 #define fio_free free
 #define fio_realloc realloc
 #define fio_realloc2(ptr, new_size, old_data_len) realloc((ptr), (new_size))
@@ -332,6 +331,100 @@ void *fio_mmap(size_t size);
 
 
 
+Logging and testing helpers
+
+
+
+
+
+
+
+
+
+
+
+
+***************************************************************************** */
+
+/** Logging level of zero (no logging). */
+#define FIO_LOG_LEVEL_NONE 0
+/** Log fatal errors. */
+#define FIO_LOG_LEVEL_FATAL 1
+/** Log errors and fatal errors. */
+#define FIO_LOG_LEVEL_ERROR 2
+/** Log warnings, errors and fatal errors. */
+#define FIO_LOG_LEVEL_WARNING 3
+/** Log every message (info, warnings, errors and fatal errors). */
+#define FIO_LOG_LEVEL_INFO 4
+/** Log everything, including debug messages. */
+#define FIO_LOG_LEVEL_DEBUG 5
+
+/** The logging level */
+extern size_t FIO_LOG_LEVEL;
+
+#ifndef FIO_LOG_PRINT
+#define FIO_LOG_PRINT(level, ...)                                              \
+  do {                                                                         \
+    if (level <= FIO_LOG_LEVEL)                                                \
+      fprintf(stderr, "\n" __VA_ARGS__);                                       \
+  } while (0)
+#define FIO_LOG_DEBUG(...)                                                     \
+  FIO_LOG_PRINT(FIO_LOG_LEVEL_DEBUG,                                           \
+                "[DEBUG ("__FILE__                                             \
+                ":" FIO_MACRO2STR(__LINE__) ")] " __VA_ARGS__)
+#define FIO_LOG_INFO(...)                                                      \
+  FIO_LOG_PRINT(FIO_LOG_LEVEL_INFO, "[INFO] " __VA_ARGS__)
+#define FIO_LOG_WARNING(...)                                                   \
+  FIO_LOG_PRINT(FIO_LOG_LEVEL_WARNING, "[WARNING] " __VA_ARGS__)
+#define FIO_LOG_ERROR(...)                                                     \
+  FIO_LOG_PRINT(FIO_LOG_LEVEL_ERROR, "[ERROR] " __VA_ARGS__)
+#define FIO_LOG_FATAL(...)                                                     \
+  FIO_LOG_PRINT(FIO_LOG_LEVEL_FATAL, "[FATAL] " __VA_ARGS__)
+#endif
+
+#if FIO_PRINT_STATE
+#define FIO_LOG_STATE(...) fprintf(stderr, "\n" __VA_ARGS__)
+#else
+#define FIO_LOG_STATE(...)
+#endif
+
+#if DEBUG
+#define FIO_ASSERT(cond, ...)                                                  \
+  if (!(cond)) {                                                               \
+    FIO_LOG_DEBUG(__VA_ARGS__);                                                \
+    exit(-1);                                                                  \
+  }
+#else
+#define FIO_ASSERT(...)
+#endif
+
+#ifndef FIO_ASSERT_ALLOC
+/** Tests for an allocation failure. The behavior can be overridden. */
+#define FIO_ASSERT_ALLOC(ptr)                                                  \
+  if (!(ptr)) {                                                                \
+    fprintf(stderr,                                                            \
+            "FATAL ERROR: memory allocation error "__FILE__                    \
+            ":%d\n",                                                           \
+            __LINE__);                                                         \
+    perror("             Error details (errno)");                              \
+    kill(0, SIGINT);                                                           \
+    exit(errno);                                                               \
+  }
+#endif
+
+/* *****************************************************************************
+
+
+
+
+
+
+
+
+
+
+
+
 Connection Callback (Protocol) Management
 
 
@@ -1082,8 +1175,6 @@ typedef struct fio_rw_hook_s {
   ssize_t (*write)(intptr_t uuid, void *udata, const void *buf, size_t count);
   /**
    * The `close` callback should close the underlying socket / file descriptor.
-   * It should also be used to release any resources associated with the
-   * connection's read/write hooks.
    *
    * If the function returns a non-zero value, it will be called again after an
    * attempt to flush the socket and any pending outgoing buffer.
@@ -1103,6 +1194,10 @@ typedef struct fio_rw_hook_s {
    * deadlock might occur.
    */
   ssize_t (*flush)(intptr_t uuid, void *udata);
+  /**
+   * Called to perform cleanup after the socket was closed.
+   * */
+  void (*cleanup)(void *udata);
 } fio_rw_hook_s;
 
 /** Sets a socket hook state (a pointer to the struct). */
@@ -1260,6 +1355,8 @@ typedef enum {
   FIO_CALL_AFTER_FORK,
   /** Called by a worker process right after forking. */
   FIO_CALL_IN_CHILD,
+  /** Called by the master process after spawning a worker (after forking). */
+  FIO_CALL_IN_MASTER,
   /** Called every time a *Worker* proceess starts. */
   FIO_CALL_ON_START,
   /** Called when facil.io enters idling mode. */
@@ -2424,9 +2521,9 @@ C++ extern end
 #undef FIO_MEMORY_BLOCK_SIZE
 #undef FIO_MEMORY_BLOCK_MASK
 #undef FIO_MEMORY_BLOCK_SLICES
-#define FIO_MEMORY_BLOCK_MASK (FIO_MEMORY_BLOCK_SIZE - 1)    /* 0b111... */
-#define FIO_MEMORY_BLOCK_SLICES (FIO_MEMORY_BLOCK_SIZE >> 4) /* 16B slices */
 #define FIO_MEMORY_BLOCK_SIZE ((uintptr_t)1 << FIO_MEMORY_BLOCK_SIZE_LOG)
+#define FIO_MEMORY_BLOCK_MASK (FIO_MEMORY_BLOCK_SIZE - 1)    /* 0b0...1... */
+#define FIO_MEMORY_BLOCK_SLICES (FIO_MEMORY_BLOCK_SIZE >> 4) /* 16B slices */
 
 #ifndef FIO_MEMORY_BLOCK_ALLOC_LIMIT
 /* defaults to 37.5% of the block, after which `mmap` is used instead */
@@ -2515,13 +2612,13 @@ FIO_FUNC inline void fio_lock_dbg(fio_lock_i *lock, const char *file,
   while (fio_trylock(lock)) {
     if (lock_cycle_count >= 8 &&
         (lock_cycle_count == 8 || !(lock_cycle_count & 511)))
-      fprintf(stderr, "INFO: fio-spinlock spin %s:%d round %zu\n", file, line,
+      fprintf(stderr, "[DEBUG] fio-spinlock spin %s:%d round %zu\n", file, line,
               lock_cycle_count);
     ++lock_cycle_count;
     fio_reschedule_thread();
   }
   if (lock_cycle_count >= 8)
-    fprintf(stderr, "INFO: fio-spinlock spin %s:%d total = %zu\n", file, line,
+    fprintf(stderr, "[DEBUG] fio-spinlock spin %s:%d total = %zu\n", file, line,
             lock_cycle_count);
 }
 #define fio_lock(lock) fio_lock_dbg((lock), __FILE__, __LINE__)
@@ -2537,7 +2634,7 @@ FIO_FUNC inline int fio_trylock_dbg(fio_lock_i *lock, const char *file,
   } else if (line == last_line) {
     ++count;
     if (count >= 2)
-      fprintf(stderr, "INFO: trying fio-spinlock %s:%d attempt %zu\n", file,
+      fprintf(stderr, "[DEBUG] trying fio-spinlock %s:%d attempt %zu\n", file,
               line, count);
   } else {
     count = 0;
@@ -2834,10 +2931,16 @@ String API - Initialization and Destruction
  * used.
  */
 typedef struct {
+#ifndef FIO_STR_NO_REF
   volatile uint32_t ref; /* reference counter for fio_str_dup */
+#endif
   uint8_t small;  /* Flag indicating the String is small and self-contained */
   uint8_t frozen; /* Flag indicating the String is frozen (don't edit) */
-  uint8_t reserved[10];    /* Align struct on 16 byte allocator boundary */
+#ifdef FIO_STR_NO_REF
+  uint8_t reserved[14]; /* Align struct on 16 byte allocator boundary */
+#else
+  uint8_t reserved[10]; /* Align struct on 16 byte allocator boundary */
+#endif
   uint64_t capa;           /* Known capacity for longer Strings */
   uint64_t len;            /* String length for longer Strings */
   void (*dealloc)(void *); /* Data deallocation function (NULL for static) */
@@ -2910,6 +3013,9 @@ inline FIO_FUNC fio_str_s *fio_str_new_copy2(fio_str_s *src);
 /**
  * Adds a references to the current String object and returns itself.
  *
+ * If refecrence counting was disabled (FIO_STR_NO_REF was defined), returns a
+ * copy of the String (free with `fio_str_free2`).
+ *
  * NOTE: Nothing is copied, reference Strings are referencing the same String.
  *       Editing one reference will effect the other.
  *
@@ -3145,7 +3251,9 @@ String Implementation - state (data pointers, length, capacity, etc')
 ***************************************************************************** */
 
 typedef struct {
+#ifndef FIO_STR_NO_REF
   volatile uint32_t ref; /* reference counter for fio_str_dup */
+#endif
   uint8_t small;  /* Flag indicating the String is small and self-contained */
   uint8_t frozen; /* Flag indicating the String is frozen (don't edit) */
 } fio_str__small_s;
@@ -3199,6 +3307,9 @@ inline FIO_FUNC fio_str_s *fio_str_new_copy2(fio_str_s *src) {
 /**
  * Adds a references to the current String object and returns itself.
  *
+ * If refecrence counting was disabled (FIO_STR_NO_REF was defined), returns a
+ * copy of the String (free with `fio_str_free2`).
+ *
  * NOTE: Nothing is copied, reference Strings are referencing the same String.
  *       Editing one reference will effect the other.
  *
@@ -3207,9 +3318,15 @@ inline FIO_FUNC fio_str_s *fio_str_new_copy2(fio_str_s *src) {
  *       were freed using `fio_str_free` / `fio_str_free2` or discarded.
  */
 inline FIO_FUNC fio_str_s *fio_str_dup(fio_str_s *s) {
+#ifdef FIO_STR_NO_REF
+  fio_str_s *s2 = fio_str_new2();
+  fio_str_concat(s2, s);
+  return s2;
+#else
   if (s)
     fio_atomic_add(&s->ref, 1);
   return s;
+#endif
 }
 
 /**
@@ -3222,13 +3339,14 @@ inline FIO_FUNC fio_str_s *fio_str_dup(fio_str_s *s) {
  * references (see fio_str_dup).
  */
 inline FIO_FUNC int fio_str_free(fio_str_s *s) {
-  if (s && fio_atomic_sub(&s->ref, 1) == (uint32_t)-1) {
-    if (!s->small && s->dealloc)
-      s->dealloc(s->data);
-    *s = FIO_STR_INIT;
-    return 0;
-  }
-  return -1;
+#ifndef FIO_STR_NO_REF
+  if (!s || fio_atomic_sub(&s->ref, 1) != (uint32_t)-1)
+    return -1;
+#endif
+  if (!s->small && s->dealloc)
+    s->dealloc(s->data);
+  *s = FIO_STR_INIT;
+  return 0;
 }
 
 /**
@@ -3323,7 +3441,7 @@ String Implementation - Memory management
  * since it uses 16 byte alignment right up until allocations are routed
  * directly to `mmap` (due to their size, usually over 12KB).
  */
-#define ROUND_UP_CAPA_2WORDS(num)                                              \
+#define ROUND_UP_CAPA2WORDS(num)                                               \
   (((num + 1) & (sizeof(long double) - 1))                                     \
        ? ((num + 1) | (sizeof(long double) - 1))                               \
        : (num))
@@ -3339,7 +3457,7 @@ FIO_FUNC fio_str_info_s fio_str_capa_assert(fio_str_s *s, size_t needed) {
     goto is_small;
   }
   if (needed > s->capa) {
-    needed = ROUND_UP_CAPA_2WORDS(needed);
+    needed = ROUND_UP_CAPA2WORDS(needed);
     if (s->dealloc == fio_free) {
       tmp = (char *)fio_realloc2(s->data, needed + 1, s->len);
       FIO_ASSERT_ALLOC(tmp);
@@ -3364,7 +3482,7 @@ is_small:
                             .len = (size_t)(s->small >> 1),
                             .data = FIO_STR_SMALL_DATA(s)};
   }
-  needed = ROUND_UP_CAPA_2WORDS(needed);
+  needed = ROUND_UP_CAPA2WORDS(needed);
   tmp = (char *)fio_malloc(needed + 1);
   FIO_ASSERT_ALLOC(tmp);
   const size_t existing_len = (size_t)((s->small >> 1) & 0xFF);
@@ -3373,6 +3491,15 @@ is_small:
   } else {
     tmp[0] = 0;
   }
+#ifdef FIO_STR_NO_REF
+  *s = (fio_str_s){
+      .small = 0,
+      .capa = needed,
+      .len = existing_len,
+      .dealloc = fio_free,
+      .data = tmp,
+  };
+#else
   *s = (fio_str_s){
       .ref = s->ref,
       .small = 0,
@@ -3381,6 +3508,7 @@ is_small:
       .dealloc = fio_free,
       .data = tmp,
   };
+#endif
   return (fio_str_info_s){
       .capa = (s->frozen ? 0 : needed), .len = existing_len, .data = s->data};
 }
@@ -3845,11 +3973,9 @@ FIO_FUNC fio_str_info_s fio_str_readfile(fio_str_s *s, const char *filename,
   const size_t org_len = fio_str_len(s);
   state = fio_str_resize(s, org_len + limit);
   if (pread(file, state.data + org_len, limit, start_at) != (ssize_t)limit) {
-    close(file);
     fio_str_resize(s, org_len);
     state.data = NULL;
     state.len = state.capa = 0;
-    goto finish;
   }
   close(file);
 finish:
@@ -3857,7 +3983,7 @@ finish:
   return state;
 #else
   /* TODO: consider adding non POSIX implementations. */
-  fprintf(stderr, "ERROR: File reading requires a posix system (ignored!).\n");
+  FIO_LOG_ERROR("File reading requires a posix system (ignored!).\n");
   return state;
 #endif
 }
@@ -3901,8 +4027,9 @@ inline FIO_FUNC ssize_t fio_str_send_free2(const intptr_t uuid,
                     .after.dealloc = (void (*)(void *))fio_str_free2);
 }
 
-#undef ROUND_UP_CAPA_2WORDS
+#undef ROUND_UP_CAPA2WORDS
 #undef FIO_STR_SMALL_DATA
+#undef FIO_STR_NO_REF
 
 #endif /* H_FIO_STR_H */
 /* *****************************************************************************
@@ -4139,7 +4266,7 @@ FIO_FUNC void FIO_NAME(free)(FIO_NAME(s) * set);
 #ifdef FIO_SET_KEY_TYPE
 
 /**
- *Locates an object in the Set, if it exists.
+ * Locates an object in the Hash Map, if it exists.
  *
  * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
  */
@@ -4148,13 +4275,15 @@ FIO_FUNC inline FIO_SET_OBJ_TYPE *
                    FIO_SET_KEY_TYPE key);
 
 /**
- * Inserts an object to the Set only if it's missing, rehashing if required,
- * returning the new (or old) object's pointer.
+ * Inserts an object to the Hash Map, rehashing if required, returning the new
+ * object's location using a pointer.
  *
- * If the object already exists in the set, no action is performed (the old
- * object is returned).
+ * If an object already exists in the Hash Map with the same key, it will be
+ * destroyed.
  *
- * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ * NOTE 1: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ *
+ * NOTE 2: This is equivelant to calling the `insert2` with `old` set to NULL.
  */
 FIO_FUNC inline void FIO_NAME(insert)(FIO_NAME(s) * set,
                                       const FIO_SET_HASH_TYPE hash_value,
@@ -4162,7 +4291,24 @@ FIO_FUNC inline void FIO_NAME(insert)(FIO_NAME(s) * set,
                                       FIO_SET_OBJ_TYPE obj);
 
 /**
- * Removes an object from the Set, rehashing if required.
+ * Inserts an object to the Hash Map, rehashing if required, returning the new
+ * object's location using a pointer.
+ *
+ * If an object already exists in the Hash Map, it will be destroyed.
+ *
+ * If `old` is set, the existing object (if any) will be copied to the location
+ * pointed to by `old` before it is destroyed.
+ *
+ * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ */
+FIO_FUNC inline void FIO_NAME(insert2)(FIO_NAME(s) * set,
+                                       const FIO_SET_HASH_TYPE hash_value,
+                                       FIO_SET_KEY_TYPE key,
+                                       FIO_SET_OBJ_TYPE obj,
+                                       FIO_SET_OBJ_TYPE *old);
+
+/**
+ * Removes an object from the Hash Map, rehashing if required.
  *
  * Returns 0 on success and -1 if the object wasn't found.
  *
@@ -4172,6 +4318,21 @@ FIO_FUNC inline int FIO_NAME(remove)(FIO_NAME(s) * set,
                                      const FIO_SET_HASH_TYPE hash_value,
                                      FIO_SET_KEY_TYPE key);
 
+/**
+ * Removes an object from the Hash Map, rehashing if required.
+ *
+ * Returns 0 on success and -1 if the object wasn't found.
+ *
+ * If `old` is set, the existing object (if any) will be copied to the location
+ * pointed to by `old`.
+ *
+ * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ */
+FIO_FUNC inline int FIO_NAME(remove2)(FIO_NAME(s) * set,
+                                      const FIO_SET_HASH_TYPE hash_value,
+                                      FIO_SET_KEY_TYPE key,
+                                      FIO_SET_OBJ_TYPE *old);
+
 #else
 
 /**
@@ -4210,6 +4371,17 @@ FIO_FUNC inline FIO_SET_OBJ_TYPE *
     FIO_NAME(overwrite)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
                         FIO_SET_OBJ_TYPE obj);
 
+/**
+ * The same as `overwrite`, only it copies the old object (if any) to the
+ * location pointed to by `old`.
+ *
+ * When setting `old` to NULL, the function behaves the same as `overwrite`.
+ */
+FIO_FUNC FIO_SET_OBJ_TYPE *FIO_NAME(replace)(FIO_NAME(s) * set,
+                                             const FIO_SET_HASH_TYPE hash_value,
+                                             FIO_SET_OBJ_TYPE obj,
+                                             FIO_SET_OBJ_TYPE *old);
+
 /**
  * Removes an object from the Set, rehashing if required.
  *
@@ -4337,7 +4509,7 @@ FIO_FUNC inline FIO_NAME(_map_s_) *
     if (FIO_SET_HASH_COMPARE(FIO_SET_HASH_INVALID, pos->hash))
       return pos;
     if (FIO_SET_HASH_COMPARE(pos->hash, hash_value)) {
-      if (!pos->pos || FIO_SET_OBJ_COMPARE(pos->pos->obj, obj))
+      if (!pos->pos || FIO_SET_COMPARE(pos->pos->obj, obj))
         return pos;
       set->has_collisions = 1;
     }
@@ -4353,7 +4525,7 @@ FIO_FUNC inline FIO_NAME(_map_s_) *
       if (FIO_SET_HASH_COMPARE(FIO_SET_HASH_INVALID, pos->hash))
         return pos;
       if (FIO_SET_HASH_COMPARE(pos->hash, hash_value)) {
-        if (!pos->pos || FIO_SET_OBJ_COMPARE(pos->pos->obj, obj))
+        if (!pos->pos || FIO_SET_COMPARE(pos->pos->obj, obj))
           return pos;
         set->has_collisions = 1;
       }
@@ -4407,10 +4579,9 @@ FIO_FUNC inline void FIO_NAME(_reallocate_set_mem_)(FIO_NAME(s) * set) {
  * If the object already exists in the set, it will be destroyed and
  * overwritten.
  */
-FIO_FUNC inline FIO_SET_TYPE *
-FIO_NAME(_insert_or_overwrite_)(FIO_NAME(s) * set,
-                                const FIO_SET_HASH_TYPE hash_value,
-                                FIO_SET_TYPE obj, int overwrite) {
+FIO_FUNC inline FIO_SET_TYPE *FIO_NAME(_insert_or_overwrite_)(
+    FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value, FIO_SET_TYPE obj,
+    int overwrite, FIO_SET_OBJ_TYPE *old) {
   if (FIO_SET_HASH_COMPARE(hash_value, FIO_SET_HASH_INVALID))
     return NULL;
 
@@ -4435,11 +4606,17 @@ FIO_NAME(_insert_or_overwrite_)(FIO_NAME(s) * set,
       return &pos->pos->obj;
     }
 #ifdef FIO_SET_KEY_TYPE
+    if (old) {
+      FIO_SET_OBJ_COPY((*old), pos->pos->obj.obj);
+    }
     /* no need to recreate the key object, just the value object */
     FIO_SET_OBJ_DESTROY(pos->pos->obj.obj);
     FIO_SET_OBJ_COPY(pos->pos->obj.obj, obj.obj);
     return &pos->pos->obj;
 #else
+    if (old) {
+      FIO_SET_COPY((*old), pos->pos->obj);
+    }
     FIO_SET_DESTROY(pos->pos->obj);
 #endif
   } else {
@@ -4484,9 +4661,9 @@ FIO_FUNC void FIO_NAME(free)(FIO_NAME(s) * s) {
  *
  * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
  */
-FIO_FUNC inline FIO_SET_OBJ_TYPE *
-FIO_NAME(find)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
-               FIO_SET_KEY_TYPE key) {
+FIO_FUNC FIO_SET_OBJ_TYPE *FIO_NAME(find)(FIO_NAME(s) * set,
+                                          const FIO_SET_HASH_TYPE hash_value,
+                                          FIO_SET_KEY_TYPE key) {
   FIO_NAME(_map_s_) *pos =
       FIO_NAME(_find_map_pos_)(set, hash_value, (FIO_SET_TYPE){.key = key});
   if (!pos || !pos->pos)
@@ -4495,28 +4672,86 @@ FIO_NAME(find)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
 }
 
 /**
- * Inserts an object to the Set only if it's missing, rehashing if required,
- * returning the new (or old) object's pointer.
+ * Inserts an object to the Hash Map, rehashing if required, returning the new
+ * object's location using a pointer.
  *
- * If the object already exists in the set, no action is performed (the old
- * object is returned).
+ * If the object already exists in the set, it will be destroyed.
  *
  * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
  */
-FIO_FUNC inline void FIO_NAME(insert)(FIO_NAME(s) * set,
+FIO_FUNC void FIO_NAME(insert)(FIO_NAME(s) * set,
+                               const FIO_SET_HASH_TYPE hash_value,
+                               FIO_SET_KEY_TYPE key, FIO_SET_OBJ_TYPE obj) {
+  FIO_NAME(_insert_or_overwrite_)
+  (set, hash_value, (FIO_SET_TYPE){.key = key, .obj = obj}, 1, NULL);
+}
+
+/**
+ * Inserts an object to the Hash Map, rehashing if required, returning the new
+ * object's location using a pointer.
+ *
+ * If an object already exists in the Hash Map, it will be destroyed.
+ *
+ * If `old` is set, the existing object (if any) will be copied to the location
+ * pointed to by `old` before it is destroyed.
+ *
+ * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ */
+FIO_FUNC void FIO_NAME(insert2)(FIO_NAME(s) * set,
+                                const FIO_SET_HASH_TYPE hash_value,
+                                FIO_SET_KEY_TYPE key, FIO_SET_OBJ_TYPE obj,
+                                FIO_SET_OBJ_TYPE *old) {
+  FIO_NAME(_insert_or_overwrite_)
+  (set, hash_value, (FIO_SET_TYPE){.key = key, .obj = obj}, 1, old);
+}
+
+/**
+ * Removes an object from the Hash Map, rehashing if required.
+ *
+ * Returns 0 on success and -1 if the object wasn't found.
+ *
+ * If `old` is set, the existing object (if any) will be copied to the location
+ * pointed to by `old`.
+ *
+ * NOTE: This is the function's Hash Map variant. See FIO_SET_KEY_TYPE.
+ */
+FIO_FUNC inline int FIO_NAME(remove2)(FIO_NAME(s) * set,
                                       const FIO_SET_HASH_TYPE hash_value,
                                       FIO_SET_KEY_TYPE key,
-                                      FIO_SET_OBJ_TYPE obj) {
-  FIO_NAME(_insert_or_overwrite_)
-  (set, hash_value, (FIO_SET_TYPE){.key = key, .obj = obj}, 1);
+                                      FIO_SET_OBJ_TYPE *old) {
+  if (FIO_SET_HASH_COMPARE(hash_value, FIO_SET_HASH_INVALID))
+    return -1;
+  FIO_NAME(_map_s_) *pos =
+      FIO_NAME(_find_map_pos_)(set, hash_value, (FIO_SET_TYPE){.key = key});
+  if (!pos || !pos->pos)
+    return -1;
+  if (old)
+    FIO_SET_OBJ_COPY((*old), pos->pos->obj.obj);
+  FIO_SET_DESTROY(pos->pos->obj);
+  --set->count;
+  pos->pos->hash = FIO_SET_HASH_INVALID;
+  if (pos->pos == set->pos + set->ordered - 1) {
+    do {
+      --set->pos;
+    } while (set->pos && FIO_SET_HASH_COMPARE(set->ordered[set->pos - 1].hash,
+                                              FIO_SET_HASH_INVALID));
+  }
+  pos->pos = NULL; /* leave pos->hash set to mark "hole" */
+  return 0;
+}
+
+FIO_FUNC inline int FIO_NAME(remove)(FIO_NAME(s) * set,
+                                     const FIO_SET_HASH_TYPE hash_value,
+                                     FIO_SET_KEY_TYPE key) {
+  return FIO_NAME(remove2)(set, hash_value, key, NULL);
 }
 
 #else
 
 /** Locates an object in the Set, if it exists. */
-FIO_FUNC inline FIO_SET_OBJ_TYPE *
-FIO_NAME(find)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
-               FIO_SET_OBJ_TYPE obj) {
+FIO_FUNC FIO_SET_OBJ_TYPE *FIO_NAME(find)(FIO_NAME(s) * set,
+                                          const FIO_SET_HASH_TYPE hash_value,
+                                          FIO_SET_OBJ_TYPE obj) {
   FIO_NAME(_map_s_) *pos = FIO_NAME(_find_map_pos_)(set, hash_value, obj);
   if (!pos || !pos->pos)
     return NULL;
@@ -4530,10 +4765,10 @@ FIO_NAME(find)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
  * If the object already exists in the set, than the new object will be
  * destroyed and the old object's address will be returned.
  */
-FIO_FUNC inline FIO_SET_OBJ_TYPE *
-FIO_NAME(insert)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
-                 FIO_SET_OBJ_TYPE obj) {
-  return FIO_NAME(_insert_or_overwrite_)(set, hash_value, obj, 0);
+FIO_FUNC FIO_SET_OBJ_TYPE *FIO_NAME(insert)(FIO_NAME(s) * set,
+                                            const FIO_SET_HASH_TYPE hash_value,
+                                            FIO_SET_OBJ_TYPE obj) {
+  return FIO_NAME(_insert_or_overwrite_)(set, hash_value, obj, 0, NULL);
 }
 
 /**
@@ -4543,35 +4778,34 @@ FIO_NAME(insert)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
  * If the object already exists in the set, it will be destroyed and
  * overwritten.
  */
-FIO_FUNC inline FIO_SET_OBJ_TYPE *
+FIO_FUNC FIO_SET_OBJ_TYPE *
 FIO_NAME(overwrite)(FIO_NAME(s) * set, const FIO_SET_HASH_TYPE hash_value,
                     FIO_SET_OBJ_TYPE obj) {
-  return FIO_NAME(_insert_or_overwrite_)(set, hash_value, obj, 1);
+  return FIO_NAME(_insert_or_overwrite_)(set, hash_value, obj, 1, NULL);
 }
 
-#endif
+/**
+ * The same as `overwrite`, only it copies the old object (if any) to the
+ * location pointed to by `old`.
+ *
+ * When setting `old` to NULL, the function behaves the same as `overwrite`.
+ */
+FIO_FUNC FIO_SET_OBJ_TYPE *FIO_NAME(replace)(FIO_NAME(s) * set,
+                                             const FIO_SET_HASH_TYPE hash_value,
+                                             FIO_SET_OBJ_TYPE obj,
+                                             FIO_SET_OBJ_TYPE *old) {
+  return FIO_NAME(_insert_or_overwrite_)(set, hash_value, obj, 1, old);
+}
 
 /**
  * Removes an object from the Set, rehashing if required.
  */
-#ifdef FIO_SET_KEY_TYPE
-
-FIO_FUNC inline int FIO_NAME(remove)(FIO_NAME(s) * set,
-                                     const FIO_SET_HASH_TYPE hash_value,
-                                     FIO_SET_KEY_TYPE key) {
-#else
-FIO_FUNC inline int FIO_NAME(remove)(FIO_NAME(s) * set,
-                                     const FIO_SET_HASH_TYPE hash_value,
-                                     FIO_SET_OBJ_TYPE obj) {
-#endif
+FIO_FUNC int FIO_NAME(remove)(FIO_NAME(s) * set,
+                              const FIO_SET_HASH_TYPE hash_value,
+                              FIO_SET_OBJ_TYPE obj) {
   if (FIO_SET_HASH_COMPARE(hash_value, FIO_SET_HASH_INVALID))
     return -1;
-#ifdef FIO_SET_KEY_TYPE
-  FIO_NAME(_map_s_) *pos =
-      FIO_NAME(_find_map_pos_)(set, hash_value, (FIO_SET_TYPE){.key = key});
-#else
   FIO_NAME(_map_s_) *pos = FIO_NAME(_find_map_pos_)(set, hash_value, obj);
-#endif
   if (!pos || !pos->pos)
     return -1;
   FIO_SET_DESTROY(pos->pos->obj);
@@ -4587,6 +4821,8 @@ FIO_FUNC inline int FIO_NAME(remove)(FIO_NAME(s) * set,
   return 0;
 }
 
+#endif
+
 /**
  * Allows a peak at the Set's last element.
  *
@@ -4638,7 +4874,7 @@ FIO_FUNC inline size_t FIO_NAME(capa_require)(FIO_NAME(s) * set,
   if (min_capa <= FIO_NAME(capa)(set))
     return FIO_NAME(capa)(set);
   set->mask = 1;
-  while (min_capa >= set->mask) {
+  while (min_capa > set->mask) {
     set->mask = (set->mask << 1) | 1;
   }
   FIO_NAME(rehash)(set);