iodine 0.4.19 → 0.5.0

data/ext/iodine/fio_mem.h

@@ -0,0 +1,140 @@
+/*
+Copyright: Boaz Segev, 2018
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_FIO_MEM_H
+
+/**
+ * This is a custom memory allocator the utilizes memory pools to allow for
+ * concurrent memory allocations across threads.
+ *
+ * Allocated memory is always zeroed out and aligned on a 16 byte boundary.
+ *
+ * Reallocated memory is always aligned on a 16 byte boundary but it might be
+ * filled with junk data after the valid data (this is true also for
+ * `fio_realloc2`).
+ *
+ * The memory allocator assumes multiple concurrent allocation/deallocation,
+ * short life spans (memory is freed shortly, but not immediately, after it was
+ * allocated) and we small allocations (realloc almost always copies data).
+ *
+ * These assumptions allow the allocator to avoid lock contention by ignoring
+ * fragmentation within a memory "block" and waiting for the whole "block" to be
+ * freed before it's memory is recycled (no "free list").
+ *
+ * This allocator should NOT be used for objects with a long life-span, because
+ * even a single persistent object will prevent the re-use of the whole memory
+ * block (128Kb by default) from which it was allocated.
+ *
+ * A memory "block" can include any number of memory pages that are a multiple
+ * of 2 (up to 1Mb of memory). However, the default value, set by
+ * MEMORY_BLOCK_SIZE, is either 128Kb (set at th end of this header).
+ *
+ * Each block includes a header that uses reference counters and position
+ * markers.
+ *
+ * The position marker (`pos`) marks the next available byte (counted in
+ * multiples of 16).
+ *
+ * The reference counter (`ref`) counts how many pointers reference memory in
+ * the block (including the "arena" that "owns" the block).
+ *
+ * Except for the position marker (`pos`) that acts the same as `sbrk`, there's
+ * no way to know which "slices" are allocated and which "slices" are available.
+ *
+ * Small allocations are differentiated by their memory alignment. If a memory
+ * allocation is placed 8 bytes after whole block alignment, the memory was
+ * allocated directly using `mmap` (and it might be using a whole page more than
+ * request, just because it needed that extra header space!).
+ *
+ * The allocator uses `mmap` when requesting memory from the system and for
+ * allocations bigger than MEMORY_BLOCK_ALLOC_LIMIT (37.5% of the block).
+ *
+ * To replace the system's `malloc` function family compile with the
+ * `FIO_OVERRIDE_MALLOC` defined (`-DFIO_OVERRIDE_MALLOC`).
+ *
+ * When using tcmalloc or jemalloc, define `FIO_FORCE_MALLOC` to prevent
+ * `fio_mem` from compiling (`-DFIO_FORCE_MALLOC`). Function wrappers will be
+ * compiled just in case, so calls to `fio_malloc` will be routed to `malloc`.
+ *
+ */
+#define H_FIO_MEM_H
+
+#include <stdlib.h>
+
+/** Allocates memory using a per-CPU core block memory pool. */
+void *fio_malloc(size_t size);
+
+/** Allocates memory using a per-CPU core block memory pool. memory is zeroed
+ * out. */
+void *fio_calloc(size_t size, size_t count);
+
+/** Frees memory that was allocated using this library. */
+void fio_free(void *ptr);
+
+/**
+ * Re-allocates memory. An attept to avoid copying the data is made only for
+ * memory allocations larger than 64Kb.
+ */
+void *fio_realloc(void *ptr, size_t new_size);
+
+/**
+ * Re-allocates memory. An attept to avoid copying the data is made only for
+ * memory allocations larger than 64Kb.
+ *
+ * This variation is slightly faster as it might copy less data.
+ */
+void *fio_realloc2(void *ptr, size_t new_size, size_t copy_length);
+
+/** Clears any memory locks, in case of a system call to `fork`. */
+void fio_malloc_after_fork(void);
+
+/** Tests the facil.io memory allocator. */
+void fio_malloc_test(void);
+
+/** If defined, `malloc` will be used instead of the fio_malloc functions */
+#if FIO_FORCE_MALLOC
+#define fio_malloc malloc
+#define fio_calloc calloc
+#define fio_free free
+#define fio_realloc realloc
+#define fio_realloc2(ptr, new_size, old_data_len) realloc((ptr), (new_size))
+#define fio_malloc_test()
+#define fio_malloc_after_fork
+
+/* allows local override as well as global override */
+#elif FIO_OVERRIDE_MALLOC
+#define malloc fio_malloc
+#define free fio_free
+#define realloc fio_realloc
+#define calloc fio_calloc
+
+#endif
+
+#ifndef FIO_MEM_MAX_BLOCKS_PER_CORE
+/**
+ * The maximum number of available memory blocks that will be pooled before
+ * memory is returned to the system.
+ */
+#define FIO_MEM_MAX_BLOCKS_PER_CORE 32 /* approx. 2Mb per CPU core */
+#endif
+
+/** Allocator default settings. */
+#ifndef FIO_MEMORY_BLOCK_SIZE
+#define FIO_MEMORY_BLOCK_SIZE ((uintptr_t)1 << 17) /* 17 == 128Kb */
+#endif
+#ifndef FIO_MEMORY_BLOCK_MASK
+#define FIO_MEMORY_BLOCK_MASK (FIO_MEMORY_BLOCK_SIZE - 1) /* 0b111... */
+#endif
+#ifndef FIO_MEMORY_BLOCK_SLICES
+#define FIO_MEMORY_BLOCK_SLICES (FIO_MEMORY_BLOCK_SIZE >> 4) /* 16B/slice */
+#endif
+#ifndef FIO_MEMORY_BLOCK_ALLOC_LIMIT
+/* defaults to 37.5% of the block */
+#define FIO_MEMORY_BLOCK_ALLOC_LIMIT                                           \
+  ((FIO_MEMORY_BLOCK_SIZE >> 2) + (FIO_MEMORY_BLOCK_SIZE >> 3))
+#endif
+
+#endif /* H_FIO_MEM_H */

data/ext/iodine/fio_random.c

@@ -0,0 +1,248 @@
+/*
+Copyright: Boaz segev, 2016-2018
+License: MIT except for any non-public-domain algorithms (none that I'm aware
+of), which might be subject to their own licenses.
+
+Feel free to copy, use and enjoy in accordance with to the license(s).
+*/
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include "fio_random.h"
+
+#include <errno.h>
+#include <stdio.h>
+
+#ifndef __has_include
+#define __has_include(x) 0
+#endif
+
+/* include intrinsics if supported */
+#if __has_include(<x86intrin.h>)
+#include <x86intrin.h>
+#define HAVE_X86Intrin
+/*
+see: https://software.intel.com/en-us/node/513411
+and: https://software.intel.com/sites/landingpage/IntrinsicsGuide/
+*/
+#endif /* __has_include(<x86intrin.h>) */
+
+/* check for unix support */
+#if defined(__unix__) || defined(__linux__) || defined(__APPLE__) ||           \
+    (__has_include(<unistd.h>) && __has_include(<pthread.h>))
+#ifndef HAS_UNIX_FEATURES
+#define HAS_UNIX_FEATURES
+#endif
+#endif
+
+// clang-format off
+#if !defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)
+#   if defined(__has_include)
+#     if __has_include(<endian.h>)
+#      include <endian.h>
+#     elif __has_include(<sys/endian.h>)
+#      include <sys/endian.h>
+#     endif
+#   endif
+#   if !defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__) && \
+                __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#      define __BIG_ENDIAN__
+#   endif
+#endif
+// clang-format on
+
+#if defined(USE_ALT_RANDOM) || !defined(HAS_UNIX_FEATURES)
+#include "fio_sha2.h"
+#include <time.h>
+#include <string.h>
+
+static inline void fio_random_data(sha2_s *sha2) {
+#ifdef RUSAGE_SELF
+  struct rusage rusage;
+  getrusage(RUSAGE_SELF, &rusage);
+  fio_sha2_write(sha2, &rusage, sizeof(rusage));
+#elif defined CLOCKS_PER_SEC
+  size_t clk = (size_t)clock();
+  fio_sha2_write(&sha2, &clk, sizeof(clk));
+  time_t the_time;
+  time(&the_time);
+  fio_sha2_write(sha2, &the_time, sizeof(the_time));
+  fio_sha2_write(sha2, &fio_rand64, sizeof(void *));
+  fio_sha2_write(sha2, &fio_rand_bytes, sizeof(void *));
+  {
+    char junk_data[64];
+    fio_sha2_write(sha2, junk_data, 64);
+  }
+#else
+#error Random alternative failed to find access to the CPU clock state.
+#endif
+  fio_sha2_write(sha2, sha2, sizeof(void *));
+}
+
+uint32_t fio_rand32(void) {
+  sha2_s sha2 = fio_sha2_init(SHA_512);
+  fio_random_data(&sha2);
+  fio_sha2_result(&sha2);
+  return *((uint32_t *)sha2.buffer);
+}
+
+uint64_t fio_rand64(void) {
+  sha2_s sha2 = fio_sha2_init(SHA_512);
+  fio_random_data(&sha2);
+  fio_sha2_result(&sha2);
+  return *((uint64_t *)sha2.buffer);
+}
+
+void fio_rand_bytes(void *target, size_t length) {
+  sha2_s sha2 = fio_sha2_init(SHA_512);
+  fio_random_data(&sha2);
+  fio_sha2_result(&sha2);
+
+  while (length >= 64) {
+    memcpy(target, sha2.digest.str, 64);
+    length -= 64;
+    target = (void *)((uintptr_t)target + 64);
+    fio_random_data(&sha2);
+    fio_sha2_result(&sha2);
+  }
+  if (length >= 32) {
+    memcpy(target, sha2.digest.str, 32);
+    length -= 32;
+    target = (void *)((uintptr_t)target + 32);
+    fio_random_data(&sha2);
+    fio_sha2_result(&sha2);
+  }
+  if (length >= 16) {
+    memcpy(target, sha2.digest.str, 16);
+    length -= 16;
+    target = (void *)((uintptr_t)target + 16);
+    fio_random_data(&sha2);
+    fio_sha2_result(&sha2);
+  }
+  if (length >= 8) {
+    memcpy(target, sha2.digest.str, 8);
+    length -= 8;
+    target = (void *)((uintptr_t)target + 8);
+    fio_random_data(&sha2);
+    fio_sha2_result(&sha2);
+  }
+  while (length) {
+    *((uint8_t *)target) = sha2.digest.str[length];
+    target = (void *)((uintptr_t)target + 1);
+    --length;
+  }
+}
+
+#else
+/* ***************************************************************************
+Unix Random Engine (use built in machine)
+*/
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <unistd.h>
+
+/* ***************************************************************************
+Machine specific changes
+*/
+// #ifdef __linux__
+// #undef bswap16
+// #undef bswap32
+// #undef bswap64
+// #include <machine/bswap.h>
+// #endif
+#ifdef HAVE_X86Intrin
+// #undef bswap16
+/*
+#undef bswap32
+#define bswap32(i) \
+  { __asm__("bswap %k0" : "+r"(i) :); }
+*/
+#undef bswap64
+#define bswap64(i)                                                             \
+  { __asm__("bswapq %0" : "+r"(i) :); }
+
+// shadow sched_yield as _mm_pause for spinwait
+#define sched_yield() _mm_pause()
+#endif
+
+/* ***************************************************************************
+Random fd
+***************************************************************************** */
+
+/* rand generator management */
+static int fio_rand_fd_ = -1;
+static void close_rand_fd(void) {
+  if (fio_rand_fd_ >= 0)
+    close(fio_rand_fd_);
+  fio_rand_fd_ = -1;
+}
+static void init_rand_fd(void) {
+  if (fio_rand_fd_ < 0) {
+    while ((fio_rand_fd_ = open("/dev/urandom", O_RDONLY)) == -1) {
+      if (errno == ENXIO) {
+        perror("FATAL ERROR: caanot initiate random generator");
+        exit(-1);
+      }
+      sched_yield();
+    }
+  }
+  atexit(close_rand_fd);
+}
+
+/* ***************************************************************************
+Random API ... (why is this not a system call?)
+***************************************************************************** */
+
+/* rand function template */
+#define MAKE_RAND_FUNC(type, func_name)                                        \
+  type func_name(void) {                                                       \
+    if (fio_rand_fd_ < 0)                                                      \
+      init_rand_fd();                                                          \
+    type ret;                                                                  \
+    while (read(fio_rand_fd_, &ret, sizeof(type)) < 0)                         \
+      sched_yield();                                                           \
+    return ret;                                                                \
+  }
+/* rand functions */
+MAKE_RAND_FUNC(uint32_t, fio_rand32)
+MAKE_RAND_FUNC(uint64_t, fio_rand64)
+/* clear template */
+#undef MAKE_RAND_FUNC
+
+void fio_rand_bytes(void *target, size_t length) {
+  if (fio_rand_fd_ < 0)
+    init_rand_fd();
+  while (read(fio_rand_fd_, target, length) < 0)
+    sched_yield();
+}
+#endif /* Unix Random */
+
+/*******************************************************************************
+Random Testing
+***************************************************************************** */
+#if DEBUG
+void fio_random_test(void) {
+  uint64_t buffer[8];
+  clock_t start, end;
+  fio_rand64();
+  start = clock();
+  for (size_t i = 0; i < 100000; i++) {
+    buffer[i & 7] = fio_rand64();
+  }
+  end = clock();
+  fprintf(stderr,
+          "+ Random generator available\n+ created 100K X 64bits "
+          "Random %lu CPU clock count\n",
+          end - start);
+  start = clock();
+  for (size_t i = 0; i < 100000; i++) {
+    fio_rand_bytes(buffer, 64);
+  }
+  end = clock();
+  fprintf(stderr,
+          "+ created 100K X 512bits "
+          "Random %lu CPU clock count\n",
+          end - start);
+}
+#endif

data/ext/iodine/{random.h → fio_random.h}

@@ -5,12 +5,15 @@ of), which might be subject to their own licenses.
 
 Feel free to copy, use and enjoy in accordance with to the license(s).
 */
-#ifndef bscrypt_RANDOM_H
-#define bscrypt_RANDOM_H
-#include "bscrypt-common.h"
+#ifndef fio_RANDOM_H
+#define fio_RANDOM_H
 /* *****************************************************************************
 C++ extern
 */
+
+#include <stdint.h>
+#include <stdlib.h>
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -20,22 +23,16 @@ Random stuff... (why is this not a system call?)
 */
 
 /** returns 32 random bits. */
-uint32_t bscrypt_rand32(void);
+uint32_t fio_rand32(void);
 
 /** returns 64 random bits. */
-uint64_t bscrypt_rand64(void);
-
-/** returns 128 random bits. */
-bits128_u bscrypt_rand128(void);
-
-/** returns 256 random bits. */
-bits256_u bscrypt_rand256(void);
+uint64_t fio_rand64(void);
 
 /** returns a variable length string of random bytes. */
-void bscrypt_rand_bytes(void *target, size_t length);
+void fio_rand_bytes(void *target, size_t length);
 
-#if defined(DEBUG) && DEBUG == 1
-void bscrypt_test_random(void);
+#if DEBUG
+void fio_random_test(void);
 #endif
 
 /* *****************************************************************************

data/ext/iodine/{sha1.c → fio_sha1.c}

@@ -8,7 +8,10 @@ Feel free to copy, use and enjoy in accordance with to the license(s).
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
 #endif
-#include "sha1.h"
+#include "fio_sha1.h"
+
+#include <string.h>
+
 /*****************************************************************************
 Useful Macros - Not all of them are used here, but it's a copy-paste convenience
 */
@@ -198,7 +201,7 @@ SHA-1 hashing
 Initialize or reset the `sha1` object. This must be performed before hashing
 data using sha1.
 */
-sha1_s bscrypt_sha1_init(void) {
+sha1_s fio_sha1_init(void) {
   return (sha1_s){.digest.i[0] = 0x67452301,
                   .digest.i[1] = 0xefcdab89,
                   .digest.i[2] = 0x98badcfe,
@@ -209,7 +212,7 @@ sha1_s bscrypt_sha1_init(void) {
 /**
 Writes data to the sha1 buffer.
 */
-void bscrypt_sha1_write(sha1_s *s, const void *data, size_t len) {
+void fio_sha1_write(sha1_s *s, const void *data, size_t len) {
   size_t in_buffer = s->length & 63;
   size_t partial = 64 - in_buffer;
   s->length += len;
@@ -234,7 +237,7 @@ void bscrypt_sha1_write(sha1_s *s, const void *data, size_t len) {
   return;
 }
 
-char *bscrypt_sha1_result(sha1_s *s) {
+char *fio_sha1_result(sha1_s *s) {
   size_t in_buffer = s->length & 63;
   if (in_buffer > 55) {
     memcpy(s->buffer + in_buffer, sha1_padding, 64 - in_buffer);
@@ -273,6 +276,7 @@ char *bscrypt_sha1_result(sha1_s *s) {
 SHA-1 testing
 */
 #if defined(DEBUG) && DEBUG == 1
+#include <stdio.h>
 #include <time.h>
 
 // clang-format off
@@ -281,10 +285,10 @@ SHA-1 testing
 #endif
 // clang-format on
 
-void bscrypt_test_sha1(void) {
+void fio_sha1_test(void) {
   struct {
     char *str;
-    char hash[21];
+    uint8_t hash[21];
   } sets[] = {
       {"The quick brown fox jumps over the lazy dog",
        {0x2f, 0xd4, 0xe1, 0xc6, 0x7a, 0x2d, 0x28, 0xfc, 0xed, 0x84, 0x9e,
@@ -300,20 +304,19 @@ void bscrypt_test_sha1(void) {
   int i = 0;
   sha1_s sha1;
   fprintf(stderr, "===================================\n");
-  fprintf(stderr, "bscrypt SHA-1 struct size: %lu\n", sizeof(sha1_s));
-  fprintf(stderr, "+ bscrypt");
+  fprintf(stderr, "fio SHA-1 struct size: %zu\n", sizeof(sha1_s));
+  fprintf(stderr, "+ fio");
   while (sets[i].str) {
-    sha1 = bscrypt_sha1_init();
-    bscrypt_sha1_write(&sha1, sets[i].str, strlen(sets[i].str));
-    if (strcmp(bscrypt_sha1_result(&sha1), sets[i].hash)) {
-      fprintf(stderr,
-              ":\n--- bscrypt SHA-1 Test FAILED!\nstring: %s\nexpected: ",
+    sha1 = fio_sha1_init();
+    fio_sha1_write(&sha1, sets[i].str, strlen(sets[i].str));
+    if (strcmp(fio_sha1_result(&sha1), (char *)sets[i].hash)) {
+      fprintf(stderr, ":\n--- fio SHA-1 Test FAILED!\nstring: %s\nexpected: ",
               sets[i].str);
-      char *p = sets[i].hash;
+      char *p = (char *)sets[i].hash;
       while (*p)
         fprintf(stderr, "%02x", *(p++) & 0xFF);
       fprintf(stderr, "\ngot: ");
-      p = bscrypt_sha1_result(&sha1);
+      p = fio_sha1_result(&sha1);
       while (*p)
         fprintf(stderr, "%02x", *(p++) & 0xFF);
       fprintf(stderr, "\n");
@@ -325,27 +328,28 @@ void bscrypt_test_sha1(void) {
 
 #ifdef HAVE_OPENSSL
   fprintf(stderr, "===================================\n");
-  fprintf(stderr, "bscrypt SHA-1 struct size: %lu\n", sizeof(sha1_s));
-  fprintf(stderr, "OpenSSL SHA-1 struct size: %lu\n", sizeof(SHA_CTX));
+  fprintf(stderr, "fio SHA-1 struct size: %lu\n",
+          (unsigned long)sizeof(sha1_s));
+  fprintf(stderr, "OpenSSL SHA-1 struct size: %lu\n",
+          (unsigned long)sizeof(SHA_CTX));
   fprintf(stderr, "===================================\n");
 
   unsigned char hash[SHA512_DIGEST_LENGTH + 1];
   hash[SHA512_DIGEST_LENGTH] = 0;
   clock_t start;
   start = clock();
-  for (size_t i = 0; i < 100000; i++) {
-    sha1 = bscrypt_sha1_init();
-    bscrypt_sha1_write(&sha1, "The quick brown fox jumps over the lazy dog ",
-                       43);
-    bscrypt_sha1_result(&sha1);
+  for (i = 0; i < 100000; i++) {
+    sha1 = fio_sha1_init();
+    fio_sha1_write(&sha1, "The quick brown fox jumps over the lazy dog ", 43);
+    fio_sha1_result(&sha1);
   }
-  fprintf(stderr, "bscrypt 100K SHA-1: %lf\n",
+  fprintf(stderr, "fio 100K SHA-1: %lf\n",
           (double)(clock() - start) / CLOCKS_PER_SEC);
 
   hash[SHA_DIGEST_LENGTH] = 0;
   SHA_CTX o_sh1;
   start = clock();
-  for (size_t i = 0; i < 100000; i++) {
+  for (i = 0; i < 100000; i++) {
     SHA1_Init(&o_sh1);
     SHA1_Update(&o_sh1, "The quick brown fox jumps over the lazy dog", 43);
     SHA1_Final(hash, &o_sh1);