geoip2_c 0.3.2 → 0.3.4

data/ext/geoip2/libmaxminddb/src/data-pool.c

@@ -0,0 +1,180 @@
+#include "data-pool.h"
+#include "maxminddb.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+static bool can_multiply(size_t const, size_t const, size_t const);
+
+// Allocate an MMDB_data_pool_s. It initially has space for size
+// MMDB_entry_data_list_s structs.
+MMDB_data_pool_s *data_pool_new(size_t const size)
+{
+    MMDB_data_pool_s *const pool = calloc(1, sizeof(MMDB_data_pool_s));
+    if (!pool) {
+        return NULL;
+    }
+
+    if (size == 0 ||
+        !can_multiply(SIZE_MAX, size, sizeof(MMDB_entry_data_list_s))) {
+        data_pool_destroy(pool);
+        return NULL;
+    }
+    pool->size = size;
+    pool->blocks[0] = calloc(pool->size, sizeof(MMDB_entry_data_list_s));
+    if (!pool->blocks[0]) {
+        data_pool_destroy(pool);
+        return NULL;
+    }
+    pool->blocks[0]->pool = pool;
+
+    pool->sizes[0] = size;
+
+    pool->block = pool->blocks[0];
+
+    return pool;
+}
+
+// Determine if we can multiply m*n. We can do this if the result will be below
+// the given max. max will typically be SIZE_MAX.
+//
+// We want to know if we'll wrap around.
+static bool can_multiply(size_t const max, size_t const m, size_t const n)
+{
+    if (m == 0) {
+        return false;
+    }
+
+    return n <= max / m;
+}
+
+// Clean up the data pool.
+void data_pool_destroy(MMDB_data_pool_s *const pool)
+{
+    if (!pool) {
+        return;
+    }
+
+    for (size_t i = 0; i <= pool->index; i++) {
+        free(pool->blocks[i]);
+    }
+
+    free(pool);
+}
+
+// Claim a new struct from the pool. Doing this may cause the pool's size to
+// grow.
+MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const pool)
+{
+    if (!pool) {
+        return NULL;
+    }
+
+    if (pool->used < pool->size) {
+        MMDB_entry_data_list_s *const element = pool->block + pool->used;
+        pool->used++;
+        return element;
+    }
+
+    // Take it from a new block of memory.
+
+    size_t const new_index = pool->index + 1;
+    if (new_index == DATA_POOL_NUM_BLOCKS) {
+        // See the comment about not growing this on DATA_POOL_NUM_BLOCKS.
+        return NULL;
+    }
+
+    if (!can_multiply(SIZE_MAX, pool->size, 2)) {
+        return NULL;
+    }
+    size_t const new_size = pool->size * 2;
+
+    if (!can_multiply(SIZE_MAX, new_size, sizeof(MMDB_entry_data_list_s))) {
+        return NULL;
+    }
+    pool->blocks[new_index] = calloc(new_size, sizeof(MMDB_entry_data_list_s));
+    if (!pool->blocks[new_index]) {
+        return NULL;
+    }
+
+    // We don't need to set this, but it's useful for introspection in tests.
+    pool->blocks[new_index]->pool = pool;
+
+    pool->index = new_index;
+    pool->block = pool->blocks[pool->index];
+
+    pool->size = new_size;
+    pool->sizes[pool->index] = pool->size;
+
+    MMDB_entry_data_list_s *const element = pool->block;
+    pool->used = 1;
+    return element;
+}
+
+// Turn the structs in the array-like pool into a linked list.
+//
+// Before calling this function, the list isn't linked up.
+MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const pool)
+{
+    if (!pool) {
+        return NULL;
+    }
+
+    if (pool->index == 0 && pool->used == 0) {
+        return NULL;
+    }
+
+    for (size_t i = 0; i <= pool->index; i++) {
+        MMDB_entry_data_list_s *const block = pool->blocks[i];
+
+        size_t size = pool->sizes[i];
+        if (i == pool->index) {
+            size = pool->used;
+        }
+
+        for (size_t j = 0; j < size - 1; j++) {
+            MMDB_entry_data_list_s *const cur = block + j;
+            cur->next = block + j + 1;
+        }
+
+        if (i < pool->index) {
+            MMDB_entry_data_list_s *const last = block + size - 1;
+            last->next = pool->blocks[i + 1];
+        }
+    }
+
+    return pool->blocks[0];
+}
+
+#ifdef TEST_DATA_POOL
+
+#include <libtap/tap.h>
+#include <maxminddb_test_helper.h>
+
+static void test_can_multiply(void);
+
+int main(void)
+{
+    plan(NO_PLAN);
+    test_can_multiply();
+    done_testing();
+}
+
+static void test_can_multiply(void)
+{
+    {
+        ok(can_multiply(SIZE_MAX, 1, SIZE_MAX), "1*SIZE_MAX is ok");
+    }
+
+    {
+        ok(!can_multiply(SIZE_MAX, 2, SIZE_MAX), "2*SIZE_MAX is not ok");
+    }
+
+    {
+        ok(can_multiply(SIZE_MAX, 10240, sizeof(MMDB_entry_data_list_s)),
+           "1024 entry_data_list_s's are okay");
+    }
+}
+
+#endif

data/ext/geoip2/libmaxminddb/src/data-pool.h

@@ -0,0 +1,52 @@
+#ifndef DATA_POOL_H
+#define DATA_POOL_H
+
+#include "maxminddb.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+// This should be large enough that we never need to grow the array of pointers
+// to blocks. 32 is enough. Even starting out of with size 1 (1 struct), the
+// 32nd element alone will provide 2**32 structs as we exponentially increase
+// the number in each block. Being confident that we do not have to grow the
+// array lets us avoid writing code to do that. That code would be risky as it
+// would rarely be hit and likely not be well tested.
+#define DATA_POOL_NUM_BLOCKS 32
+
+// A pool of memory for MMDB_entry_data_list_s structs. This is so we can
+// allocate multiple up front rather than one at a time for performance
+// reasons.
+//
+// The order you add elements to it (by calling data_pool_alloc()) ends up as
+// the order of the list.
+//
+// The memory only grows. There is no support for releasing an element you take
+// back to the pool.
+typedef struct MMDB_data_pool_s {
+    // Index of the current block we're allocating out of.
+    size_t index;
+
+    // The size of the current block, counting by structs.
+    size_t size;
+
+    // How many used in the current block, counting by structs.
+    size_t used;
+
+    // The current block we're allocating out of.
+    MMDB_entry_data_list_s *block;
+
+    // The size of each block.
+    size_t sizes[DATA_POOL_NUM_BLOCKS];
+
+    // An array of pointers to blocks of memory holding space for list
+    // elements.
+    MMDB_entry_data_list_s *blocks[DATA_POOL_NUM_BLOCKS];
+} MMDB_data_pool_s;
+
+MMDB_data_pool_s *data_pool_new(size_t const);
+void data_pool_destroy(MMDB_data_pool_s *const);
+MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const);
+MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const);
+
+#endif

data/ext/geoip2/libmaxminddb/src/maxminddb.c

@@ -1,6 +1,7 @@
 #if HAVE_CONFIG_H
 #include <config.h>
 #endif
+#include "data-pool.h"
 #include "maxminddb.h"
 #include "maxminddb-compat-util.h"
 #include <assert.h>
@@ -130,6 +131,9 @@ typedef struct record_info_s {
 /* This is 128kb */
 #define METADATA_BLOCK_MAX_SIZE 131072
 
+// 64 leads us to allocating 4 KiB on a 64bit system.
+#define MMDB_POOL_INIT_SIZE 64
+
 /* *INDENT-OFF* */
 /* --prototypes automatically generated by dev-bin/regen-prototypes.pl - don't remove this comment */
 LOCAL int map_file(MMDB_s *const mmdb);
@@ -176,8 +180,10 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
 LOCAL int get_ext_type(int raw_ext_type);
 LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr,
                             int ptr_size);
-LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
+LOCAL int get_entry_data_list(MMDB_s *mmdb,
+                              uint32_t offset,
                               MMDB_entry_data_list_s *const entry_data_list,
+                              MMDB_data_pool_s *const pool,
                               int depth);
 LOCAL float get_ieee754_float(const uint8_t *restrict p);
 LOCAL double get_ieee754_double(const uint8_t *restrict p);
@@ -186,7 +192,6 @@ LOCAL uint32_t get_uint24(const uint8_t *p);
 LOCAL uint32_t get_uint16(const uint8_t *p);
 LOCAL uint64_t get_uintX(const uint8_t *p, int length);
 LOCAL int32_t get_sintX(const uint8_t *p, int length);
-LOCAL MMDB_entry_data_list_s *new_entry_data_list(void);
 LOCAL void free_mmdb_struct(MMDB_s *const mmdb);
 LOCAL void free_languages_metadata(MMDB_s *mmdb);
 LOCAL void free_descriptions_metadata(MMDB_s *mmdb);
@@ -241,7 +246,7 @@ int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
     }
     mmdb->flags = flags;
 
-    if (MMDB_SUCCESS != (status = map_file(mmdb)) ) {
+    if (MMDB_SUCCESS != (status = map_file(mmdb))) {
         goto cleanup;
     }
 
@@ -297,6 +302,15 @@ int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
     mmdb->ipv4_start_node.node_value = 0;
     mmdb->ipv4_start_node.netmask = 0;
 
+    // We do this immediately as otherwise there is a race to set
+    // ipv4_start_node.node_value and ipv4_start_node.netmask.
+    if (mmdb->metadata.ip_version == 6) {
+        status = find_ipv4_start_node(mmdb);
+        if (status != MMDB_SUCCESS) {
+            goto cleanup;
+        }
+    }
+
  cleanup:
     if (MMDB_SUCCESS != status) {
         int saved_errno = errno;
@@ -361,7 +375,11 @@ LOCAL int map_file(MMDB_s *const mmdb)
     ssize_t size;
     int status = MMDB_SUCCESS;
 
-    int fd = open(mmdb->filename, O_RDONLY);
+    int flags = O_RDONLY;
+#ifdef O_CLOEXEC
+    flags |= O_CLOEXEC;
+#endif
+    int fd = open(mmdb->filename, flags);
     struct stat s;
     if (fd < 0 || fstat(fd, &s)) {
         status = MMDB_FILE_OPEN_ERROR;
@@ -1634,15 +1652,33 @@ int MMDB_get_metadata_as_entry_data_list(
 int MMDB_get_entry_data_list(
     MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list)
 {
-    *entry_data_list = new_entry_data_list();
-    if (NULL == *entry_data_list) {
+    MMDB_data_pool_s *const pool = data_pool_new(MMDB_POOL_INIT_SIZE);
+    if (!pool) {
+        return MMDB_OUT_OF_MEMORY_ERROR;
+    }
+
+    MMDB_entry_data_list_s *const list = data_pool_alloc(pool);
+    if (!list) {
+        data_pool_destroy(pool);
+        return MMDB_OUT_OF_MEMORY_ERROR;
+    }
+
+    int const status = get_entry_data_list(start->mmdb, start->offset, list,
+                                           pool, 0);
+
+    *entry_data_list = data_pool_to_list(pool);
+    if (!*entry_data_list) {
+        data_pool_destroy(pool);
         return MMDB_OUT_OF_MEMORY_ERROR;
     }
-    return get_entry_data_list(start->mmdb, start->offset, *entry_data_list, 0);
+
+    return status;
 }
 
-LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
+LOCAL int get_entry_data_list(MMDB_s *mmdb,
+                              uint32_t offset,
                               MMDB_entry_data_list_s *const entry_data_list,
+                              MMDB_data_pool_s *const pool,
                               int depth)
 {
     if (depth >= MAXIMUM_DATA_STRUCTURE_DEPTH) {
@@ -1672,7 +1708,7 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
 
                 int status =
                     get_entry_data_list(mmdb, last_offset, entry_data_list,
-                                        depth);
+                                        pool, depth);
                 if (MMDB_SUCCESS != status) {
                     DEBUG_MSG("get_entry_data_list on pointer failed.");
                     return status;
@@ -1685,26 +1721,22 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
         {
             uint32_t array_size = entry_data_list->entry_data.data_size;
             uint32_t array_offset = entry_data_list->entry_data.offset_to_next;
-            MMDB_entry_data_list_s *previous = entry_data_list;
             while (array_size-- > 0) {
-                MMDB_entry_data_list_s *entry_data_list_to = previous->next =
-                                                                 new_entry_data_list();
-                if (NULL == entry_data_list_to) {
+                MMDB_entry_data_list_s *entry_data_list_to =
+                    data_pool_alloc(pool);
+                if (!entry_data_list_to) {
                     return MMDB_OUT_OF_MEMORY_ERROR;
                 }
 
                 int status =
                     get_entry_data_list(mmdb, array_offset, entry_data_list_to,
-                                        depth);
+                                        pool, depth);
                 if (MMDB_SUCCESS != status) {
                     DEBUG_MSG("get_entry_data_list on array element failed.");
                     return status;
                 }
 
                 array_offset = entry_data_list_to->entry_data.offset_to_next;
-                while (previous->next) {
-                    previous = previous->next;
-                }
             }
             entry_data_list->entry_data.offset_to_next = array_offset;
 
@@ -1715,45 +1747,33 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
             uint32_t size = entry_data_list->entry_data.data_size;
 
             offset = entry_data_list->entry_data.offset_to_next;
-            MMDB_entry_data_list_s *previous = entry_data_list;
             while (size-- > 0) {
-                MMDB_entry_data_list_s *entry_data_list_to = previous->next =
-                                                                 new_entry_data_list();
-                if (NULL == entry_data_list_to) {
+                MMDB_entry_data_list_s *list_key = data_pool_alloc(pool);
+                if (!list_key) {
                     return MMDB_OUT_OF_MEMORY_ERROR;
                 }
 
                 int status =
-                    get_entry_data_list(mmdb, offset, entry_data_list_to,
-                                        depth);
+                    get_entry_data_list(mmdb, offset, list_key, pool, depth);
                 if (MMDB_SUCCESS != status) {
                     DEBUG_MSG("get_entry_data_list on map key failed.");
                     return status;
                 }
 
-                while (previous->next) {
-                    previous = previous->next;
-                }
-
-                offset = entry_data_list_to->entry_data.offset_to_next;
-                entry_data_list_to = previous->next =
-                                         new_entry_data_list();
+                offset = list_key->entry_data.offset_to_next;
 
-                if (NULL == entry_data_list_to) {
+                MMDB_entry_data_list_s *list_value = data_pool_alloc(pool);
+                if (!list_value) {
                     return MMDB_OUT_OF_MEMORY_ERROR;
                 }
 
-                status = get_entry_data_list(mmdb, offset, entry_data_list_to,
+                status = get_entry_data_list(mmdb, offset, list_value, pool,
                                              depth);
                 if (MMDB_SUCCESS != status) {
                     DEBUG_MSG("get_entry_data_list on map element failed.");
                     return status;
                 }
-
-                while (previous->next) {
-                    previous = previous->next;
-                }
-                offset = entry_data_list_to->entry_data.offset_to_next;
+                offset = list_value->entry_data.offset_to_next;
             }
             entry_data_list->entry_data.offset_to_next = offset;
         }
@@ -1832,22 +1852,12 @@ LOCAL int32_t get_sintX(const uint8_t *p, int length)
     return (int32_t)get_uintX(p, length);
 }
 
-LOCAL MMDB_entry_data_list_s *new_entry_data_list(void)
-{
-    /* We need calloc here in order to ensure that the ->next pointer in the
-     * struct doesn't point to some random address. */
-    return calloc(1, sizeof(MMDB_entry_data_list_s));
-}
-
 void MMDB_free_entry_data_list(MMDB_entry_data_list_s *const entry_data_list)
 {
     if (entry_data_list == NULL) {
         return;
     }
-    if (entry_data_list->next) {
-        MMDB_free_entry_data_list(entry_data_list->next);
-    }
-    free(entry_data_list);
+    data_pool_destroy(entry_data_list->pool);
 }
 
 void MMDB_close(MMDB_s *const mmdb)

data/ext/geoip2/libmaxminddb/t/Makefile.am

@@ -12,12 +12,16 @@ libmmdbtest_la_SOURCES = maxminddb_test_helper.c
 
 check_PROGRAMS = \
 	bad_pointers_t bad_databases_t basic_lookup_t data_entry_list_t \
-    data_types_t dump_t get_value_t get_value_pointer_bug_t \
+    data-pool-t data_types_t dump_t get_value_t get_value_pointer_bug_t \
 	ipv4_start_cache_t ipv6_lookup_in_ipv4_t metadata_t metadata_pointers_t \
 	no_map_get_value_t read_node_t threads_t version_t
 
+data_pool_t_CFLAGS = $(CFLAGS) -I$(top_srcdir)/src
+data_pool_t_LDFLAGS = $(AM_LDFLAGS) -lm
+data_pool_t_SOURCES = data-pool-t.c ../src/data-pool.c
+
 threads_t_CFLAGS = $(CFLAGS) -pthread
 
-TESTS = $(check_PROGRAMS) compile_c++_t.pl mmdblookup_t.pl
+TESTS = $(check_PROGRAMS) compile_c++_t.pl external_symbols_t.pl mmdblookup_t.pl
 
 LDADD = libmmdbtest.la libtap/libtap.a

data/ext/geoip2/libmaxminddb/t/bad_databases_t.c

@@ -26,6 +26,7 @@ int test_read(const char *path, const struct stat *UNUSED(
 
     if (status != MMDB_SUCCESS) {
         ok(1, "received error when opening %s", path);
+        free(mmdb);
         return 0;
     }
 

data/ext/geoip2/libmaxminddb/t/basic_lookup_t.c

@@ -1,5 +1,7 @@
 #include "maxminddb_test_helper.h"
 
+static void test_big_lookup(void);
+
 /* These globals are gross but it's the easiest way to mix calling
  * for_all_modes() and for_all_record_sizes() */
 static int Current_Mode;
@@ -164,9 +166,42 @@ void all_record_sizes(int mode, const char *description)
     }
 }
 
+static void test_big_lookup(void)
+{
+    const char *const db_filename = "GeoIP2-Precision-Enterprise-Test.mmdb";
+    const char *const db_path = test_database_path(db_filename);
+    ok(db_path != NULL, "got database path");
+
+    MMDB_s * const mmdb = open_ok(db_path, MMDB_MODE_MMAP, "mmap mode");
+    ok(mmdb != NULL, "opened MMDB");
+    free((char *)db_path);
+
+    int gai_err = 0, mmdb_err = 0;
+    const char *const ip_address = "81.2.69.160";
+    MMDB_lookup_result_s result = MMDB_lookup_string(mmdb, ip_address, &gai_err,
+                                                     &mmdb_err);
+    ok(gai_err == 0, "no getaddrinfo error");
+    ok(mmdb_err == MMDB_SUCCESS, "no error from maxminddb library");
+    ok(result.found_entry, "found IP");
+
+    MMDB_entry_data_list_s *entry_data_list = NULL;
+    ok(
+        MMDB_get_entry_data_list(&result.entry,
+                                 &entry_data_list) == MMDB_SUCCESS,
+        "successfully looked up entry data list"
+        );
+    ok(entry_data_list != NULL, "got an entry_data_list");
+
+    MMDB_free_entry_data_list(entry_data_list);
+
+    MMDB_close(mmdb);
+    free(mmdb);
+}
+
 int main(void)
 {
     plan(NO_PLAN);
     for_all_modes(&all_record_sizes);
+    test_big_lookup();
     done_testing();
 }