prometheus-client-mmap 0.9.4 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 282d373787d189c268737c396aa12cec78914357
-  data.tar.gz: 2952e20d2b4297fe16826a42b53e092f48013892
+  metadata.gz: ffa6a21d29d372519ce95f9cfd004aac10fb9b4c
+  data.tar.gz: 2d8e3faa150d597c1f78d88341cd7eec6b58a6c1
 SHA512:
-  metadata.gz: 4fe32ca10b96c5ae37e4fc2d17b810c84749875e498739aefb3ada8908aec672a58448d15cb0ca00b2d190055e50b95f7382df1e09e8eb4ffb4d3c3add1e1e74
-  data.tar.gz: be318ffec99871cb2bb608e164bebb6cf4b0da5616824c606e9b63bddcdcfc5456bd804775f03a05c485d494d855097407347dcbfd1a54c91ecde5976132253e
+  metadata.gz: b971c4fa6d54421a8178c1c0d16fbe019cf878ba4574affd199d74c3aa72723f554f25eafd66f77f432d451bdd68a77118bf22938e6ea857997978f892f8d124
+  data.tar.gz: 71d75733d5107297c62f9fde343257dd9d73d079c8a7bb586d90fc5fac1f3f2c8121ac03d765a66bcccad0490c4d3b4214eee244c8345f25fa48fae87c8b0d93

data/README.md

@@ -164,6 +164,16 @@ summary.get({ service: 'database' })
 # => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
 ```
 
+## Configuration
+
+### Memory mapped files storage location
+
+Set `prometheus_multiproc_dir` environment variable to the path where you want metric files to be stored. Example:
+
+```
+prometheus_multiproc_dir=/tmp
+```
+
 ## Pitfalls
 
 ### PID cardinality

data/lib/prometheus/client/configuration.rb

@@ -1,6 +1,7 @@
 require 'prometheus/client/registry'
 require 'prometheus/client/mmaped_value'
 require 'logger'
+require 'tmpdir'
 
 module Prometheus
   module Client
@@ -9,10 +10,14 @@ module Prometheus
 
       def initialize
         @value_class = ::Prometheus::Client::MmapedValue
-        @multiprocess_files_dir = ENV['prometheus_multiproc_dir']
         @initial_mmap_file_size = 4 * 1024
         @logger = Logger.new($stdout)
         @pid_provider = Process.method(:pid)
+        @multiprocess_files_dir = ENV.fetch('prometheus_multiproc_dir') do
+          Dir.mktmpdir("prometheus-mmap").tap do |tmp|
+            @logger.warn("Please set prometheus_multiproc_dir to persistent directory path. Temporarily writing mmap files to #{tmp}")
+          end
+        end
       end
     end
   end

data/lib/prometheus/client/version.rb

@@ -1,5 +1,5 @@
 module Prometheus
   module Client
-    VERSION = '0.9.4'.freeze
+    VERSION = '0.9.5'.freeze
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prometheus-client-mmap
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.5
 platform: ruby
 authors:
 - Tobias Schmidt
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-28 00:00:00.000000000 Z
+date: 2018-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fuzzbert
@@ -92,8 +92,6 @@ files:
 - ext/fast_mmaped_file/file_reading.c
 - ext/fast_mmaped_file/file_reading.h
 - ext/fast_mmaped_file/globals.h
-- ext/fast_mmaped_file/hashmap.c
-- ext/fast_mmaped_file/jsmn.c
 - ext/fast_mmaped_file/mmap.c
 - ext/fast_mmaped_file/mmap.h
 - ext/fast_mmaped_file/rendering.c
@@ -102,7 +100,6 @@ files:
 - ext/fast_mmaped_file/utils.h
 - ext/fast_mmaped_file/value_access.c
 - ext/fast_mmaped_file/value_access.h
-- lib/fast_mmaped_file.bundle
 - lib/prometheus.rb
 - lib/prometheus/client.rb
 - lib/prometheus/client/configuration.rb

data/ext/fast_mmaped_file/hashmap.c

@@ -1,692 +0,0 @@
-/*
- * Copyright (c) 2016-2017 David Leeds <davidesleeds@gmail.com>
- *
- * Hashmap is free software; you can redistribute it and/or modify
- * it under the terms of the MIT license. See LICENSE for details.
- */
-
-#include <stdlib.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <string.h>
-#include <errno.h>
-
-#include "hashmap.h"
-
-#ifndef HASHMAP_NOASSERT
-#include <assert.h>
-#define HASHMAP_ASSERT(expr)	assert(expr)
-#else
-#define HASHMAP_ASSERT(expr)
-#endif
-
-/* Table sizes must be powers of 2 */
-#define HASHMAP_SIZE_MIN		(1 << 5)	/* 32 */
-#define HASHMAP_SIZE_DEFAULT		(1 << 8)	/* 256 */
-#define HASHMAP_SIZE_MOD(map, val)	((val) & ((map)->table_size - 1))
-
-/* Limit for probing is 1/2 of table_size */
-#define HASHMAP_PROBE_LEN(map)		((map)->table_size >> 1)
-/* Return the next linear probe index */
-#define HASHMAP_PROBE_NEXT(map, index)	 HASHMAP_SIZE_MOD(map, (index) + 1)
-
-/* Check if index b is less than or equal to index a */
-#define HASHMAP_INDEX_LE(map, a, b)	\
-	((a) == (b) || (((b) - (a)) & ((map)->table_size >> 1)) != 0)
-
-
-struct hashmap_entry {
-	void *key;
-	void *data;
-#ifdef HASHMAP_METRICS
-	size_t num_collisions;
-#endif
-};
-
-
-/*
- * Enforce a maximum 0.75 load factor.
- */
-static inline size_t hashmap_table_min_size_calc(size_t num_entries)
-{
-	return num_entries + (num_entries / 3);
-}
-
-/*
- * Calculate the optimal table size, given the specified max number
- * of elements.
- */
-static size_t hashmap_table_size_calc(size_t num_entries)
-{
-	size_t table_size;
-	size_t min_size;
-
-	table_size = hashmap_table_min_size_calc(num_entries);
-
-	/* Table size is always a power of 2 */
-	min_size = HASHMAP_SIZE_MIN;
-	while (min_size < table_size) {
-		min_size <<= 1;
-	}
-	return min_size;
-}
-
-/*
- * Get a valid hash table index from a key.
- */
-static inline size_t hashmap_calc_index(const struct hashmap *map,
-	const void *key)
-{
-	return HASHMAP_SIZE_MOD(map, map->hash(key));
-}
-
-/*
- * Return the next populated entry, starting with the specified one.
- * Returns NULL if there are no more valid entries.
- */
-static struct hashmap_entry *hashmap_entry_get_populated(
-	const struct hashmap *map, struct hashmap_entry *entry)
-{
-	for (; entry < &map->table[map->table_size]; ++entry) {
-		if (entry->key) {
-			return entry;
-		}
-	}
-	return NULL;
-}
-
-/*
- * Find the hashmap entry with the specified key, or an empty slot.
- * Returns NULL if the entire table has been searched without finding a match.
- */
-static struct hashmap_entry *hashmap_entry_find(const struct hashmap *map,
-	const void *key, bool find_empty)
-{
-	size_t i;
-	size_t index;
-	size_t probe_len = HASHMAP_PROBE_LEN(map);
-	struct hashmap_entry *entry;
-
-	index = hashmap_calc_index(map, key);
-
-	/* Linear probing */
-	for (i = 0; i < probe_len; ++i) {
-		entry = &map->table[index];
-		if (!entry->key) {
-			if (find_empty) {
-#ifdef HASHMAP_METRICS
-				entry->num_collisions = i;
-#endif
-				return entry;
-			}
-			return NULL;
-		}
-		if (map->key_compare(key, entry->key) == 0) {
-			return entry;
-		}
-		index = HASHMAP_PROBE_NEXT(map, index);
-	}
-	return NULL;
-}
-
-/*
- * Removes the specified entry and processes the proceeding entries to reduce
- * the load factor and keep the chain continuous.  This is a required
- * step for hash maps using linear probing.
- */
-static void hashmap_entry_remove(struct hashmap *map,
-	struct hashmap_entry *removed_entry)
-{
-	size_t i;
-#ifdef HASHMAP_METRICS
-	size_t removed_i = 0;
-#endif
-	size_t index;
-	size_t entry_index;
-	size_t removed_index = (removed_entry - map->table);
-	struct hashmap_entry *entry;
-
-	/* Free the key */
-	if (map->key_free) {
-		map->key_free(removed_entry->key);
-	}
-	--map->num_entries;
-
-	/* Fill the free slot in the chain */
-	index = HASHMAP_PROBE_NEXT(map, removed_index);
-	for (i = 1; i < map->table_size; ++i) {
-		entry = &map->table[index];
-		if (!entry->key) {
-			/* Reached end of chain */
-			break;
-		}
-		entry_index = hashmap_calc_index(map, entry->key);
-		/* Shift in entries with an index <= to the removed slot */
-		if (HASHMAP_INDEX_LE(map, removed_index, entry_index)) {
-#ifdef HASHMAP_METRICS
-			entry->num_collisions -= (i - removed_i);
-			removed_i = i;
-#endif
-			memcpy(removed_entry, entry, sizeof(*removed_entry));
-			removed_index = index;
-			removed_entry = entry;
-		}
-		index = HASHMAP_PROBE_NEXT(map, index);
-	}
-	/* Clear the last removed entry */
-	memset(removed_entry, 0, sizeof(*removed_entry));
-}
-
-/*
- * Reallocates the hash table to the new size and rehashes all entries.
- * new_size MUST be a power of 2.
- * Returns 0 on success and -errno on allocation or hash function failure.
- */
-static int hashmap_rehash(struct hashmap *map, size_t new_size)
-{
-	size_t old_size;
-	struct hashmap_entry *old_table;
-	struct hashmap_entry *new_table;
-	struct hashmap_entry *entry;
-	struct hashmap_entry *new_entry;
-
-	HASHMAP_ASSERT(new_size >= HASHMAP_SIZE_MIN);
-	HASHMAP_ASSERT((new_size & (new_size - 1)) == 0);
-
-	new_table = (struct hashmap_entry *)calloc(new_size,
-	    sizeof(struct hashmap_entry));
-	if (!new_table) {
-		return -ENOMEM;
-	}
-	/* Backup old elements in case of rehash failure */
-	old_size = map->table_size;
-	old_table = map->table;
-	map->table_size = new_size;
-	map->table = new_table;
-	/* Rehash */
-	for (entry = old_table; entry < &old_table[old_size]; ++entry) {
-		if (!entry->data) {
-			/* Only copy entries with data */
-			continue;
-		}
-		new_entry = hashmap_entry_find(map, entry->key, true);
-		if (!new_entry) {
-			/*
-			 * The load factor is too high with the new table
-			 * size, or a poor hash function was used.
-			 */
-			goto revert;
-		}
-		/* Shallow copy (intentionally omits num_collisions) */
-		new_entry->key = entry->key;
-		new_entry->data = entry->data;
-	}
-	free(old_table);
-	return 0;
-revert:
-	map->table_size = old_size;
-	map->table = old_table;
-	free(new_table);
-	return -EINVAL;
-}
-
-/*
- * Iterate through all entries and free all keys.
- */
-static void hashmap_free_keys(struct hashmap *map)
-{
-	struct hashmap_iter *iter;
-
-	if (!map->key_free) {
-		return;
-	}
-	for (iter = hashmap_iter(map); iter;
-	    iter = hashmap_iter_next(map, iter)) {
-		map->key_free((void *)hashmap_iter_get_key(iter));
-	}
-}
-
-/*
- * Initialize an empty hashmap.  A hash function and a key comparator are
- * required.
- *
- * hash_func should return an even distribution of numbers between 0
- * and SIZE_MAX varying on the key provided.
- *
- * key_compare_func should return 0 if the keys match, and non-zero otherwise.
- *
- * initial_size is optional, and may be set to the max number of entries
- * expected to be put in the hash table.  This is used as a hint to
- * pre-allocate the hash table to the minimum size needed to avoid
- * gratuitous rehashes.  If initial_size 0, a default size will be used.
- *
- * Returns 0 on success and -errno on failure.
- */
-int hashmap_init(struct hashmap *map, size_t (*hash_func)(const void *),
-	int (*key_compare_func)(const void *, const void *),
-	size_t initial_size)
-{
-	HASHMAP_ASSERT(map != NULL);
-	HASHMAP_ASSERT(hash_func != NULL);
-	HASHMAP_ASSERT(key_compare_func != NULL);
-
-	if (!initial_size) {
-		initial_size = HASHMAP_SIZE_DEFAULT;
-	} else {
-		/* Convert init size to valid table size */
-		initial_size = hashmap_table_size_calc(initial_size);
-	}
-	map->table_size_init = initial_size;
-	map->table_size = initial_size;
-	map->num_entries = 0;
-	map->table = (struct hashmap_entry *)calloc(initial_size,
-	    sizeof(struct hashmap_entry));
-	if (!map->table) {
-		return -ENOMEM;
-	}
-	map->hash = hash_func;
-	map->key_compare = key_compare_func;
-	map->key_alloc = NULL;
-	map->key_free = NULL;
-	return 0;
-}
-
-/*
- * Free the hashmap and all associated memory.
- */
-void hashmap_destroy(struct hashmap *map)
-{
-	if (!map) {
-		return;
-	}
-	hashmap_free_keys(map);
-	free(map->table);
-	memset(map, 0, sizeof(*map));
-}
-
-/*
- * Enable internal memory management of hash keys.
- */
-void hashmap_set_key_alloc_funcs(struct hashmap *map,
-	void *(*key_alloc_func)(const void *),
-	void (*key_free_func)(void *))
-{
-	HASHMAP_ASSERT(map != NULL);
-
-	map->key_alloc = key_alloc_func;
-	map->key_free = key_free_func;
-}
-
-/*
- * Add an entry to the hashmap.  If an entry with a matching key already
- * exists and has a data pointer associated with it, the existing data
- * pointer is returned, instead of assigning the new value.  Compare
- * the return value with the data passed in to determine if a new entry was
- * created.  Returns NULL if memory allocation failed.
- */
-void *hashmap_put(struct hashmap *map, const void *key, void *data)
-{
-	struct hashmap_entry *entry;
-
-	HASHMAP_ASSERT(map != NULL);
-	HASHMAP_ASSERT(key != NULL);
-
-	/* Rehash with 2x capacity if load factor is approaching 0.75 */
-	if (map->table_size <= hashmap_table_min_size_calc(map->num_entries)) {
-		hashmap_rehash(map, map->table_size << 1);
-	}
-	entry = hashmap_entry_find(map, key, true);
-	if (!entry) {
-		/*
-		 * Cannot find an empty slot.  Either out of memory, or using
-		 * a poor hash function.  Attempt to rehash once to reduce
-		 * chain length.
-		 */
-		if (hashmap_rehash(map, map->table_size << 1) < 0) {
-			return NULL;
-		}
-		entry = hashmap_entry_find(map, key, true);
-		if (!entry) {
-			return NULL;
-		}
-	}
-	if (!entry->key) {
-		/* Allocate copy of key to simplify memory management */
-		if (map->key_alloc) {
-			entry->key = map->key_alloc(key);
-			if (!entry->key) {
-				return NULL;
-			}
-		} else {
-			entry->key = (void *)key;
-		}
-		++map->num_entries;
-	} else if (entry->data) {
-		/* Do not overwrite existing data */
-		return entry->data;
-	}
-	entry->data = data;
-	return data;
-}
-
-/*
- * Return the data pointer, or NULL if no entry exists.
- */
-void *hashmap_get(const struct hashmap *map, const void *key)
-{
-	struct hashmap_entry *entry;
-
-	HASHMAP_ASSERT(map != NULL);
-	HASHMAP_ASSERT(key != NULL);
-
-	entry = hashmap_entry_find(map, key, false);
-	if (!entry) {
-		return NULL;
-	}
-	return entry->data;
-}
-
-/*
- * Remove an entry with the specified key from the map.
- * Returns the data pointer, or NULL, if no entry was found.
- */
-void *hashmap_remove(struct hashmap *map, const void *key)
-{
-	struct hashmap_entry *entry;
-	void *data;
-
-	HASHMAP_ASSERT(map != NULL);
-	HASHMAP_ASSERT(key != NULL);
-
-	entry = hashmap_entry_find(map, key, false);
-	if (!entry) {
-		return NULL;
-	}
-	data = entry->data;
-	/* Clear the entry and make the chain contiguous */
-	hashmap_entry_remove(map, entry);
-	return data;
-}
-
-/*
- * Remove all entries.
- */
-void hashmap_clear(struct hashmap *map)
-{
-	HASHMAP_ASSERT(map != NULL);
-
-	hashmap_free_keys(map);
-	map->num_entries = 0;
-	memset(map->table, 0, sizeof(struct hashmap_entry) * map->table_size);
-}
-
-/*
- * Remove all entries and reset the hash table to its initial size.
- */
-void hashmap_reset(struct hashmap *map)
-{
-	struct hashmap_entry *new_table;
-
-	HASHMAP_ASSERT(map != NULL);
-
-	hashmap_clear(map);
-	if (map->table_size == map->table_size_init) {
-		return;
-	}
-	new_table = (struct hashmap_entry *)realloc(map->table,
-	    sizeof(struct hashmap_entry) * map->table_size_init);
-	if (!new_table) {
-		return;
-	}
-	map->table = new_table;
-	map->table_size = map->table_size_init;
-}
-
-/*
- * Return the number of entries in the hash map.
- */
-size_t hashmap_size(const struct hashmap *map)
-{
-	HASHMAP_ASSERT(map != NULL);
-
-	return map->num_entries;
-}
-
-/*
- * Get a new hashmap iterator.  The iterator is an opaque
- * pointer that may be used with hashmap_iter_*() functions.
- * Hashmap iterators are INVALID after a put or remove operation is performed.
- * hashmap_iter_remove() allows safe removal during iteration.
- */
-struct hashmap_iter *hashmap_iter(const struct hashmap *map)
-{
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!map->num_entries) {
-		return NULL;
-	}
-	return (struct hashmap_iter *)hashmap_entry_get_populated(map,
-		map->table);
-}
-
-/*
- * Return an iterator to the next hashmap entry.  Returns NULL if there are
- * no more entries.
- */
-struct hashmap_iter *hashmap_iter_next(const struct hashmap *map,
-	const struct hashmap_iter *iter)
-{
-	struct hashmap_entry *entry = (struct hashmap_entry *)iter;
-
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!iter) {
-		return NULL;
-	}
-	return (struct hashmap_iter *)hashmap_entry_get_populated(map,
-		entry + 1);
-}
-
-/*
- * Remove the hashmap entry pointed to by this iterator and return an
- * iterator to the next entry.  Returns NULL if there are no more entries.
- */
-struct hashmap_iter *hashmap_iter_remove(struct hashmap *map,
-	const struct hashmap_iter *iter)
-{
-	struct hashmap_entry *entry = (struct hashmap_entry *)iter;
-
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!iter) {
-		return NULL;
-	}
-	if (!entry->key) {
-		/* Iterator is invalid, so just return the next valid entry */
-		return hashmap_iter_next(map, iter);
-	}
-	hashmap_entry_remove(map, entry);
-	return (struct hashmap_iter *)hashmap_entry_get_populated(map, entry);
-}
-
-/*
- * Return the key of the entry pointed to by the iterator.
- */
-const void *hashmap_iter_get_key(const struct hashmap_iter *iter)
-{
-	if (!iter) {
-		return NULL;
-	}
-	return (const void *)((struct hashmap_entry *)iter)->key;
-}
-
-/*
- * Return the data of the entry pointed to by the iterator.
- */
-void *hashmap_iter_get_data(const struct hashmap_iter *iter)
-{
-	if (!iter) {
-		return NULL;
-	}
-	return ((struct hashmap_entry *)iter)->data;
-}
-
-/*
- * Set the data pointer of the entry pointed to by the iterator.
- */
-void hashmap_iter_set_data(const struct hashmap_iter *iter, void *data)
-{
-	if (!iter) {
-		return;
-	}
-	((struct hashmap_entry *)iter)->data = data;
-}
-
-/*
- * Invoke func for each entry in the hashmap.  Unlike the hashmap_iter_*()
- * interface, this function supports calls to hashmap_remove() during iteration.
- * However, it is an error to put or remove an entry other than the current one,
- * and doing so will immediately halt iteration and return an error.
- * Iteration is stopped if func returns non-zero.  Returns func's return
- * value if it is < 0, otherwise, 0.
- */
-int hashmap_foreach(const struct hashmap *map,
-	int (*func)(const void *, void *, void *), void *arg)
-{
-	struct hashmap_entry *entry;
-	size_t num_entries;
-	const void *key;
-	int rc;
-
-	HASHMAP_ASSERT(map != NULL);
-	HASHMAP_ASSERT(func != NULL);
-
-	entry = map->table;
-	for (entry = map->table; entry < &map->table[map->table_size];
-	    ++entry) {
-		if (!entry->key) {
-			continue;
-		}
-		num_entries = map->num_entries;
-		key = entry->key;
-		rc = func(entry->key, entry->data, arg);
-		if (rc < 0) {
-			return rc;
-		}
-		if (rc > 0) {
-			return 0;
-		}
-		/* Run this entry again if func() deleted it */
-		if (entry->key != key) {
-			--entry;
-		} else if (num_entries != map->num_entries) {
-			/* Stop immediately if func put/removed another entry */
-			return -1;
-		}
-	}
-	return 0;
-}
-
-/*
- * Default hash function for string keys.
- * This is an implementation of the well-documented Jenkins one-at-a-time
- * hash function.
- */
-size_t hashmap_hash_string(const void *key)
-{
-	const char *key_str = (const char *)key;
-	size_t hash = 0;
-
-	for (; *key_str; ++key_str) {
-		hash += *key_str;
-		hash += (hash << 10);
-		hash ^= (hash >> 6);
-	}
-	hash += (hash << 3);
-	hash ^= (hash >> 11);
-	hash += (hash << 15);
-	return hash;
-}
-
-/*
- * Default key comparator function for string keys.
- */
-int hashmap_compare_string(const void *a, const void *b)
-{
-	return strcmp((const char *)a, (const char *)b);
-}
-
-/*
- * Default key allocation function for string keys.  Use free() for the
- * key_free_func.
- */
-void *hashmap_alloc_key_string(const void *key)
-{
-	return (void *)strdup((const char *)key);
-}
-
-#ifdef HASHMAP_METRICS
-/*
- * Return the load factor.
- */
-double hashmap_load_factor(const struct hashmap *map)
-{
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!map->table_size) {
-		return 0;
-	}
-	return (double)map->num_entries / map->table_size;
-}
-
-/*
- * Return the average number of collisions per entry.
- */
-double hashmap_collisions_mean(const struct hashmap *map)
-{
-	struct hashmap_entry *entry;
-	size_t total_collisions = 0;
-
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!map->num_entries) {
-		return 0;
-	}
-	for (entry = map->table; entry < &map->table[map->table_size];
-	    ++entry) {
-		if (!entry->key) {
-			continue;
-		}
-		total_collisions += entry->num_collisions;
-	}
-	return (double)total_collisions / map->num_entries;
-}
-
-/*
- * Return the variance between entry collisions.  The higher the variance,
- * the more likely the hash function is poor and is resulting in clustering.
- */
-double hashmap_collisions_variance(const struct hashmap *map)
-{
-	struct hashmap_entry *entry;
-	double mean_collisions;
-	double variance;
-	double total_variance = 0;
-
-	HASHMAP_ASSERT(map != NULL);
-
-	if (!map->num_entries) {
-		return 0;
-	}
-	mean_collisions = hashmap_collisions_mean(map);
-	for (entry = map->table; entry < &map->table[map->table_size];
-	    ++entry) {
-		if (!entry->key) {
-			continue;
-		}
-		variance = (double)entry->num_collisions - mean_collisions;
-		total_variance += variance * variance;
-	}
-	return total_variance / map->num_entries;
-}
-#endif

data/ext/fast_mmaped_file/jsmn.c

@@ -1,314 +0,0 @@
-#include "jsmn.h"
-
-/**
- * Allocates a fresh unused token from the token pull.
- */
-static jsmntok_t *jsmn_alloc_token(jsmn_parser *parser,
-		jsmntok_t *tokens, size_t num_tokens) {
-	jsmntok_t *tok;
-	if (parser->toknext >= num_tokens) {
-		return NULL;
-	}
-	tok = &tokens[parser->toknext++];
-	tok->start = tok->end = -1;
-	tok->size = 0;
-#ifdef JSMN_PARENT_LINKS
-	tok->parent = -1;
-#endif
-	return tok;
-}
-
-/**
- * Fills token type and boundaries.
- */
-static void jsmn_fill_token(jsmntok_t *token, jsmntype_t type,
-                            int start, int end) {
-	token->type = type;
-	token->start = start;
-	token->end = end;
-	token->size = 0;
-}
-
-/**
- * Fills next available token with JSON primitive.
- */
-static int jsmn_parse_primitive(jsmn_parser *parser, const char *js,
-		size_t len, jsmntok_t *tokens, size_t num_tokens) {
-	jsmntok_t *token;
-	int start;
-
-	start = parser->pos;
-
-	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
-		switch (js[parser->pos]) {
-#ifndef JSMN_STRICT
-			/* In strict mode primitive must be followed by "," or "}" or "]" */
-			case ':':
-#endif
-			case '\t' : case '\r' : case '\n' : case ' ' :
-			case ','  : case ']'  : case '}' :
-				goto found;
-		}
-		if (js[parser->pos] < 32 || js[parser->pos] >= 127) {
-			parser->pos = start;
-			return JSMN_ERROR_INVAL;
-		}
-	}
-#ifdef JSMN_STRICT
-	/* In strict mode primitive must be followed by a comma/object/array */
-	parser->pos = start;
-	return JSMN_ERROR_PART;
-#endif
-
-found:
-	if (tokens == NULL) {
-		parser->pos--;
-		return 0;
-	}
-	token = jsmn_alloc_token(parser, tokens, num_tokens);
-	if (token == NULL) {
-		parser->pos = start;
-		return JSMN_ERROR_NOMEM;
-	}
-	jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
-#ifdef JSMN_PARENT_LINKS
-	token->parent = parser->toksuper;
-#endif
-	parser->pos--;
-	return 0;
-}
-
-/**
- * Fills next token with JSON string.
- */
-static int jsmn_parse_string(jsmn_parser *parser, const char *js,
-		size_t len, jsmntok_t *tokens, size_t num_tokens) {
-	jsmntok_t *token;
-
-	int start = parser->pos;
-
-	parser->pos++;
-
-	/* Skip starting quote */
-	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
-		char c = js[parser->pos];
-
-		/* Quote: end of string */
-		if (c == '\"') {
-			if (tokens == NULL) {
-				return 0;
-			}
-			token = jsmn_alloc_token(parser, tokens, num_tokens);
-			if (token == NULL) {
-				parser->pos = start;
-				return JSMN_ERROR_NOMEM;
-			}
-			jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos);
-#ifdef JSMN_PARENT_LINKS
-			token->parent = parser->toksuper;
-#endif
-			return 0;
-		}
-
-		/* Backslash: Quoted symbol expected */
-		if (c == '\\' && parser->pos + 1 < len) {
-			int i;
-			parser->pos++;
-			switch (js[parser->pos]) {
-				/* Allowed escaped symbols */
-				case '\"': case '/' : case '\\' : case 'b' :
-				case 'f' : case 'r' : case 'n'  : case 't' :
-					break;
-				/* Allows escaped symbol \uXXXX */
-				case 'u':
-					parser->pos++;
-					for(i = 0; i < 4 && parser->pos < len && js[parser->pos] != '\0'; i++) {
-						/* If it isn't a hex character we have an error */
-						if(!((js[parser->pos] >= 48 && js[parser->pos] <= 57) || /* 0-9 */
-									(js[parser->pos] >= 65 && js[parser->pos] <= 70) || /* A-F */
-									(js[parser->pos] >= 97 && js[parser->pos] <= 102))) { /* a-f */
-							parser->pos = start;
-							return JSMN_ERROR_INVAL;
-						}
-						parser->pos++;
-					}
-					parser->pos--;
-					break;
-				/* Unexpected symbol */
-				default:
-					parser->pos = start;
-					return JSMN_ERROR_INVAL;
-			}
-		}
-	}
-	parser->pos = start;
-	return JSMN_ERROR_PART;
-}
-
-/**
- * Parse JSON string and fill tokens.
- */
-int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
-		jsmntok_t *tokens, unsigned int num_tokens) {
-	int r;
-	int i;
-	jsmntok_t *token;
-	int count = parser->toknext;
-
-	for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
-		char c;
-		jsmntype_t type;
-
-		c = js[parser->pos];
-		switch (c) {
-			case '{': case '[':
-				count++;
-				if (tokens == NULL) {
-					break;
-				}
-				token = jsmn_alloc_token(parser, tokens, num_tokens);
-				if (token == NULL)
-					return JSMN_ERROR_NOMEM;
-				if (parser->toksuper != -1) {
-					tokens[parser->toksuper].size++;
-#ifdef JSMN_PARENT_LINKS
-					token->parent = parser->toksuper;
-#endif
-				}
-				token->type = (c == '{' ? JSMN_OBJECT : JSMN_ARRAY);
-				token->start = parser->pos;
-				parser->toksuper = parser->toknext - 1;
-				break;
-			case '}': case ']':
-				if (tokens == NULL)
-					break;
-				type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
-#ifdef JSMN_PARENT_LINKS
-				if (parser->toknext < 1) {
-					return JSMN_ERROR_INVAL;
-				}
-				token = &tokens[parser->toknext - 1];
-				for (;;) {
-					if (token->start != -1 && token->end == -1) {
-						if (token->type != type) {
-							return JSMN_ERROR_INVAL;
-						}
-						token->end = parser->pos + 1;
-						parser->toksuper = token->parent;
-						break;
-					}
-					if (token->parent == -1) {
-						if(token->type != type || parser->toksuper == -1) {
-							return JSMN_ERROR_INVAL;
-						}
-						break;
-					}
-					token = &tokens[token->parent];
-				}
-#else
-				for (i = parser->toknext - 1; i >= 0; i--) {
-					token = &tokens[i];
-					if (token->start != -1 && token->end == -1) {
-						if (token->type != type) {
-							return JSMN_ERROR_INVAL;
-						}
-						parser->toksuper = -1;
-						token->end = parser->pos + 1;
-						break;
-					}
-				}
-				/* Error if unmatched closing bracket */
-				if (i == -1) return JSMN_ERROR_INVAL;
-				for (; i >= 0; i--) {
-					token = &tokens[i];
-					if (token->start != -1 && token->end == -1) {
-						parser->toksuper = i;
-						break;
-					}
-				}
-#endif
-				break;
-			case '\"':
-				r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
-				if (r < 0) return r;
-				count++;
-				if (parser->toksuper != -1 && tokens != NULL)
-					tokens[parser->toksuper].size++;
-				break;
-			case '\t' : case '\r' : case '\n' : case ' ':
-				break;
-			case ':':
-				parser->toksuper = parser->toknext - 1;
-				break;
-			case ',':
-				if (tokens != NULL && parser->toksuper != -1 &&
-						tokens[parser->toksuper].type != JSMN_ARRAY &&
-						tokens[parser->toksuper].type != JSMN_OBJECT) {
-#ifdef JSMN_PARENT_LINKS
-					parser->toksuper = tokens[parser->toksuper].parent;
-#else
-					for (i = parser->toknext - 1; i >= 0; i--) {
-						if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT) {
-							if (tokens[i].start != -1 && tokens[i].end == -1) {
-								parser->toksuper = i;
-								break;
-							}
-						}
-					}
-#endif
-				}
-				break;
-#ifdef JSMN_STRICT
-			/* In strict mode primitives are: numbers and booleans */
-			case '-': case '0': case '1' : case '2': case '3' : case '4':
-			case '5': case '6': case '7' : case '8': case '9':
-			case 't': case 'f': case 'n' :
-				/* And they must not be keys of the object */
-				if (tokens != NULL && parser->toksuper != -1) {
-					jsmntok_t *t = &tokens[parser->toksuper];
-					if (t->type == JSMN_OBJECT ||
-							(t->type == JSMN_STRING && t->size != 0)) {
-						return JSMN_ERROR_INVAL;
-					}
-				}
-#else
-			/* In non-strict mode every unquoted value is a primitive */
-			default:
-#endif
-				r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
-				if (r < 0) return r;
-				count++;
-				if (parser->toksuper != -1 && tokens != NULL)
-					tokens[parser->toksuper].size++;
-				break;
-
-#ifdef JSMN_STRICT
-			/* Unexpected char in strict mode */
-			default:
-				return JSMN_ERROR_INVAL;
-#endif
-		}
-	}
-
-	if (tokens != NULL) {
-		for (i = parser->toknext - 1; i >= 0; i--) {
-			/* Unmatched opened object or array */
-			if (tokens[i].start != -1 && tokens[i].end == -1) {
-				return JSMN_ERROR_PART;
-			}
-		}
-	}
-
-	return count;
-}
-
-/**
- * Creates a new parser based over a given  buffer with an array of tokens
- * available.
- */
-void jsmn_init(jsmn_parser *parser) {
-	parser->pos = 0;
-	parser->toknext = 0;
-	parser->toksuper = -1;
-}
-