whistlepig 0.1

data/ext/whistlepig/index.c

@@ -0,0 +1,294 @@
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include "whistlepig.h"
+
+#define PATH_BUF_SIZE 4096
+
+int wp_index_exists(const char* pathname_base) {
+  char buf[PATH_BUF_SIZE];
+  snprintf(buf, PATH_BUF_SIZE, "%s0", pathname_base);
+  return wp_segment_exists(buf);
+}
+
+wp_error* wp_index_create(wp_index** indexptr, const char* pathname_base) {
+  char buf[PATH_BUF_SIZE];
+
+  snprintf(buf, PATH_BUF_SIZE, "%s0", pathname_base);
+  if(wp_segment_exists(buf)) RAISE_ERROR("index with base path '%s' already exists", pathname_base);
+
+  wp_index* index = *indexptr = malloc(sizeof(wp_index));
+  index->pathname_base = pathname_base;
+  index->num_segments = 1;
+  index->sizeof_segments = 1;
+  index->open = 1;
+  index->segments = malloc(sizeof(wp_segment));
+  index->docid_offsets = malloc(sizeof(uint64_t));
+
+  RELAY_ERROR(wp_segment_create(&index->segments[0], buf));
+  index->docid_offsets[0] = 0;
+
+  return NO_ERROR;
+}
+
+RAISING_STATIC(ensure_num_segments(wp_index* index)) {
+  if(index->num_segments >= index->sizeof_segments) {
+    index->sizeof_segments *= 2;
+    index->segments = realloc(index->segments, sizeof(wp_segment) * index->sizeof_segments);
+    index->docid_offsets = realloc(index->docid_offsets, sizeof(uint64_t) * index->sizeof_segments);
+    if(index->segments == NULL) RAISE_ERROR("oom");
+  }
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_load(wp_index** indexptr, const char* pathname_base) {
+  char buf[PATH_BUF_SIZE];
+  snprintf(buf, PATH_BUF_SIZE, "%s0", pathname_base);
+  if(!wp_segment_exists(buf)) RAISE_ERROR("index with base path '%s' does not exist", pathname_base);
+
+  wp_index* index = *indexptr = malloc(sizeof(wp_index));
+
+  index->pathname_base = pathname_base;
+  index->num_segments = 0;
+  index->sizeof_segments = 1;
+  index->open = 1;
+  index->segments = malloc(sizeof(wp_segment));
+  index->docid_offsets = malloc(sizeof(uint64_t));
+
+  // load all the segments we can
+  while(index->num_segments < WP_MAX_SEGMENTS) {
+    snprintf(buf, PATH_BUF_SIZE, "%s%d", pathname_base, index->num_segments);
+    if(!wp_segment_exists(buf)) break;
+
+    RELAY_ERROR(ensure_num_segments(index));
+    DEBUG("loading segment %s", buf);
+    RELAY_ERROR(wp_segment_load(&index->segments[index->num_segments], buf));
+    if(index->num_segments == 0)
+      index->docid_offsets[index->num_segments] = 0;
+    else {
+      // segments return docids 1 through N, so the num_docs in a segment is
+      // also the max document id
+      postings_region* prevpr = MMAP_OBJ(index->segments[index->num_segments - 1].postings, postings_region);
+      index->docid_offsets[index->num_segments] = prevpr->num_docs + index->docid_offsets[index->num_segments - 1];
+    }
+
+    index->num_segments++;
+  }
+
+  return NO_ERROR;
+}
+
+// we have two special values at our disposal to mark where we are in
+// the sequence of segments
+#define SEGMENT_UNINITIALIZED WP_MAX_SEGMENTS
+#define SEGMENT_DONE (WP_MAX_SEGMENTS + 1)
+
+wp_error* wp_index_setup_query(wp_index* index, wp_query* query) {
+  (void)index;
+  query->segment_idx = SEGMENT_UNINITIALIZED;
+
+  return NO_ERROR;
+}
+
+// can be called multiple times to resume
+wp_error* wp_index_run_query(wp_index* index, wp_query* query, uint32_t max_num_results, uint32_t* num_results, uint64_t* results) {
+  *num_results = 0;
+  if(index->num_segments == 0) return NO_ERROR;
+
+  if(query->segment_idx == SEGMENT_UNINITIALIZED) {
+    query->segment_idx = index->num_segments - 1;
+    DEBUG("setting up segment %u", query->segment_idx);
+    RELAY_ERROR(wp_search_init_search_state(query, &index->segments[query->segment_idx]));
+  }
+
+  // at this point, we assume we're initialized and query->segment_idx is the index
+  // of the segment we're searching against
+  while((*num_results < max_num_results) && (query->segment_idx != SEGMENT_DONE)) {
+    uint32_t want_num_results = max_num_results - *num_results;
+    uint32_t got_num_results = 0;
+    search_result* segment_results = malloc(sizeof(search_result) * want_num_results);
+
+    DEBUG("searching segment %d", query->segment_idx);
+    RELAY_ERROR(wp_search_run_query_on_segment(query, &index->segments[query->segment_idx], want_num_results, &got_num_results, segment_results));
+    DEBUG("asked segment %d for %d results, got %d", query->segment_idx, want_num_results, got_num_results);
+
+    // extract the per-segment docids from the search results and adjust by
+    // each segment's docid offset to form global docids
+    for(uint32_t i = 0; i < got_num_results; i++) {
+      results[*num_results + i] = index->docid_offsets[query->segment_idx] + segment_results[i].doc_id;
+      wp_search_result_free(&segment_results[i]);
+    }
+    free(segment_results);
+    *num_results += got_num_results;
+
+    if(got_num_results < want_num_results) { // this segment is finished; move to the next one
+      DEBUG("releasing index %d", query->segment_idx);
+      RELAY_ERROR(wp_search_release_search_state(query));
+      if(query->segment_idx > 0) {
+        query->segment_idx--;
+        DEBUG("setting up index %d", query->segment_idx);
+        RELAY_ERROR(wp_search_init_search_state(query, &index->segments[query->segment_idx]));
+      }
+      else query->segment_idx = SEGMENT_DONE;
+    }
+  }
+
+  return NO_ERROR;
+}
+
+#define RESULT_BUF_SIZE 1024
+// count the results by just running the query until it stops. slow!
+wp_error* wp_index_count_results(wp_index* index, wp_query* query, uint32_t* num_results) {
+  uint64_t results[RESULT_BUF_SIZE];
+
+  *num_results = 0;
+  RELAY_ERROR(wp_index_setup_query(index, query));
+  while(1) {
+    uint32_t this_num_results;
+    RELAY_ERROR(wp_index_run_query(index, query, RESULT_BUF_SIZE, &this_num_results, results));
+    *num_results += this_num_results;
+    if(this_num_results < RESULT_BUF_SIZE) break; // done
+  }
+
+  RELAY_ERROR(wp_index_teardown_query(index, query));
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_teardown_query(wp_index* index, wp_query* query) {
+  (void)index;
+  if((query->segment_idx != SEGMENT_UNINITIALIZED) && (query->segment_idx != SEGMENT_DONE)) {
+    RELAY_ERROR(wp_search_release_search_state(query));
+  }
+  query->segment_idx = SEGMENT_UNINITIALIZED;
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_add_entry(wp_index* index, wp_entry* entry, uint64_t* doc_id) {
+  int success;
+  wp_segment* seg = &index->segments[index->num_segments - 1];
+
+  // first, ensure we have enough space in the current segment
+  uint32_t postings_bytes;
+  RELAY_ERROR(wp_entry_sizeof_postings_region(entry, seg, &postings_bytes));
+  RELAY_ERROR(wp_segment_ensure_fit(seg, postings_bytes, 0, &success));
+
+  // if not, we need to open a new one
+  if(!success) {
+    DEBUG("segment %d is full, loading a new one", index->num_segments - 1);
+    char buf[PATH_BUF_SIZE];
+    snprintf(buf, PATH_BUF_SIZE, "%s%d", index->pathname_base, index->num_segments);
+    RELAY_ERROR(ensure_num_segments(index));
+    RELAY_ERROR(wp_segment_create(&index->segments[index->num_segments], buf));
+    index->num_segments++;
+
+    // set the docid_offset
+    postings_region* prevpr = MMAP_OBJ(index->segments[index->num_segments - 2].postings, postings_region);
+    index->docid_offsets[index->num_segments - 1] = prevpr->num_docs + index->docid_offsets[index->num_segments - 2];
+
+    seg = &index->segments[index->num_segments - 1];
+    DEBUG("loaded new segment %d at %p", index->num_segments - 1, &index->segments[index->num_segments - 1]);
+
+    RELAY_ERROR(wp_entry_sizeof_postings_region(entry, seg, &postings_bytes));
+    RELAY_ERROR(wp_segment_ensure_fit(seg, postings_bytes, 0, &success));
+    if(!success) RAISE_ERROR("can't fit new entry into fresh segment. that's crazy");
+  }
+
+  docid_t seg_doc_id;
+  RELAY_ERROR(wp_segment_grab_docid(seg, &seg_doc_id));
+  RELAY_ERROR(wp_entry_write_to_segment(entry, seg, seg_doc_id));
+  *doc_id = seg_doc_id + index->docid_offsets[index->num_segments - 1];
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_unload(wp_index* index) {
+  for(uint16_t i = 0; i < index->num_segments; i++) RELAY_ERROR(wp_segment_unload(&index->segments[i]));
+  index->open = 0;
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_free(wp_index* index) {
+  if(index->open) RELAY_ERROR(wp_index_unload(index));
+  free(index->segments);
+  free(index->docid_offsets);
+  free(index);
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_dumpinfo(wp_index* index, FILE* stream) {
+  fprintf(stream, "index has %d segments\n", index->num_segments);
+  for(int i = 0; i < index->num_segments; i++) {
+    fprintf(stream, "\nsegment %d:\n", i);
+    RELAY_ERROR(wp_segment_dumpinfo(&index->segments[i], stream));
+  }
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_delete(const char* pathname_base) {
+  char buf[PATH_BUF_SIZE];
+
+  int i = 0;
+  while(1) {
+    snprintf(buf, PATH_BUF_SIZE, "%s%d", pathname_base, i);
+    if(wp_segment_exists(buf)) {
+      DEBUG("deleting segment %s", buf);
+      RELAY_ERROR(wp_segment_delete(buf));
+      i++;
+    }
+    else break;
+  }
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_add_label(wp_index* index, const char* label, uint64_t doc_id) {
+  int found = 0;
+
+  for(uint32_t i = index->num_segments; i > 0; i--) {
+    if(doc_id > index->docid_offsets[i - 1]) {
+      DEBUG("found doc %llu in segment %u", doc_id, i - 1);
+      RELAY_ERROR(wp_segment_add_label(&index->segments[i - 1], label, (docid_t)(doc_id - index->docid_offsets[i - 1])));
+      found = 1;
+      break;
+    }
+    else DEBUG("did not find doc %llu in segment %u", doc_id, i - 1);
+  }
+
+  if(!found) RAISE_ERROR("couldn't find doc id %llu", doc_id);
+
+  return NO_ERROR;
+}
+
+wp_error* wp_index_remove_label(wp_index* index, const char* label, uint64_t doc_id) {
+  int found = 0;
+
+  for(uint32_t i = index->num_segments; i > 0; i--) {
+    if(doc_id > index->docid_offsets[i - 1]) {
+      DEBUG("found doc %llu in segment %u", doc_id, i - 1);
+      RELAY_ERROR(wp_segment_remove_label(&index->segments[i - 1], label, (docid_t)(doc_id - index->docid_offsets[i - 1])));
+      found = 1;
+      break;
+    }
+    else DEBUG("did not find doc %llu in segment %u", doc_id, i - 1);
+  }
+
+  if(!found) RAISE_ERROR("couldn't find doc id %llu", doc_id);
+
+  return NO_ERROR;
+}
+
+uint64_t wp_index_num_docs(wp_index* index) {
+  uint64_t ret = 0;
+
+  // TODO check for overflow or some shit
+  for(uint32_t i = index->num_segments; i > 0; i--) ret += wp_segment_num_docs(&index->segments[i - 1]);
+
+  return ret;
+}

data/ext/whistlepig/index.h

@@ -0,0 +1,88 @@
+#ifndef WP_INDEX_H_
+#define WP_INDEX_H_
+
+// whistlepig index
+// (c) 2011 William Morgan. See COPYING for license terms.
+//
+// the main public interaction point with whistlepig, in addition to the
+// supporting entry and query objects. it holds a collection of segments and
+// essentially relays commands to the appropriate ones, creating new segments
+// as needed.
+
+#include "defaults.h"
+#include "segment.h"
+#include "error.h"
+#include "entry.h"
+
+#define WP_MAX_SEGMENTS 65534 // max value of wp_search_query->segment_idx - 2 because we need two special numbers
+
+typedef struct wp_index {
+  const char* pathname_base;
+  uint16_t num_segments;
+  uint16_t sizeof_segments;
+  uint64_t* docid_offsets;
+  struct wp_segment* segments;
+  uint8_t open;
+} wp_index;
+
+// API methods
+
+// public: returns non-zero if an index with base pathname pathname_base
+// exists, zero otherwise
+int wp_index_exists(const char* pathname_base);
+
+// public: creates an index, raising an exception if it already exists
+wp_error* wp_index_create(wp_index** index, const char* pathname_base) RAISES_ERROR;
+
+// public: loads an existing index, raising an exception if it doesn't exist
+wp_error* wp_index_load(wp_index** index, const char* pathname_base) RAISES_ERROR;
+
+// public: releases an index
+wp_error* wp_index_unload(wp_index* index) RAISES_ERROR;
+
+// public: frees all memory. can be called after unload, or not. don't call
+// anything on the index after calling this, though...
+wp_error* wp_index_free(wp_index* index) RAISES_ERROR;
+
+// public: returns the number of documents in the index.
+uint64_t wp_index_num_docs(wp_index* index);
+
+// public: initializes a query for use on the index. must be called before
+// run_query
+wp_error* wp_index_setup_query(wp_index* index, wp_query* query) RAISES_ERROR;
+
+// public: tears down a query from use on the index. must be called after
+// run_query, or memory will leak.
+wp_error* wp_index_teardown_query(wp_index* index, wp_query* query) RAISES_ERROR;
+
+// public: runs a query on an index. must be called in between setup_query and
+// teardown_query. can be called multiple times and the query will be resumed.
+// when the number of documents returned is < num_results, then you're at the
+// end!
+wp_error* wp_index_run_query(wp_index* index, wp_query* query, uint32_t max_num_results, uint32_t* num_results, uint64_t* results) RAISES_ERROR;
+
+// public: returns the number of results that match a query. note that this is
+// roughly as expensive as just running the query competely, modulo some memory
+// allocations here and there...
+wp_error* wp_index_count_results(wp_index* index, wp_query* query, uint32_t* num_results) RAISES_ERROR;
+
+// public: adds an entry to the index. sets doc_id to the new docid.
+wp_error* wp_index_add_entry(wp_index* index, wp_entry* entry, uint64_t* doc_id) RAISES_ERROR;
+
+// public: adds an label to a doc_id. throws an exception if the document
+// doesn't exist. does nothing if the label has already been added to the
+// document.
+wp_error* wp_index_add_label(wp_index* index, const char* label, uint64_t doc_id);
+
+// public: removes a label from a doc_id. throws an exception if the document
+// doesn't exist. does nothing if the label has already been added to the
+// document.
+wp_error* wp_index_remove_label(wp_index* index, const char* label, uint64_t doc_id);
+
+// dumps some index to the stream.
+wp_error* wp_index_dumpinfo(wp_index* index, FILE* stream) RAISES_ERROR;
+
+// public: deletes a document from disk.
+wp_error* wp_index_delete(const char* path) RAISES_ERROR;
+
+#endif

data/ext/whistlepig/khash.h

@@ -0,0 +1,316 @@
+/* The MIT License
+
+   Copyright (c) 2008, by Attractive Chaos <attractivechaos@aol.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+	int ret, is_missing;
+	khiter_t k;
+	khash_t(32) *h = kh_init(32);
+	k = kh_put(32, h, 5, &ret);
+	if (!ret) kh_del(32, h, k);
+	kh_value(h, k) = 10;
+	k = kh_get(32, h, 10);
+	is_missing = (k == kh_end(h));
+	k = kh_get(32, h, 5);
+	kh_del(32, h, k);
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) kh_value(h, k) = 1;
+	kh_destroy(32, h);
+	return 0;
+}
+*/
+
+/*
+  2008-09-19 (0.2.3):
+
+	* Corrected the example
+	* Improved interfaces
+
+  2008-09-11 (0.2.2):
+
+	* Improved speed a little in kh_put()
+
+  2008-09-10 (0.2.1):
+
+	* Added kh_clear()
+	* Fixed a compiling error
+
+  2008-09-02 (0.2.0):
+
+	* Changed to token concatenation which increases flexibility.
+
+  2008-08-31 (0.1.2):
+
+	* Fixed a bug in kh_get(), which has not been tested previously.
+
+  2008-08-31 (0.1.1):
+
+	* Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+#define AC_VERSION_KHASH_H "0.2.2"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef uint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_HASH_PRIME_SIZE 32
+static const uint32_t __ac_prime_list[__ac_HASH_PRIME_SIZE] =
+{
+  0ul,          3ul,          11ul,         23ul,         53ul,
+  97ul,         193ul,        389ul,        769ul,        1543ul,
+  3079ul,       6151ul,       12289ul,      24593ul,      49157ul,
+  98317ul,      196613ul,     393241ul,     786433ul,     1572869ul,
+  3145739ul,    6291469ul,    12582917ul,   25165843ul,   50331653ul,
+  100663319ul,  201326611ul,  402653189ul,  805306457ul,  1610612741ul,
+  3221225473ul, 4294967291ul
+};
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	typedef struct {													\
+		khint_t n_buckets, size, n_occupied, upper_bound;				\
+		uint32_t *flags;												\
+		khkey_t *keys;													\
+		khval_t *vals;													\
+	} kh_##name##_t;													\
+	static inline kh_##name##_t *kh_init_##name() {						\
+		return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t));		\
+	}																	\
+	static inline void kh_destroy_##name(kh_##name##_t *h)				\
+	{																	\
+		if (h) {														\
+			free(h->keys); free(h->flags);								\
+			free(h->vals);												\
+			free(h);													\
+		}																\
+	}																	\
+	static inline void kh_clear_##name(kh_##name##_t *h)				\
+	{																	\
+		if (h && h->flags) { \
+			memset(h->flags, 0xaa, ((h->n_buckets>>4) + 1) * sizeof(uint32_t)); \
+			h->size = h->n_occupied = 0;								\
+		}																\
+	}																	\
+	static inline khint_t kh_get_##name(kh_##name##_t *h, khkey_t key)	\
+	{																	\
+		if (h->n_buckets) {												\
+			khint_t inc, k, i, last;									\
+			k = __hash_func(key); i = k % h->n_buckets;					\
+			inc = 1 + k % (h->n_buckets - 1); last = i;					\
+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+				if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
+				else i += inc;											\
+				if (i == last) return h->n_buckets;						\
+			}															\
+			return __ac_iseither(h->flags, i)? h->n_buckets : i;			\
+		} else return 0;												\
+	}																	\
+	static inline void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	{																	\
+		uint32_t *new_flags = 0;										\
+		khint_t j = 1;													\
+		{																\
+			khint_t t = __ac_HASH_PRIME_SIZE - 1;						\
+			while (__ac_prime_list[t] > new_n_buckets) --t;				\
+			new_n_buckets = __ac_prime_list[t+1];						\
+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	\
+			else {														\
+				new_flags = (uint32_t*)malloc(((new_n_buckets>>4) + 1) * sizeof(uint32_t));	\
+				memset(new_flags, 0xaa, ((new_n_buckets>>4) + 1) * sizeof(uint32_t)); \
+				if (h->n_buckets < new_n_buckets) {						\
+					h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
+					if (kh_is_map)										\
+						h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
+				}														\
+			}															\
+		}																\
+		if (j) {														\
+			for (j = 0; j != h->n_buckets; ++j) {						\
+				if (__ac_iseither(h->flags, j) == 0) {					\
+					khkey_t key = h->keys[j];							\
+					khval_t val;										\
+					if (kh_is_map) val = h->vals[j];					\
+					__ac_set_isdel_true(h->flags, j);					\
+					while (1) {											\
+						khint_t inc, k, i;								\
+						k = __hash_func(key);							\
+						i = k % new_n_buckets;							\
+						inc = 1 + k % (new_n_buckets - 1);				\
+						while (!__ac_isempty(new_flags, i)) {			\
+							if (i + inc >= new_n_buckets) i = i + inc - new_n_buckets; \
+							else i += inc;								\
+						}												\
+						__ac_set_isempty_false(new_flags, i);			\
+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { \
+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+							__ac_set_isdel_true(h->flags, i);			\
+						} else {										\
+							h->keys[i] = key;							\
+							if (kh_is_map) h->vals[i] = val;			\
+							break;										\
+						}												\
+					}													\
+				}														\
+			}															\
+			if (h->n_buckets > new_n_buckets) {							\
+				h->keys = (khkey_t*)realloc(h->keys, new_n_buckets * sizeof(khkey_t)); \
+				if (kh_is_map)											\
+					h->vals = (khval_t*)realloc(h->vals, new_n_buckets * sizeof(khval_t)); \
+			}															\
+			free(h->flags);												\
+			h->flags = new_flags;										\
+			h->n_buckets = new_n_buckets;								\
+			h->n_occupied = h->size;									\
+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+		}																\
+	}																	\
+	static inline khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+	{																	\
+		khint_t x;														\
+		if (h->n_occupied >= h->upper_bound) {							\
+			if (h->n_buckets > (h->size<<1)) kh_resize_##name(h, h->n_buckets - 1); \
+			else kh_resize_##name(h, h->n_buckets + 1);					\
+		}																\
+		{																\
+			khint_t inc, k, i, site, last;								\
+			x = site = h->n_buckets; k = __hash_func(key); i = k % h->n_buckets; \
+			if (__ac_isempty(h->flags, i)) x = i;						\
+			else {														\
+				inc = 1 + k % (h->n_buckets - 1); last = i;				\
+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+					if (__ac_isdel(h->flags, i)) site = i;				\
+					if (i + inc >= h->n_buckets) i = i + inc - h->n_buckets; \
+					else i += inc;										\
+					if (i == last) { x = site; break; }					\
+				}														\
+				if (x == h->n_buckets) {								\
+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+					else x = i;											\
+				}														\
+			}															\
+		}																\
+		if (__ac_isempty(h->flags, x)) {								\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size; ++h->n_occupied;									\
+			*ret = 1;													\
+		} else if (__ac_isdel(h->flags, x)) {							\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size;													\
+			*ret = 2;													\
+		} else *ret = 0;												\
+		return x;														\
+	}																	\
+	static inline void kh_del_##name(kh_##name##_t *h, khint_t x)		\
+	{																	\
+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\
+			__ac_set_isdel_true(h->flags, x);							\
+			--h->size;													\
+		}																\
+	}
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+#define kh_int_hash_func(key) (uint32_t)(key)
+#define kh_int_hash_equal(a, b) (a == b)
+#define kh_int64_hash_func(key) (uint32_t)((key)>>33^(key)^(key)<<11)
+#define kh_int64_hash_equal(a, b) (a == b)
+static inline khint_t __ac_X31_hash_string(const char *s)
+{
+	khint_t h = *s;
+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + *s;
+	return h;
+}
+#define kh_str_hash_func(key) __ac_X31_hash_string(key)
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other necessary macros... */
+
+#define khash_t(name) kh_##name##_t
+
+#define kh_init(name) kh_init_##name()
+#define kh_destroy(name, h) kh_destroy_##name(h)
+#define kh_clear(name, h) kh_clear_##name(h)
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+#define kh_get(name, h, k) kh_get_##name(h, k)
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+#define kh_key(h, x) ((h)->keys[x])
+#define kh_val(h, x) ((h)->vals[x])
+#define kh_value(h, x) ((h)->vals[x])
+#define kh_begin(h) (khint_t)(0)
+#define kh_end(h) ((h)->n_buckets)
+#define kh_size(h) ((h)->size)
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/* More conenient interfaces */
+
+#define KHASH_SET_INIT_INT(name)										\
+	KHASH_INIT(name, uint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+#define KHASH_MAP_INIT_INT(name, khval_t)								\
+	KHASH_INIT(name, uint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+#define KHASH_SET_INIT_INT64(name)										\
+	KHASH_INIT(name, uint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+#define KHASH_MAP_INIT_INT64(name, khval_t)								\
+	KHASH_INIT(name, uint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+#define KHASH_SET_INIT_STR(name)										\
+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+#define KHASH_MAP_INIT_STR(name, khval_t)								\
+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */