libcouchbase 0.0.1

data/ext/libcouchbase/src/rdb/bigalloc.c

@@ -0,0 +1,225 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <stddef.h>
+#include "rope.h"
+#include "bigalloc.h"
+
+#define MAXIMUM(a, b) (a) > (b) ? a : b
+
+static void
+alloc_decref(rdb_ALLOCATOR *abase)
+{
+    lcb_list_t *llcur, *llnext;
+    rdb_BIGALLOC *alloc = (rdb_BIGALLOC *)abase;
+    if (--alloc->refcount) {
+        return;
+    }
+
+    LCB_LIST_SAFE_FOR(llcur, llnext, (lcb_list_t *)&alloc->bufs) {
+        rdb_ROPESEG *seg = LCB_LIST_ITEM(llcur, rdb_ROPESEG, llnode);
+        lcb_clist_delete(&alloc->bufs, &seg->llnode);
+        free(seg->root);
+        free(seg);
+    }
+    free(alloc);
+}
+
+static void
+recheck_thresholds(rdb_BIGALLOC *alloc)
+{
+    if (++alloc->n_requests % RDB_BIGALLOC_RECHECK_RATE) {
+        return;
+    }
+
+    alloc->total_requests += alloc->n_requests;
+    alloc->total_toobig += alloc->n_toobig;
+    alloc->total_toosmall += alloc->n_toosmall;
+
+    if (alloc->n_toobig == alloc->n_toosmall) {
+        /* all is ok */
+
+    } else if (alloc->n_toobig > alloc->n_toosmall) {
+        /* seems we need to allocate bigger chunks? */
+        if (alloc->n_toobig*2 > alloc->n_toosmall) {
+            alloc->min_blk_alloc *= 2;
+            alloc->max_blk_alloc *= 2;
+        }
+    } else if (alloc->n_toosmall > alloc->n_toobig) {
+        if (alloc->n_toosmall*2 > alloc->n_toobig) {
+            alloc->min_blk_alloc /= 2;
+            alloc->max_blk_alloc /= 2;
+        }
+    }
+
+    alloc->n_requests = 0;
+    alloc->n_toobig = 0;
+    alloc->n_toosmall = 0;
+}
+
+static rdb_ROPESEG *
+seg_alloc(rdb_ALLOCATOR *abase, unsigned size)
+{
+    lcb_list_t *llcur;
+    rdb_ROPESEG *newseg = NULL;
+    rdb_BIGALLOC *alloc = (rdb_BIGALLOC *)abase;
+
+    recheck_thresholds(alloc);
+    /**
+     * If the allocation reaches a certain threshold (for example, for a really
+     * huge packet), then don't bother caching it.
+     */
+    if (size > alloc->max_blk_alloc) {
+        alloc->n_toobig++;
+        alloc->total_malloc++;
+        newseg = calloc(1, sizeof(*newseg));
+        newseg->root = malloc(size);
+        newseg->nalloc = size;
+        goto GT_RETNEW;
+    } else if (size < alloc->min_blk_alloc) {
+        alloc->n_toosmall++;
+    }
+
+    LCB_LIST_FOR(llcur, (lcb_list_t *)&alloc->bufs) {
+        rdb_ROPESEG *cur = LCB_LIST_ITEM(llcur, rdb_ROPESEG, llnode);
+        if (cur->nalloc < size) {
+            continue;
+        }
+
+        newseg = cur;
+        lcb_clist_delete(&alloc->bufs, llcur);
+        break;
+    }
+
+    if (!newseg) {
+        unsigned newsize = alloc->min_blk_alloc;
+        if (LCB_CLIST_SIZE(&alloc->bufs) >= alloc->max_blk_count) {
+            lcb_list_t *llold = lcb_clist_pop(&alloc->bufs);
+            newseg = LCB_LIST_ITEM(llold, rdb_ROPESEG, llnode);
+            free(newseg->root);
+        } else {
+            newseg = calloc(1, sizeof(*newseg));
+            alloc->total_malloc++;
+        }
+
+        while (newsize < size) {
+            newsize = (unsigned) ((double)newsize * 1.5);
+        }
+
+        newseg->root = malloc(newsize);
+        newseg->nalloc = newsize;
+    }
+
+    GT_RETNEW:
+    newseg->shflags = RDB_ROPESEG_F_LIB;
+    newseg->allocator = abase;
+    newseg->allocid = RDB_ALLOCATOR_CHUNKED;
+    newseg->start = 0;
+    newseg->nused = 0;
+    alloc->refcount++;
+    return newseg;
+}
+
+static void
+buf_reserve(rdb_pALLOCATOR abase, rdb_ROPEBUF *buf, unsigned size)
+{
+    rdb_BIGALLOC *alloc = (rdb_BIGALLOC *)abase;
+    rdb_ROPESEG *newseg, *lastseg;
+
+    lastseg = RDB_SEG_LAST(buf);
+    if (lastseg && RDB_SEG_SPACE(lastseg) + buf->nused >= size) {
+        return;
+    }
+
+    newseg = seg_alloc(&alloc->base, size);
+    lcb_list_append(&buf->segments, &newseg->llnode);
+}
+
+static rdb_ROPESEG *
+seg_realloc(rdb_ALLOCATOR *abase, rdb_ROPESEG *seg, unsigned size)
+{
+    rdb_BIGALLOC *alloc = (rdb_BIGALLOC *)abase;
+
+    if (size < alloc->n_toosmall) {
+        alloc->n_toosmall++;
+    } else if (size > alloc->n_toobig) {
+        alloc->n_toobig++;
+    }
+
+    seg->root = realloc(seg->root, size);
+    seg->nalloc = size;
+    alloc->total_malloc++;
+    recheck_thresholds((rdb_BIGALLOC *)abase);
+    return seg;
+}
+
+static void
+seg_release(rdb_ALLOCATOR *abase, rdb_ROPESEG *seg)
+{
+    rdb_BIGALLOC *alloc = (rdb_BIGALLOC *)abase;
+    if (LCB_CLIST_SIZE(&alloc->bufs) >= alloc->max_blk_count ||
+            seg->nalloc > alloc->max_blk_alloc ||
+            seg->nalloc < alloc->min_blk_alloc) {
+        free(seg->root);
+        free(seg);
+    } else {
+        lcb_clist_prepend(&alloc->bufs, &seg->llnode);
+    }
+    alloc_decref(abase);
+}
+
+static void
+dump_wrap(rdb_pALLOCATOR alloc, FILE *fp) { rdb_bigalloc_dump((rdb_BIGALLOC*)alloc, fp); }
+
+rdb_ALLOCATOR *
+rdb_bigalloc_new(void)
+{
+    rdb_ALLOCATOR *abase;
+    rdb_BIGALLOC *alloc = calloc(1, sizeof(*alloc));
+    lcb_clist_init(&alloc->bufs);
+    alloc->max_blk_alloc = RDB_BIGALLOC_ALLOCSZ_MAX;
+    alloc->min_blk_alloc = RDB_BIGALLOC_ALLOCSZ_MIN;
+    alloc->max_blk_count = RDB_BIGALLOC_BLKCNT_MAX;
+    alloc->refcount = 1;
+
+    abase = &alloc->base;
+    abase->r_reserve = buf_reserve;
+    abase->s_release = seg_release;
+    abase->s_alloc = seg_alloc;
+    abase->s_realloc = seg_realloc;
+    abase->a_release = alloc_decref;
+    abase->dump = dump_wrap;
+    return &alloc->base;
+}
+
+void
+rdb_bigalloc_dump(rdb_BIGALLOC *alloc, FILE *fp)
+{
+    static const char *indent = "  ";
+    fprintf(fp, "BIGALLOC @%p\n", (void *)alloc);
+    fprintf(fp, "%sPooled Blocks: %lu\n", indent, LCB_CLIST_SIZE(&alloc->bufs));
+    fprintf(fp, "%sMinAlloc: %u\n", indent, alloc->min_blk_alloc);
+    fprintf(fp, "%sMaxAlloc: %u\n", indent, alloc->max_blk_alloc);
+    fprintf(fp, "%sMaxBlocks: %u\n", indent, alloc->max_blk_count);
+
+    fprintf(fp, "%sTotalMalloc: %u\n", indent, alloc->total_malloc);
+    fprintf(fp, "%sTotalRequests: %u\n", indent, alloc->total_requests);
+    fprintf(fp, "%sTotalToobig: %u\n", indent, alloc->total_toobig);
+    fprintf(fp, "%sTotalToosmall: %u\n", indent, alloc->total_toosmall);
+
+}

data/ext/libcouchbase/src/rdb/bigalloc.h

@@ -0,0 +1,73 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#ifndef RDB_BIGALLOC
+#define RDB_BIGALLOC
+#include "list.h"
+#include <stdio.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Big block allocator. This allocator will allocate large chunks of memory with
+ * the assumption that will typically not be wasted too quickly. It also keeps
+ * track of an allocation history, so that it can adjust to the current
+ * "climate".
+ *
+ * This header file exists for internal use. To create an allocator instance,
+ * refer to rdb_bigalloc_new() in rope.h
+ */
+
+typedef struct {
+    rdb_ALLOCATOR base;
+    lcb_clist_t bufs; /* list of pooled segments */
+    unsigned refcount;
+    unsigned min_blk_alloc; /* minimum alloc size */
+    unsigned max_blk_alloc; /* maximum alloc size (bigger than this is not pooled) */
+    unsigned max_blk_count; /* maximum number of blocks to pool */
+    unsigned n_requests; /* number of requests. Reset every RECHECK_RATE */
+    unsigned n_toobig; /* number of requests > max_blk_alloc */
+    unsigned n_toosmall; /* number of requests < min_blk_alloc */
+
+    /** counters updated at the end only */
+    unsigned total_malloc;
+    unsigned total_requests;
+    unsigned total_toobig;
+    unsigned total_toosmall;
+} rdb_BIGALLOC;
+
+#define RDB_BIGALLOC_ALLOCSZ_MAX 65536
+#define RDB_BIGALLOC_ALLOCSZ_MIN 256
+#define RDB_BIGALLOC_BLKCNT_MAX 8
+
+/** Readjust thresholds every <n> requests. <n> is defined here */
+#define RDB_BIGALLOC_RECHECK_RATE 15
+
+/**
+ * Dumps a textual representation of the specified allocator to a FILE
+ * @param alloc
+ * @param fp
+ */
+void
+rdb_bigalloc_dump(rdb_BIGALLOC *alloc, FILE *fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/ext/libcouchbase/src/rdb/chunkalloc.c

@@ -0,0 +1,174 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2014 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+#include "rope.h"
+
+/**
+ * This is a fixed size chunk allocator. It tries to allocate chunks of a
+ * certain size. For constraints needing "Special" block sizes, it simply
+ * defers to malloc/free (in truth, it could utilize the 'bigalloc'
+ * allocator for malloc/frees, but that might not be necessary).
+ */
+
+typedef struct {
+    rdb_ALLOCATOR base;
+    lcb_clist_t chunks;
+    unsigned refcount;
+    unsigned chunksize;
+    unsigned max_chunks;
+} my_CHUNKALLOC;
+
+static void
+alloc_decref(rdb_ALLOCATOR *abase)
+{
+    lcb_list_t *llcur, *llnext;
+    my_CHUNKALLOC *alloc = (my_CHUNKALLOC *)abase;
+    if (--alloc->refcount) {
+        return;
+    }
+
+    LCB_LIST_SAFE_FOR(llcur, llnext, (lcb_list_t *)&alloc->chunks) {
+        rdb_ROPESEG *seg = LCB_LIST_ITEM(llcur, rdb_ROPESEG, llnode);
+        free(seg->root);
+        free(seg);
+    }
+    free(alloc);
+}
+
+static void
+release_chunk(rdb_ALLOCATOR *abase, rdb_ROPESEG *seg)
+{
+    my_CHUNKALLOC *alloc = (my_CHUNKALLOC *)abase;
+    if (seg->nalloc != alloc->chunksize ||
+            LCB_CLIST_SIZE(&alloc->chunks) > alloc->max_chunks) {
+        free(seg->root);
+        free(seg);
+    } else {
+        lcb_clist_prepend(&alloc->chunks, &seg->llnode);
+    }
+}
+
+static rdb_ROPESEG *
+standalone_alloc(rdb_ALLOCATOR *abase, unsigned size)
+{
+    my_CHUNKALLOC *alloc = (my_CHUNKALLOC *)abase;
+    rdb_ROPESEG *seg;
+
+    seg = calloc(1, sizeof(*seg));
+    seg->root = malloc(size);
+    seg->nalloc = size;
+    seg->shflags = RDB_ROPESEG_F_LIB;
+    seg->allocid = RDB_ALLOCATOR_CHUNKED;
+    seg->allocator = abase;
+    alloc->refcount++;
+    return seg;
+}
+
+static rdb_ROPESEG *
+chunked_alloc(my_CHUNKALLOC *alloc)
+{
+    rdb_ROPESEG *chunk = NULL;
+    lcb_list_t *llcur, *llnext;
+
+    LCB_LIST_SAFE_FOR(llcur, llnext, (lcb_list_t *)&alloc->chunks) {
+        rdb_ROPESEG *cur = LCB_LIST_ITEM(llcur, rdb_ROPESEG, llnode);
+
+        lcb_list_delete(&cur->llnode);
+        if (cur->nalloc != alloc->chunksize) {
+            release_chunk(&alloc->base, cur);
+        } else {
+            chunk = cur;
+            break;
+        }
+    }
+
+    if (chunk) {
+        alloc->refcount++;
+    } else {
+        chunk = standalone_alloc(&alloc->base, alloc->chunksize);
+    }
+
+    chunk->start = 0;
+    chunk->nused = 0;
+    chunk->shflags = RDB_ROPESEG_F_LIB;
+    return chunk;
+}
+
+static void
+buf_reserve(rdb_ALLOCATOR *abase, rdb_ROPEBUF *buf, unsigned n)
+{
+    my_CHUNKALLOC *alloc = (my_CHUNKALLOC *)abase;
+    rdb_ROPESEG *lastseg = RDB_SEG_LAST(buf);
+    unsigned allocated = 0;
+
+    if (lastseg) {
+        if (buf->nused + RDB_SEG_SPACE(lastseg) >= n) {
+            return;
+        }
+
+        n -= (buf->nused + RDB_SEG_SPACE(lastseg));
+    }
+
+    while (allocated < n) {
+        rdb_ROPESEG *seg = chunked_alloc(alloc);
+        lcb_list_append(&buf->segments, &seg->llnode);
+        allocated += seg->nalloc;
+    }
+}
+
+static rdb_ROPESEG *
+seg_realloc(rdb_ALLOCATOR *abase, rdb_ROPESEG *seg, unsigned n)
+{
+    seg->nalloc = n;
+    seg->root = realloc(seg->root, n);
+
+    (void)abase;
+    return seg;
+}
+
+static void
+seg_release(rdb_ALLOCATOR *abase, rdb_ROPESEG *seg)
+{
+    release_chunk(abase, seg);
+    alloc_decref(abase);
+}
+
+LCB_INTERNAL_API
+rdb_ALLOCATOR *
+rdb_chunkalloc_new(unsigned chunksize)
+{
+    rdb_ALLOCATOR *ret;
+    my_CHUNKALLOC *alloc = calloc(1, sizeof(*alloc));
+    alloc->refcount = 1;
+    alloc->chunksize = chunksize;
+    alloc->max_chunks = 512;
+
+    lcb_clist_init(&alloc->chunks);
+    ret = &alloc->base;
+
+    ret->a_release = alloc_decref;
+    ret->r_reserve = buf_reserve;
+    ret->s_alloc = standalone_alloc;
+    ret->s_realloc = seg_realloc;
+    ret->s_release = seg_release;
+    return ret;
+}