oinky 0.1.0

data/ext/include/oinky/nky_core.hpp

@@ -0,0 +1,1603 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+namespace Oinky
+{
+namespace Internal
+{
+using namespace Oinky::Errors;
+using namespace Oinky::Utils;
+
+
+//  This stores the rows in an arbitrary (but consistent) order.  They are 
+//  ordered by sequence number (rowkey for fixed rows).
+typedef boost::intrusive::set_base_hook<> row_tree_hook;
+
+typedef boost::intrusive::rbtree_node_traits<void *,false> row_index_hook_traits;
+typedef row_index_hook_traits::node row_index_hook;
+typedef boost::intrusive::rbtree_algorithms<row_index_hook_traits> treealg;
+
+struct prow_shape_t
+{
+    //  Number of bytes allocated for column data.
+    uint32 column_bytes;
+
+    //  This is the number of index entries included in the pending_row 
+    //  allocation.  This should be >= the number of active indexes.
+    //  The entire allocation need not be used, and the allocation usage
+    //  need not be dense.  If an index is deleted from the table,
+    //  it may leave a hole.
+    index_idx_t idx_count;
+    
+    bool contains(const prow_shape_t &other) const {
+        return (column_bytes >= other.column_bytes) && (idx_count >= other.idx_count);
+    }
+    
+    static prow_shape_t compute_reserve_shape(prow_shape_t min) {
+        prow_shape_t newshape;
+        newshape.idx_count = compute_reserve_size(min.idx_count);
+        newshape.column_bytes = (compute_reserve_size(min.column_bytes) + 7) & ~7;
+        return newshape;
+    }
+    
+    //  When reserving arrays, we over-alloc by some margin.  Given a minimum 
+    //  size, this computes the actual size we allocate.
+    template<typename T>
+    static inline T compute_reserve_size(T min) {
+        uint32 k = ((min * 5) + 7) >> 2;
+        //  Ensure the cast doesn't cause us to return something too small.
+        T v = (T) k;
+        if (v < min) {
+            return min;
+        } else {
+            return v;
+        }
+    }
+};
+
+//  The row itself is in an intrustive list.  The rows are not sorted in their
+//  primary index.
+template<typename TABLE_CTX>
+class pending_row : public row_tree_hook, boost::noncopyable
+{
+public:
+    typedef TABLE_CTX table_ctx;
+    typedef index_ctx_t<table_ctx> index_ctx;
+    typedef pending_row<table_ctx> pending_row_t;
+
+    table_ctx *table() const { return tbl; }
+    uint32 crs_ref_count() const { return cursor_ref_count; }
+    bool is_inactive() const { return pending_destruction || ls.is_deleted(); }
+
+    //  insert/delete markers for this row.
+    //  
+    //  There is no such thing as update.  update ==> delete;insert
+    ls_marker ls;
+private:
+    //  Parent table.
+    table_ctx *tbl;
+
+    //  The row is partitioned into several arrays.  This keeps track of
+    //  how large each array is, and where each begins.
+    prow_shape_t shape;
+    
+    //  How many cursors refer to this row?
+    //  This is more or less the only reference counting we do anywhere.
+    uint32 cursor_ref_count : 31;
+    //  Did we intend to destroy this object?  If the cursor_ref_count
+    //  was nonzero, we might have had to delay destruction, in which
+    //  case we'll do it when the cursor_ref_count reaches zero.
+    //  This never gets unset once it's set.
+    uint32 pending_destruction : 1;
+    
+public:
+    //  This is set during serialization.  It is the rowkey that the row will
+    //  be assigned in the newly serialized table.  It would be nice to combine
+    //  this with the sequence_number below, but that's more difficult than
+    //  it seems.
+    row_idx_t new_rowkey;
+
+    //  This is the sequence number of insertion into the table.  It is the
+    //  secondary sort key for all indices, after the index values themselves.
+    //  Even if the row gets updated or grown, the sequence number will remain
+    //  the same, allowing the cursor position to behave consistently in 
+    //  non-unique indices.  Some mainstream DBs permit cursoring over unique
+    //  indices only, but that's a bit obnoxious.  This sequence number is 
+    //  unique to every row insertion for a particular table, over the lifetime
+    //  of that table.  (It can probably be uint32).  But We never serialize it,
+    //  and it restarts at zero with each mount.
+    //
+    //  It need not increase monotonically, but it does need to be unique over the table.
+    uint64 sequence_number;
+
+    //  The live tree is ordered by sequence number.
+    struct live_tree_compare
+    {
+        bool operator()(uint64 lseq, const pending_row_t &right) const {
+            return right.sequence_number < lseq;
+        }
+        bool operator()(const pending_row_t &left, uint64 rseq) const {
+            return left.sequence_number < rseq;
+        }
+        bool operator()(const pending_row_t &left, const pending_row_t &right) const {
+            return left.sequence_number < right.sequence_number;
+        }
+    };
+    typedef boost::intrusive::rbtree<pending_row_t,
+        boost::intrusive::compare<live_tree_compare> > row_tree_t;
+
+    //  Interlock not necessary, since we are still strictly non-reentrant.
+    //  even in reference counting.  Dereference is never implicit.
+    void cursor_reference() {
+        //  If it's pending destruction or deleted, the only way we can be adding
+        //  a reference is by cloning a cursor.
+        OINKY_ASSERT(!pending_destruction || cursor_ref_count);
+        OINKY_ASSERT(!ls.is_deleted() || cursor_ref_count);
+        cursor_ref_count += 1;
+    }
+    void cursor_dereference() {
+        OINKY_ASSERT(cursor_ref_count);
+        cursor_ref_count -= 1;
+        if (!cursor_ref_count && pending_destruction) {
+            do_delete();
+        }
+    }
+    
+    inline const prow_shape_t &get_shape() const { return shape; }
+
+public:
+    //  These are the table column_values.  The index values are not 
+    //  stored redundantly.  The index contexts use the index definition to
+    //  peek back into these source values.
+    char *column_data() {
+        return (char *) hooks_end();
+    }
+    const char *column_data() const {
+        return (const char *) hooks_end();
+    }
+    const char *column_data_end() const {
+        return column_data() + shape.column_bytes;
+    }
+
+    //  The index hooks are stored immediately after the pending_row
+    //  structure itself.  This is the only way we can backtrack to get
+    //  from the index hook to the row pointer.  The column values are stored
+    //  after the index hooks.
+    row_index_hook *hooks_begin() {
+        return (row_index_hook *) (this + 1);
+    }
+    row_index_hook *hooks_end() {
+        return hooks_begin() + shape.idx_count;
+    }
+    const row_index_hook *hooks_begin() const {
+        return (row_index_hook *) (this + 1);
+    }
+    const row_index_hook *hooks_end() const {
+        return hooks_begin() + shape.idx_count;
+    }
+
+    //  Convert an index entry to a row object.
+    static const pending_row_t *from_index_hook(const row_index_hook *hook, const index_ctx *idx) {
+        OINKY_ASSERT(hook && (hook != &idx->head));
+        index_idx_t index_idx = idx->index_idx;
+        const pending_row_t *row = (const pending_row_t *)(hook - index_idx) - 1;
+        OINKY_ASSERT(idx->table == row->tbl);
+        OINKY_ASSERT(index_idx < row->shape.idx_count);
+        return row;
+    }
+    
+    static pending_row_t *from_index_hook(row_index_hook *hook, const index_ctx *idx) {
+        OINKY_ASSERT(hook && (hook != &idx->head));
+        index_idx_t index_idx = idx->index_idx;
+        pending_row_t *row = (pending_row_t *)(hook - index_idx) - 1;
+        OINKY_ASSERT(idx->table == row->tbl);
+        OINKY_ASSERT(index_idx < row->shape.idx_count);
+        return row;
+    }
+
+    row_index_hook *get_hook(index_idx_t index_idx) {
+        OINKY_ASSERT(index_idx < shape.idx_count);
+        return hooks_begin() + index_idx;
+    }
+    
+    static size_t comp_size(const prow_shape_t &rs) {
+        return 
+            sizeof(pending_row_t) + 
+            rs.column_bytes +
+            (rs.idx_count * sizeof(row_index_hook));
+    }
+
+private:
+    //  The DB handle tells us which pool to use.  This gives better cache
+    //  locality as well as a lock-free pool implementation.
+    //
+    //  I've never overloaded 'operator new' before, and I'm pretty
+    //  sure I don't want to.  I just make the constructor
+    //  private and create this static factory.
+    //
+    pending_row(table_ctx *_tbl, const prow_shape_t &_shape) : 
+    tbl(_tbl),
+        shape(_shape),
+        cursor_ref_count(0),
+        pending_destruction(0)
+    {}
+    ~pending_row() {
+        OINKY_ASSERT(!cursor_ref_count);
+    }
+    
+public:    
+    static pending_row_t *make_new(table_ctx *tbl) {
+        prow_shape_t shape = tbl->get_minimum_prow_shape();
+        return make_new(tbl, shape);
+    }
+    
+    static pending_row_t *make_new(table_ctx *tbl, const prow_shape_t &_shape) {
+        pending_row_t *row = (pending_row_t *) 
+            tbl->db->allocator.malloc(pending_row_t::comp_size(_shape));
+            
+        //  Invoke the placement constructor.
+        ::new(row) pending_row_t(
+            tbl, 
+            _shape);
+        return row;
+    }
+    
+    void try_delete()
+    {
+        if (cursor_ref_count) {
+            pending_destruction = 1;
+        } else {
+            do_delete();
+        }
+    }
+    
+private:
+    void do_delete()
+    {
+        table_ctx *table = tbl;
+        //  destroy
+        this->~pending_row();
+        //  free
+        table->db->allocator.free(this);
+    }
+    
+    void operator delete(void *_row) {
+        //  Should not be calling this.
+        OINKY_ASSERT(false);
+    }
+    
+public:    
+    //  If an index is added subsequent to the creation of this row, we may
+    //  need to expand this row's allocation to fit the new index entry.
+    //  This method does that check, and if necessary, allocates a new object
+    //  to replace this one, preserving all existing indexes.
+    //
+    //  NOTE:  This routine can delete this!  If it does, it returns its
+    //  replacement.
+    pending_row_t *cond_grow_self(const prow_shape_t &_minshape, const prow_shape_t &_newshape) 
+    {
+        //  Why grow a dead entry???
+        //  Also we depend on the fact that we're on the live list below.
+        OINKY_ASSERT( ls.delete_sp == 0 );
+        OINKY_ASSERT(!pending_destruction);
+        
+        //  Logic.
+        OINKY_ASSERT( _newshape.contains(_minshape) );
+        OINKY_ASSERT( _newshape.column_bytes >= tbl->prow_byte_limit );
+        OINKY_ASSERT( (uint32) _newshape.idx_count >= tbl->indexes_by_position.limit() );
+        
+        //  _newshape is what we allocate, but specified mins are what we need.
+        if (shape.contains(_minshape)) {
+            return this;
+        }
+        
+        pending_row_t *nr = make_new(tbl, _newshape);
+        nr->ls = ls;
+        
+        //  Copy the sequence number.
+        nr->sequence_number = sequence_number;
+        
+        //  Copy all the column data.
+        memcpy(nr->column_data(), column_data(), MIN(shape.column_bytes, _newshape.column_bytes));
+        
+        //  Replace every index entry with the new node, fully unlinking this node
+        //  in the process.  Make sure we only replace the index nodes that are
+        //  actually valid.
+        typedef typename table_ctx::index_positional_itr_t tbl_idx_itr_t;
+        tbl_idx_itr_t idx_end = tbl->indexes_by_position.end();
+        for (tbl_idx_itr_t i = tbl->indexes_by_position.begin(); i != idx_end; ++i) {
+            //  Not all index trees are valid.  Unused ones are uninitialized.
+            if (*i) {
+                index_idx_t index_idx = (*i)->index_idx;
+                treealg::replace_node(get_hook(index_idx), &(*i)->head, nr->get_hook(index_idx));
+            }
+        }
+        //  Now swap the primary list entry.
+        typename row_tree_t::iterator i = row_tree_t::s_iterator_to(*this);
+        tbl->live_pending.insert_equal(i, *nr);
+        tbl->live_pending.erase(i);
+        //  Delete the old and return the new.
+        try_delete();
+        return nr;
+    }
+    
+    template<template<typename T> class CB, typename CB_CC>
+    bool dispatch_value(const column_ctx &cc, CB_CC *ctx) const {
+        const char *data = column_data() + cc.prow_byte_offset;
+        switch (cc.ctype) {
+            case column_types::Bit : {
+                uint8 mask = 1 << cc.prow_bit_offset;
+                bool v = (*data & mask) != 0;
+                return CB<bool>::call(cc, v,ctx);
+            }
+            //int
+            case column_types::Int8 :
+                return CB<int8>::call(cc, *(const int8 *)data,ctx);
+            case column_types::Int16 :
+                return CB<int16>::call(cc, *(const int16 *)data,ctx);
+            case column_types::Int32 :
+                return CB<int32>::call(cc, *(const int32 *)data,ctx);
+            case column_types::Int64 :
+                return CB<int64>::call(cc, *(const int64 *)data,ctx);
+            //uint
+            case column_types::Uint8 :
+                return CB<uint8>::call(cc, *(const uint8 *)data,ctx);
+            case column_types::Uint16 :
+                return CB<uint16>::call(cc, *(const uint16 *)data,ctx);
+            case column_types::Uint32 :
+                return CB<uint32>::call(cc, *(const uint32 *)data,ctx);
+            case column_types::Uint64 :
+                return CB<uint64>::call(cc, *(const uint64 *)data,ctx);
+            //float
+            case column_types::Float32 :
+                return CB<float32>::call(cc, *(const float32 *)data,ctx);
+            case column_types::Float64 :
+                return CB<float64>::call(cc, *(const float64 *)data,ctx);
+
+            case column_types::Datetime :
+                return CB<datetime_t>::call(cc, *(const datetime_t *)data,ctx);
+            //  Variants and strings are stored as safe_cv_t values.
+            case column_types::String :
+            case column_types::Variant :
+                return CB<safe_cv_t>::call(cc, *(const safe_cv_t *)data,ctx);
+        }
+        throw_error(implementation_bug());
+    }
+
+    safe_cv_t get_value(const column_ctx &ctx) const {
+        safe_cv_t cv;
+        dispatch_value<get_value_cb>(ctx, &cv);
+        return cv;
+    }
+
+    template<typename T>
+    struct get_value_cb {
+        static inline bool call(const column_ctx &ctx, const T &x, safe_cv_t *val) {
+            *val = x;
+            return true;
+        }
+    };
+
+private:
+    template<typename T>
+    inline void priv_set_value(const column_ctx &ctx, const T &v) {
+        char *data = column_data() + ctx.prow_byte_offset;
+        *(T*)data = v;
+    }
+public:
+    //  These avoid the extra switch if we get direct dispatch from the
+    //  fixed enumerator, for example.
+    //int
+    void set_value(const column_ctx &cc, int8 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Int8);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, int16 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Int16);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, int32 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Int32);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, int64 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Int64);
+        priv_set_value(cc,val);
+    }
+    //uint
+    void set_value(const column_ctx &cc, uint8 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Uint8);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, uint16 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Uint16);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, uint32 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Uint32);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, uint64 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Uint64);
+        priv_set_value(cc,val);
+    }
+    //float
+    void set_value(const column_ctx &cc, float32 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Float32);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, float64 val) {
+        OINKY_ASSERT(cc.ctype == column_types::Float64);
+        priv_set_value(cc,val);
+    }
+    
+    void set_value(const column_ctx &cc, datetime_t val) {
+        OINKY_ASSERT(cc.ctype == column_types::Datetime);
+        priv_set_value(cc,val);
+    }
+    void set_value(const column_ctx &cc, bool v) {
+        OINKY_ASSERT(cc.ctype == column_types::Bit);
+        char *data = column_data() + cc.prow_byte_offset;
+        uint8 mask = 1 << cc.prow_bit_offset;
+        if (v) {
+            *data |= mask;
+        } else {
+            *data &= ~mask;
+        }
+    }
+    
+    void set_value(const column_ctx &cc, const safe_cv_t &val) {
+        const char *data = column_data() + cc.prow_byte_offset;
+        switch (cc.ctype) {
+            case column_types::Bit :
+                set_value(cc, val.bit_value());
+                return;
+            //int
+            case column_types::Int8 :
+                priv_set_value<int8>(cc, val.int_value());
+                return;
+            case column_types::Int16 :
+                priv_set_value<int16>(cc, val.int_value());
+                return;
+            case column_types::Int32 :
+                priv_set_value<int32>(cc, val.int_value());
+                return;
+            case column_types::Int64 :
+                priv_set_value<int64>(cc, val.int_value());
+                return;
+            //uint
+            case column_types::Uint8 :
+                priv_set_value<uint8>(cc, val.uint_value());
+                return;
+            case column_types::Uint16 :
+                priv_set_value<uint16>(cc, val.uint_value());
+                return;
+            case column_types::Uint32 :
+                priv_set_value<uint32>(cc, val.uint_value());
+                return;
+            case column_types::Uint64 :
+                priv_set_value<uint64>(cc, val.uint_value());
+                return;
+            //float
+            case column_types::Float32 :
+                priv_set_value<float32>(cc, val.f32_value());
+                return;
+            case column_types::Float64 :
+                priv_set_value<float64>(cc, val.f64_value());
+                return;
+                
+            case column_types::Datetime :
+                priv_set_value<datetime_t>(cc, val.dt_value());
+                return;
+            //  Variants and strings are stored as safe_cv_t values.
+            case column_types::String :
+            case column_types::Variant :
+                priv_set_value<safe_cv_t>(cc, val);
+                return;
+        }
+        throw_error(implementation_bug());
+    }
+
+    template<template<typename T> class CB, typename CTX>
+    void each_raw_value(const column_ctx *const* cols_begin, const column_ctx *const* cols_end, CTX *ctx) const {
+        for (const column_ctx *const* i = cols_begin; i != cols_end; ++i) {
+            const column_ctx &col(**i);
+            if (!dispatch_value<CB>(col, ctx)) {
+                return;
+            }
+        }
+    }
+
+    template<template<typename T> class CB, typename SELECTOR, typename CTX>
+    void each_raw_value(const SELECTOR &_selector, CTX *ctx) const {
+        _selector.check_valid(tbl);
+        column_selector_accessor<SELECTOR> selector(_selector);
+        uint32 l = selector.colcount();
+        const column_ctx *const * cols_begin = selector.colrefs();
+        const column_ctx *const * cols_end = cols_begin + l;
+        each_raw_value<CB>(cols_begin, cols_end, ctx);
+    }
+
+    template<typename FN>
+    struct variant_cb {
+        template<typename T>
+        struct fn {
+            inline static bool call(const column_ctx &ctx, T value, const FN *fn) {
+                safe_cv_t v(value);
+                return (*fn)(ctx, v);
+            }
+        };
+    };
+    
+    //  Parameters are a column definition and a value
+    template<typename SELECTOR, typename FN>
+    void each_column_value(const SELECTOR &_selector, FN fn) const {
+        //  This does nothing at runtime.  But it does verify the FN signature, 
+        //  and if in error, it generates an error with fewer levels of 
+        //  template indirection than the next line.
+        check_function_concept<bool(const column_ctx &, const safe_cv_t &cv)>(fn); 
+
+        //  Map the raw callback values to safe_cv_t values before passing to 
+        //  the callback.
+        each_raw_value<variant_cb<FN>::template fn>(_selector, &fn);
+    }
+};
+
+//
+//  The default traits just assume that the context can be cast to 
+//  context_base.
+//
+template<typename CONTEXT_T, typename INDEX_T, typename ALLOCATOR_T>
+struct default_ctx_map_traits
+{
+    typedef CONTEXT_T context_t;
+    typedef INDEX_T index_t;
+    typedef ALLOCATOR_T allocator_t;
+    
+    //  These are template methods so that they can be individually overriden,
+    //  and nonsense implementations will be ignored until they are instantiated.
+    template<typename CTX_T> 
+    static const mstring_safe &key_from_ctx(const CTX_T *ctx) { 
+        return ctx->name;
+    }
+    template<typename CTX_T> 
+    static ls_marker &ls_from_ctx(CTX_T *ctx) {
+        return ctx->ls;
+    }
+    template<typename CTX_T> 
+    static const ls_marker &ls_from_ctx(const CTX_T *ctx) {
+        return ctx->ls;
+    }
+    template<typename CTX_T> 
+    static void on_deactivation(CTX_T *ctx) {
+        ctx->on_deactivation();
+    }
+};
+
+template<typename TRAITS_T>
+class context_map_t
+{
+    typedef context_map_t<TRAITS_T> context_map;
+    
+    typedef typename TRAITS_T::allocator_t allocator_t;
+    typedef typename TRAITS_T::index_t index_t;
+    typedef typename TRAITS_T::context_t context_t;
+
+    meta_stringtable_t *stringtable;
+    db_fixed_vector<context_t> f_contexts;
+    uint32 _live_count;
+    
+    context_t *fbegin() { return f_contexts.begin(); }
+    context_t *fend() { return f_contexts.end(); }
+    const context_t *fbegin() const { return f_contexts.begin(); }
+    const context_t *fend() const { return f_contexts.end(); }
+
+    //
+    //  NOTE:
+    //  We use the global heap for the context hosts.  It's not a big deal.
+    //  They only get allocated when the schema's being modified anyway.
+    //  This is more expediency than any good reason.
+    //  
+    //  Note also that there is no indirection in the vector.  The contexts
+    //  are allocated contiguously.  This can't be done with the dynamic
+    //  object because the contexts may be noncopyable.
+    
+    //Define a set using the base hook that will store values in reverse order
+
+    typedef boost::intrusive::avl_set_base_hook<> tree_hook;
+    struct context_host : tree_hook
+    {
+        context_t ctx;
+        int in_live : 1;
+        int in_deleted : 1;
+
+        const mstring_safe &key() const { 
+            return TRAITS_T::key_from_ctx(&ctx);
+        }
+        ls_marker &ls() { 
+            return TRAITS_T::ls_from_ctx(&ctx);
+        }
+        const ls_marker &ls() const { 
+            return TRAITS_T::ls_from_ctx(&ctx);
+        }
+        
+        context_host() : 
+        in_live(0),
+            in_deleted(0)
+        {}
+    };
+
+    struct host_compare_live
+    {
+        template<typename KEY>
+        bool operator()(const KEY &lkey, const context_host &right) const
+        {
+            return -(*this)(right,lkey);
+        }
+        template<typename KEY>
+        bool operator()(const context_host &left, const KEY &rkey) const
+        {
+            OINKY_ASSERT(!left.ls().is_deleted());
+            OINKY_ASSERT(left.in_live);
+            return left.key().compare_to(rkey) < 0;
+        }
+        bool operator()(const context_host &left, const context_host &right) const
+        {
+            OINKY_ASSERT(right.in_live && left.in_live);
+            OINKY_ASSERT(!right.in_deleted && !left.in_deleted);
+            return left.key().compare_to(right.key()) < 0;
+        }
+    };
+
+    struct host_compare_deleted
+    {
+        bool operator()(const context_host &left, const context_host &right) const
+        {
+            OINKY_ASSERT(left.ls().is_deleted());
+            OINKY_ASSERT(right.ls().is_deleted());
+            OINKY_ASSERT(!right.in_live && !left.in_live);
+            OINKY_ASSERT(right.in_deleted && left.in_deleted);
+            int k = left.key().compare_to(right.key());
+            if (k) return (k < 0);
+            //  We compare the insertion SP, not the delete SP.  The
+            //  insertion SP should never match, because an intervening
+            //  delete would have been required, which would also have
+            //  the same SP as the inserts, which should have caused
+            //  a hard delete of the first.
+            OINKY_ASSERT(left.ls().insert_sp != right.ls().insert_sp);
+            return left.ls().insert_sp < right.ls().insert_sp;
+        }
+    };
+
+    //  The live contexts.
+    typedef boost::intrusive::avl_set<
+        context_host, 
+        boost::intrusive::compare<host_compare_live> > live_tree_t;
+    typedef typename live_tree_t::iterator lt_itr_t;
+    typedef typename live_tree_t::const_iterator lt_citr_t;
+    
+    //  The deleted contexts. (Preserved for savepoint rollback.)
+    typedef boost::intrusive::avl_set<
+        context_host, 
+        boost::intrusive::compare<host_compare_deleted> > deleted_tree_t;
+    typedef typename deleted_tree_t::iterator dt_itr_t;
+    typedef typename deleted_tree_t::const_iterator dt_citr_t;
+    
+    allocator_t *allocator;
+    live_tree_t live;
+    deleted_tree_t deleted;
+    
+    uint32 computed_packed_header_bytes;
+
+public:
+    context_map_t(allocator_t *_allocator, meta_stringtable_t *_stringtable) :
+    stringtable(_stringtable),
+        f_contexts(_allocator),
+        allocator(_allocator),
+        _live_count(0)
+    {}
+    
+    ~context_map_t() {
+        destroy_dynamic();
+    }
+
+private:
+    static void destroy_item(context_host *h) {
+        delete h;
+    }
+
+    void destroy_dynamic() {
+        live.clear_and_dispose(&destroy_item);
+        deleted.clear_and_dispose(&destroy_item);
+    }
+public:
+    struct get_name_ref
+    {
+        typedef m_strtable_ref result_type;
+        m_strtable_ref operator()(const context_t *ref) const { return key(ref).ref; }
+    };
+    typedef boost::transform_iterator<get_name_ref,context_t *> fixed_name_ref_itr;
+
+    //  This is the fastest way of finding a specific table.
+    context_t *find(const db_string &name)
+    {
+        //  Search the dynamic table.
+        lt_itr_t i = live.find(name);
+        if (i != live.end()) {
+            //  This should have been moved to the deleted list if it
+            //  was deleted.
+            OINKY_ASSERT(!i->is_deleted());
+            return &i->ctx;
+        }
+        //  Before we can search the fixed table, must convert to a ref.
+        m_strtable_ref mref;
+        typename meta_stringtable_t::iterator smi = stringtable->find(name);
+        if (smi == stringtable->end()) {
+            //  Not in the string table, which means an object with that name
+            //  can't exist in serialized form yet.
+            return NULL;
+        }
+        mref = *smi;
+        //  Find the ref in the fixed table.
+        fixed_name_ref_itr fni = std::find(fixed_name_ref_itr(fbegin()), fixed_name_ref_itr(fend()), mref);
+        //  Convert back to a context iterator.
+        context_t * k = fni - fbegin();
+        if (k == fend()) {
+            //  Not in the fixed set.
+            return NULL;
+        }
+        //  It's in the fixed set, but we now have to check if it was deleted.
+        if (k->ls().is_deleted()) {
+            return NULL;
+        }
+        return k;
+    }    
+private:    
+    context_host *host_from_ctx(context_t *ctx)
+    {
+        if ((ctx >= fbegin()) && (ctx < fend())) {
+            //  Here we don't free the context explicitly.
+            return NULL;
+        } else {
+            int offset = ((char *) &((context_host *)(5))->ctx) - ((char *) 5);
+            context_host *host = (context_host *) (((char *)ctx) - offset);
+            return host;
+        }
+    }
+    
+    //  Move from the live to the deleted tree.
+    void delete_ctx(context_t *ctx)
+    {
+        OINKY_ASSERT(ls(ctx).is_deleted());
+        context_host *h = host_from_ctx(ctx);
+        OINKY_ASSERT(h);
+        if (h) {
+            delete h;
+        }
+    }
+    
+    static const mstring_safe &key(const context_t *ctx) { 
+        return TRAITS_T::key_from_ctx(ctx);
+    }
+    static const ls_marker &ls(const context_t *ctx) { 
+        return TRAITS_T::ls_from_ctx(ctx);
+    }
+    static ls_marker &ls(context_t *ctx) { 
+        return TRAITS_T::ls_from_ctx(ctx);
+    }
+
+    lt_itr_t insert_live(context_host *h) 
+    {
+        OINKY_ASSERT(h->ls().insert_sp > 0);
+        OINKY_ASSERT(!h->ls().is_deleted());
+        OINKY_ASSERT(!h->in_live && !h->in_deleted);
+        
+        h->in_live = 1;
+        std::pair<lt_itr_t, bool> r = live.insert(*h);
+        
+        //  A conficting name is an exception, which at this level generally 
+        //  points to an internal implementation error.
+        if (!r.second) {
+            h->in_live = 0;
+            OINKY_ASSERT(false);
+            throw_error(object_exists());
+        }
+        
+        return r.first;
+    }
+
+    static inline bool notmodified_cb(const context_t *ctx, sp_marker_t tsp) { return false; }
+public:
+    template<typename CB>
+    bool modified_since(sp_marker_t tsp, CB cb) const
+    {
+        check_function_concept<bool(const context_t *ctx, sp_marker_t tsp)>(cb); 
+
+        lt_citr_t lend = live.end();
+        for (lt_citr_t i = live.begin(); i != lend; ++i) {
+            if (i->ls().insert_sp > tsp) {
+                return true;
+            }
+            const context_host *h = &(*i);
+            if ((i->ls().delete_sp == 0) && cb(&h->ctx, tsp)) {
+                return true;
+            }
+        }
+        
+        for (const context_t * i = fbegin(); i != fend(); ++i) {
+            OINKY_ASSERT(ls(i).insert_sp == 0);
+            if (ls(i).delete_sp > tsp) {
+                return true;
+            }
+            if ((ls(i).delete_sp == 0) && cb(i, tsp)) {
+                return true;
+            }
+        }
+        dt_citr_t dend = deleted.end();
+        for (dt_citr_t i = deleted.begin(); i != dend; ) {
+            const context_host *h = &(*i);
+            OINKY_ASSERT(h->ls().insert_sp > 0);
+            if (h->ls().delete_sp > tsp) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    bool modified_since(sp_marker_t tsp) const {
+        return modified_since(tsp, &notmodified_cb);
+    }
+    
+    //  roll back all drops/creates which have an SP marker higher than target_sp.
+    template<typename UNCREATE_CB, typename UNDROP_CB, typename UNCREATE_DROP_CB>
+    void sp_rollback(sp_marker_t target_sp, UNCREATE_CB uncreate_cb, UNDROP_CB undrop_cb, UNCREATE_DROP_CB uncreate_drop_cb)
+    {
+        //  To avoid namespace collisions, we first do all the uncreates.
+        lt_itr_t lend = live.end();
+        for (lt_itr_t i = live.begin(); i != lend; ) {
+            if (i->ls().insert_sp > target_sp) {
+                context_host *h = &(*i);
+                _live_count -= 1;
+                uncreate_cb(&h->ctx, target_sp);
+                TRAITS_T::on_deactivation(&h->ctx);
+                lt_itr_t tmp = i;
+                ++i;
+                live.erase(tmp);
+                h->in_live = 0;
+                delete h;
+            } else {
+                ++i;
+            }
+        }
+        //  Then we do the undrops.  Here we have to be sure to check the 
+        //  fixed set as well, which we don't have to worry about un-creating.
+        for (context_t * i = fbegin(); i != fend(); ++i) {
+            OINKY_ASSERT(ls(i).insert_sp == 0);
+            if (ls(i).delete_sp > target_sp) {
+                ls(i).delete_sp = 0;
+                _live_count += 1;
+                undrop_cb(i, target_sp);
+            }
+        }
+        dt_itr_t dend = deleted.end();
+        for (dt_itr_t i = deleted.begin(); i != dend; ) {
+            context_host *h = &(*i);
+            OINKY_ASSERT(h->ls().insert_sp > 0);
+            if (h->ls().insert_sp > target_sp) {
+                //  Here we can hard-delete if we advance beyond create
+                dt_itr_t tmp = i;
+                ++i;
+                deleted.erase(tmp);
+                h->in_deleted = 0;
+                //  callback for rolling back both the create and the drop.
+                uncreate_drop_cb(&h->ctx, target_sp);
+                delete h;
+            } else if (h->ls().delete_sp > target_sp) {
+                //  Otherwise, we are rolling back to between create
+                //  and drop, which means we un-drop
+                dt_itr_t tmp = i;
+                ++i;
+                deleted.erase(tmp);
+                h->in_deleted = 0;
+                h->in_live = 1;
+                h->ls().delete_sp = 0;
+                live.insert(*h);
+                _live_count += 1;
+                undrop_cb(&h->ctx, target_sp);
+            } else {
+                ++i;
+            }
+        }
+    }
+
+    //  Serialization and user-accesses always iterate in order, for different 
+    //  reasons.  Context sequences are always ordered by name in serialization.
+    //  User accesses are ordered for simplicity and consistency.
+    class by_name_t
+    {
+        //  The left iterator is over the fixed contexts...skipping the deleted
+        //  ones, and returning pointers to the context objects.
+        class fixed_nod_itr_t : public 
+            boost::iterator_facade<
+                fixed_nod_itr_t,
+                context_t *,
+                boost::random_access_traversal_tag,
+                context_t *
+                >
+        {
+            typedef fixed_nod_itr_t this_t;
+            
+            context_t *dereference() const { return ptr; }
+            bool equal(const this_t &other) const { return ptr == other.ptr; }
+            void increment() { ++ptr; }
+            void decrement() { --ptr; }
+            void advance(int x) { ptr += x; }
+            int distance_to(const this_t &other) const { return ptr - other.ptr; }
+
+            context_t *ptr;
+            friend class boost::iterator_core_access;
+        public:
+            fixed_nod_itr_t() : ptr(NULL) {}
+            fixed_nod_itr_t(context_t *_ptr) : ptr(_ptr) {}
+        };
+        
+        struct skipdeleted {
+            bool operator()(const fixed_nod_itr_t &i) const {
+                return !ls(*i).is_deleted();
+            }
+        };
+
+        typedef boost::filter_iterator<skipdeleted, fixed_nod_itr_t> bn_fixed;
+                
+        //  The right iterator returns pointers to context objects from the live 
+        //  tree.  This does not require skipping anything, but it does
+        //  still require a transform from the context_host object.
+        struct from_host {
+            typedef context_t *result_type;
+            context_t *operator()(context_host &i) const {
+                return &(i.ctx);
+            }
+        };
+        typedef boost::transform_iterator<from_host, lt_itr_t> bn_dynamic;
+
+        bn_dynamic make_dynamic_itr(const lt_itr_t &where) const {
+            return bn_dynamic(where, from_host());
+        }
+
+        //  The merge iterator zips the two
+        BOOST_STATIC_ASSERT((boost::is_same<typename bn_fixed::value_type, context_t *>::value));
+        BOOST_STATIC_ASSERT((boost::is_same<typename bn_dynamic::value_type, context_t *>::value));
+        
+        struct compare_fd {
+            template<typename LEFT, typename RIGHT>
+            int operator()(const LEFT &left, const RIGHT &right) const {
+                return key(*left).compare_to(key(*right));
+            }
+        };
+        
+        typedef symmetric_iterator<bn_fixed> fixed_range_t;
+        typedef symmetric_iterator<bn_dynamic> dynamic_range_t;
+        typedef merge_iterator<bn_fixed, bn_dynamic, compare_fd> itr_base_t;
+        
+    public:
+        class iterator : public boost::iterator_adaptor<
+            iterator,
+            itr_base_t
+            >
+        {
+            typedef boost::iterator_adaptor<iterator,itr_base_t> base_t;
+            
+        public:
+            iterator() {}
+            
+            iterator(const fixed_range_t &f, const dynamic_range_t &d) : 
+            base_t(itr_base_t(f,d))
+            {}
+            
+        private:
+            const bn_fixed &fixed() const { return base_t::base().left(); }
+            const bn_dynamic &dynamic() const { return base_t::base().right(); }
+            
+            friend class by_name_t;
+            friend class context_map_t<TRAITS_T>;
+        };
+        
+    private:
+        context_map &map;
+        by_name_t(context_map &_map) : map(_map) {}
+        friend class context_map_t<TRAITS_T>;
+
+        bn_fixed bn_begin() { return make_fixed(map.fbegin()); }
+        bn_fixed bn_end() { return make_fixed(map.fend()); }
+
+        bn_fixed make_fixed(context_t *where) {
+            OINKY_ASSERT(where >= map.fbegin());
+            OINKY_ASSERT(where <= map.fend());
+            return bn_fixed(fixed_nod_itr_t(where), fixed_nod_itr_t(map.fend()));
+        }
+
+        iterator make(
+            const bn_fixed &fi, 
+            const bn_dynamic &di) 
+        { 
+            return iterator(
+                fixed_range_t(fi, bn_begin(), bn_end()), 
+                dynamic_range_t(di, bn_dynamic(map.live.begin()), bn_dynamic(map.live.end()))); 
+        }
+
+        struct get_name
+        {
+            typedef mstring_safe result_type;
+            const mstring_safe &operator()(const context_t &ref) const { return key(&ref); }
+        };
+        typedef boost::transform_iterator<get_name,context_t *> fixed_name_itr;
+        
+    public:
+        iterator make(const bn_fixed &fi, const lt_itr_t &li) {
+            return make(fi, bn_dynamic(li));
+        }
+    
+        iterator begin() { 
+            return make(bn_begin(), bn_dynamic(map.live.begin())); 
+        }
+        iterator end() { 
+            return make(bn_end(), bn_dynamic(map.live.end())); 
+        }
+        
+        iterator lower_bound(const db_string &name) {
+            if (!name.validate_metastring()) {
+                throw_error(invalid_argument());
+            }
+            //  FNI is a random-access iterator, which is useful for search.
+            fixed_name_itr fnb = fixed_name_itr(map.fbegin());
+            fixed_name_itr fni = std::lower_bound(fnb,fixed_name_itr(map.fend()), name);
+            //  bn_fixed is a traversal iterator, which we can obtain from the above random-access
+            //  iterator once we know the offset.
+            bn_fixed fitr = make_fixed(map.fbegin() + (fni - fnb));
+            return make(
+                fitr,
+                bn_dynamic(map.live.lower_bound(name, host_compare_live()))); 
+        }
+        iterator upper_bound(const db_string &name) {
+            if (!name.validate_metastring()) {
+                throw_error(invalid_argument());
+            }
+            //  FNI is a random-access iterator, which is useful for search.
+            fixed_name_itr fnb = fixed_name_itr(map.fbegin());
+            fixed_name_itr fni = std::upper_bound(fnb,fixed_name_itr(map.fend()), name);
+            //  bn_fixed is a traversal iterator, which we can obtain from the above random-access
+            //  iterator once we know the offset.
+            bn_fixed fitr = make_fixed(map.fbegin() + (fni - fnb));
+            return make(
+                fitr,
+                bn_dynamic(map.live.upper_bound(name, host_compare_live()))); 
+        }
+        iterator find(const db_string &name) {
+            //  Find is tricky.  Can't just invoke find on both iterators, as we do
+            //  in the case of upper/lower bound, because one may be valid, and 
+            //  the other at the end.  A merge_iterator cannot be constructed from
+            //  two such misaligned iterators.  Iterating from that point won't 
+            //  increment or decrement correctly.
+            iterator i = lower_bound(name);
+            if ((i == end()) || (key(*i) != name)) { 
+                return end(); 
+            }
+            return i;
+        }
+
+        template<typename FN>
+        iterator insert(const FN &fn, const mstring_safe &name, sp_marker_t sp)
+        {
+            return map.insert(fn, name, sp);
+        }
+    };
+    typedef typename by_name_t::iterator by_name_iterator;
+
+    by_name_t by_name() { return by_name_t(*this); }
+
+    inline uint32 hard_context_count_limit() const {
+        return (uint32) (index_t) -1;
+    }
+
+private:
+
+    //  Allocates a context and returns iterator to it.  The context is 
+    //  constructed from a copy of the given context.  No reference to the
+    //  given context is retained.
+    template<typename FN>
+    by_name_iterator insert(const FN &fn, const mstring_safe &name, sp_marker_t sp)
+    {
+        by_name_iterator existing = by_name().lower_bound(name);
+        if ((existing != by_name().end()) && (key(*existing) == name)) {
+            //  Caller is still internal, and should have checked this before
+            //  (potentially) extending the index range.
+            OINKY_ASSERT(false);
+            throw_error(object_exists());
+        }
+        
+        //  Check that we haven't reached a max-live-count.
+        if (_live_count == hard_context_count_limit()) {
+            throw_error(limit_exceeded());
+        }
+
+        //  This can raise.
+        std::auto_ptr<context_host> h(new context_host());
+        context_t *ctx = &(h->ctx);
+        
+        //  And so can this.
+        fn(ctx, name);
+        //  But nothing else.
+
+        //  The prototype should set the name.
+        OINKY_ASSERT(TRAITS_T::key_from_ctx(ctx) == name);
+        //  We set this here, not in the prototype
+        OINKY_ASSERT(TRAITS_T::ls_from_ctx(ctx).insert_sp == 0);
+        OINKY_ASSERT(TRAITS_T::ls_from_ctx(ctx).delete_sp == 0);
+        TRAITS_T::ls_from_ctx(ctx).insert_sp = sp;
+
+        //  Now that we've set the SP and initialized everything, we
+        //  insert in the live tree.
+        lt_itr_t li = insert_live(h.get());
+        _live_count += 1;
+        
+        //  Now we're linked, so better not free.
+        h.release();
+        
+        //  The fixed position will be the same (one beyond) but the dynamic 
+        //  will now point precisely to the new entry.
+        by_name_iterator new_bn = by_name().make(existing.fixed(), li);
+        by_name_iterator tmp = new_bn; (void) tmp;
+        OINKY_ASSERT( ++tmp == existing );
+        return new_bn;
+    }
+    
+    //  To drop (column/index/table), we must specify the savepoint responsible
+    //  for the delete, so we know when to roll back.
+    void drop(context_t *ctx, sp_marker_t sp) {
+        //  If this context is being deleted in the same SP in which it was
+        //  inserted, we will do a hard-delete.
+        OINKY_ASSERT(ls(ctx).delete_sp == 0);
+        OINKY_ASSERT(sp >= ls(ctx).insert_sp);
+        OINKY_ASSERT(sp > 0);
+
+        //  Do the target callback
+        TRAITS_T::on_deactivation(ctx);
+        _live_count -= 1;
+
+        //  Remove from the name index
+        context_host *h = host_from_ctx(ctx);
+        if (h) {
+            //  This context was created dynamically.
+            OINKY_ASSERT(h->in_live && !h->in_deleted);
+            OINKY_ASSERT(ls(ctx).insert_sp != 0);
+            live.erase(*h);
+            ls(ctx).delete_sp = sp;
+            h->in_live = 0;
+            
+            //  If we are dropping in same SP as we deleted, then we 
+            //  can hard-delete.
+            if (ls(ctx).delete_sp == ls(ctx).insert_sp) {
+                delete_ctx(ctx);
+            } else {
+                h->in_deleted = 1;
+                std::pair<dt_itr_t, bool> r = deleted.insert(*h);
+                OINKY_ASSERT(r.second);
+            }
+        } else {
+            //  This context was unpacked from serialized form.
+            OINKY_ASSERT(ls(ctx).insert_sp == 0);
+            ls(ctx).delete_sp = sp;
+        }
+    }
+private:
+    
+    template<typename T, typename SCHEME>
+    friend class ::Oinky::Serialization::Serializer;
+
+    template<typename BASE>
+    struct set_accessor_identity : public BASE
+    {
+        set_accessor_identity(context_map &_map) : BASE(_map) {}
+    };
+
+public:    
+    template<typename ITR_VALUE, 
+        template<typename BASE> class ACCESSOR = set_accessor_identity>
+    class set_accessor_t
+    {
+        class accessor_base
+        {
+        protected:
+            context_map &map;
+        public:
+            class iterator : public boost::iterator_adaptor<
+                iterator,
+                by_name_iterator,
+                ITR_VALUE
+                >
+            {
+                typedef boost::iterator_adaptor<iterator,by_name_iterator,ITR_VALUE> base_t;
+                friend class accessor_base;
+                
+                const by_name_iterator &source_itr() const { return base_t::base(); }
+                
+            public:
+                iterator() {}
+                iterator(const by_name_iterator &b) : base_t(b) {}
+                
+                const ITR_VALUE &operator*() { return dereference(); }
+                const ITR_VALUE *operator->() { return &dereference(); }
+                
+            private:
+                //  Obviously, we do NOT want to return the raw context_t ptr
+                //  to the user.
+                //
+                //  The context thus gets passed to the constructor of the 
+                //  external type before the user can do anything with it.
+                //
+                ITR_VALUE v;
+                const ITR_VALUE &dereference() { return v = ITR_VALUE(*base_t::base()); }
+                friend class boost::iterator_core_access;
+            };
+        
+            accessor_base(context_map &_map) : map(_map) {}
+            
+            iterator begin() const { return iterator(map.by_name().begin()); }
+            iterator end() const { return iterator(map.by_name().end()); }
+            
+            iterator lower_bound(const db_string &name) const { 
+                return iterator(map.by_name().lower_bound(name)); 
+            }
+            iterator upper_bound(const db_string &name) const { 
+                return iterator(map.by_name().upper_bound(name)); 
+            }
+            iterator find(const db_string &name) const { 
+                return iterator(map.by_name().find(name)); 
+            }
+            
+            ITR_VALUE operator[](const char *name) const { 
+                db_string str(name);
+                return (*this)[str];
+            }
+            ITR_VALUE operator[](const db_string &name) const { 
+                iterator i = find(name);
+                if (i == end()) {
+                    throw_error(object_not_found());
+                }
+                return *i;
+            }
+            
+        protected:
+            template<typename FN>
+            iterator create(const db_string &name, sp_marker_t sp, const FN &fn) {
+                std::pair<iterator, bool> res = create_if(name, sp, fn);
+                if (!res.second) {
+                    throw_error(object_exists());
+                }
+                return res.first;
+            }
+            
+            //  First field of the result-pair is true if the object is new.
+            template<typename FN>
+            std::pair<iterator, bool> create_if(const db_string &name, sp_marker_t sp, const FN &fn)
+            {
+                if (!name.validate_metastring()) {
+                    throw_error(invalid_argument());
+                }
+                iterator i = find(name);
+                if (i != end()) {
+                    return std::make_pair(i, false);
+                }
+
+                //  Make a safe metastring to store the object name permanently.
+                mstring_safe mstr(map.stringtable->make_safestring(name));
+                
+                i = iterator(map.by_name().insert(fn, mstr, sp));
+                return std::make_pair(i,true);
+            }
+            
+            context_t *extract_ctx(const iterator &itr) const {
+                return *(itr.source_itr());
+            }
+            
+            void drop(iterator &itr, sp_marker_t sp)
+            {
+                context_t *ctx = extract_ctx(itr);
+                map.drop(ctx, sp);
+                itr = end();
+            }
+        };
+    public:
+        typedef ACCESSOR<accessor_base> type;
+    };
+};
+
+
+//  Allocates a new context from the limit.  This does not
+//  reuse vacated entries.
+template<typename context_t, bool REUSE_SLOTS>
+class position_map;
+
+template<typename context_t>
+class position_map_base
+{
+protected:
+    db_vector<context_t *> ptrs;
+    uint32 _limit;
+    uint32 _live_count;
+    
+public:
+    typedef typename context_t::position_idx_t position_idx_t;
+
+    uint32 capacity() const { return ptrs.size(); }
+
+    position_map_base(nky_allocator_t *_alloc) : 
+    ptrs(_alloc),
+        _limit(0), 
+        _live_count(0) 
+    {}
+
+    //  
+    void assign_sequential(context_t *begin, context_t *end) {
+        if (ptrs.size() < end - begin) {
+            ptrs.resize(end - begin);
+        }
+        context_t *s = begin;
+        int i=0;
+        for (;s<end;++s,++i) {
+            ptrs[i] = s;
+        }
+        _live_count = _limit = i;
+        for (;i < ptrs.size();++i) {
+            ptrs[i] = NULL;
+        }
+    }
+    
+    void resize(uint32 newsize) {
+        OINKY_ASSERT(newsize >= _limit);
+        ptrs.resize(newsize);
+        for (int i=_limit;i<newsize;++i) {
+            ptrs[i] = NULL;
+        }
+    }
+
+    void drop(position_idx_t position, context_t *ctx) {
+        OINKY_ASSERT(ptrs.size() > (uint32) position);
+        OINKY_ASSERT(ptrs[position] == ctx);
+        ptrs[position] = NULL;
+        _live_count -= 1;
+    }
+    
+    context_t * const *begin() const { return ptrs.begin(); }
+    context_t * const *end() const { return ptrs.end(); }
+    context_t *operator[](position_idx_t x) const { return begin()[x]; }
+    
+    void cond_grow(uint32 min_size) 
+    {
+        if (min_size < capacity) { 
+            return; 
+        }
+        uint32 newsize = prow_shape_t::compute_reserve_size(min_size);
+        resize(newsize);
+    }
+    
+    uint32 limit() const { return _limit; }
+    uint32 live_count() const { return _live_count; }
+};
+
+template<typename context_t>
+class position_map<context_t, false> : public position_map_base<context_t>
+{
+protected:
+    using position_map_base<context_t>::ptrs;
+    using position_map_base<context_t>::_limit;
+    using position_map_base<context_t>::_live_count;
+public:
+    using position_map_base<context_t>::resize;
+    typedef typename context_t::position_idx_t position_idx_t;
+    
+    position_map(nky_allocator_t *_alloc) : position_map_base<context_t>(_alloc) {}
+
+    void uncreate(uint32 position, context_t *ctx)
+    {
+        OINKY_ASSERT(ptrs[position] == ctx);
+        ptrs[position] = NULL;
+        _live_count -= 1;
+        //  This is the only time we roll back the limit.  If we 
+        //  are rolling back the create, then anything
+        //  referencing it must be getting destroyed also.
+        //
+        //  This test is necessary, since uncreates may be performed
+        //  in batches, and not necessarily in perfect reverse-order.
+        if (_limit > position) {
+            _limit = position;
+        }
+    }
+    void undrop(uint32 position, context_t *ctx)
+    {
+        OINKY_ASSERT(ptrs[position] == NULL);
+        ptrs[position] = ctx;
+        _live_count += 1;
+        //  Unlike the case for uncreate, we know that the create stamp of
+        //  this context precedes anything being uncretated, thus we can
+        //  assert that the position is lower (still reserved).
+        OINKY_ASSERT(_limit > position);
+    }
+    
+    void create(uint32 position, context_t *ctx) {
+        OINKY_ASSERT(ptrs.size() > position);
+        OINKY_ASSERT(ptrs[position] == NULL);
+        OINKY_ASSERT(position == _limit);
+        _limit = position + 1;
+        
+        ptrs[position] = ctx;
+        _live_count += 1;
+    }
+    
+    uint32 next_position() {
+        if (_limit == ptrs.size()) {
+            resize(((_limit * 5) >> 2) + 3);
+        }
+        OINKY_ASSERT(!ptrs[_limit]);
+        return _limit;
+    }
+};
+
+template<typename context_t>
+class position_map<context_t, true> : public position_map_base<context_t>
+{
+protected:
+    using position_map_base<context_t>::ptrs;
+    using position_map_base<context_t>::_limit;
+    using position_map_base<context_t>::_live_count;
+    using position_map_base<context_t>::resize;
+public:
+    typedef typename context_t::position_idx_t position_idx_t;
+    
+    position_map(nky_allocator_t *_alloc) : position_map_base<context_t>(_alloc) {}
+
+    void uncreate(position_idx_t position, context_t *ctx)
+    {
+        OINKY_ASSERT(ptrs[position] == ctx);
+        ptrs[position] = NULL;
+        _live_count -= 1;
+        //  Unlike the un-reusing implementation, we do not allocate in
+        //  monotonically increasing positions, so we can't drop the 
+        //  limit, EVER.
+    }
+    void undrop(position_idx_t position, context_t *ctx)
+    {
+        OINKY_ASSERT(_limit > (uint32) position);
+        OINKY_ASSERT(ptrs[position] == NULL);
+        ptrs[position] = ctx;
+        _live_count += 1;
+    }
+
+    void create(position_idx_t position, context_t *ctx) {
+        OINKY_ASSERT(ptrs.size() > (uint32) position);
+        OINKY_ASSERT(ptrs[position] == NULL);
+        if (_limit <= (uint32) position) {
+            _limit = (uint32) position + 1;
+        }
+        ptrs[position] = ctx;
+        _live_count += 1;
+    }
+    
+    position_idx_t next_position() {
+        uint32 i;
+        if (_live_count == _limit) {
+            if (_limit == ptrs.size()) {
+                //  Conservative overallocation.
+                resize(((_limit * 5) >> 2) + 3);
+            }
+            i = _limit;
+        } else {
+            for (i=0;ptrs[i];++i) {
+                //NOTHING
+            }
+            OINKY_ASSERT(i < _limit);
+        }
+        OINKY_ASSERT(!ptrs[i]);
+        //  Check that the index is within range of the index type.
+        position_idx_t pos = (position_idx_t) i;
+        if (i != (uint32) pos) {
+            //  We should have checked this earlier before trying to assign a 
+            //  position.
+            OINKY_ASSERT(false);                
+            throw_error(limit_exceeded());
+        }
+        return pos;
+    }
+};
+
+template<typename ROW>
+struct row_value_extractor
+{
+    //  This class must be specialized.
+    BOOST_STATIC_ASSERT(sizeof(ROW) != sizeof(ROW));
+};
+
+template<typename TBL>
+struct row_value_extractor<pending_row<TBL> >
+{
+    typedef pending_row<TBL> pending_row_t;
+    typedef safe_cv_t ex_value_t;
+    
+    static inline ex_value_t get_column_value(const pending_row_t &row, const column_ctx &col, const TBL *tbl) {
+        OINKY_ASSERT(row.table() == tbl);
+        return row.get_value(col);
+    }
+};
+
+template<typename INDEX, typename ROW>
+struct index_row_compare_t
+{
+    const INDEX *idx;
+    typedef typename INDEX::idx_column_iterator idx_column_iterator;
+    typedef row_value_extractor<ROW> extractor_t;
+    typedef typename extractor_t::ex_value_t ex_value_t;
+    
+    index_row_compare_t(const INDEX *_idx) : idx(_idx) {}
+        
+    template<typename ITR_T>
+    int compare(const ROW &left, const simple_sequence<ITR_T> &right) const 
+    {
+        //  An index spec is just a list of column refs and corresponding
+        //  orderings.  The refs are indexes into the row's normal column 
+        //  set.
+        idx_column_iterator i = idx->coldefs_begin();
+        idx_column_iterator end = idx->coldefs_end();
+        ITR_T ri(right.begin());
+        for (;i != end;++i, ++ri) {
+            if (ri == right.end()) {
+                //  Premature end of seq means seq<itr
+                //
+                //  NOTE: !!!
+                //  This semantic does not change depending on the 
+                //  sort-order of the index column.
+                return 1;
+            }
+            idx_column_ctx icc(*i);
+            ex_value_t lv(extractor_t::get_column_value(left, *icc.column, idx->table));
+            int k = lv.compare_to(*ri);
+            if (k != 0) {
+                return icc.ascending ? k : -k;
+            }
+        }
+        //  If seq has not yet reached end, we do not treat it as greater.
+        //  values beyond the set we require are ignored completely.
+        return 0;
+    }
+    template<typename ITR_T>
+    int compare(const simple_sequence<ITR_T> &left, const ROW &right) const
+    {
+        return -compare(right, left);
+    }
+    int compare(const ROW &left, const ROW &right) const
+    {
+        OINKY_ASSERT(left.table() == right.table());
+        
+        //  An index spec is just a list of column refs and corresponding
+        //  orderings.  The refs are indexes into the row's normal column 
+        //  set.
+        idx_column_iterator i = idx->coldefs_begin();
+        idx_column_iterator end = idx->coldefs_end();
+        for (;i != end;++i) {
+            idx_column_ctx icc(*i);
+            ex_value_t lv(extractor_t::get_column_value(left, *icc.column, idx->table));
+            ex_value_t rv(extractor_t::get_column_value(right, *icc.column, idx->table));
+            int k = lv.compare_to(rv);
+            if (k != 0) {
+                return icc.ascending ? k : -k;
+            }
+        }
+        return 0;
+    }
+};
+
+
+} //namespace Internal
+} //namespace Oinky
+