oinky 0.1.0

data/ext/include/oinky/nky_handle.hpp

@@ -0,0 +1,334 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+namespace Oinky
+{
+using namespace Oinky::Utils;
+using namespace Oinky::Errors;
+
+namespace Internal
+{
+
+//  Internally, DB references look like this.
+template<typename ALLOCATOR_T>
+class db_handle_internal_t
+{
+public:
+    //  NOTE: This MUST be defined FIRST in the class.  That way it gets 
+    //  destroyed last.  Destructors for other members depend on the allocator.
+    //  
+    //  Each database instance has its own private allocator.  This can be
+    //  accessed without any mutex cover because accesses to the database
+    //  itself are externally serialized.
+    typedef ALLOCATOR_T allocator_t;
+    allocator_t allocator;
+    
+    typedef db_handle_internal_t<ALLOCATOR_T> db_handle_internal;
+    typedef table_ctx_t<db_handle_internal> table_ctx;
+
+    struct tablemap_traits : default_ctx_map_traits<table_ctx, uint32, allocator_t>
+    {
+        static const mstring_safe &key_from_ctx(const table_ctx *ctx) { 
+            return ctx->tablename;
+        }
+    };
+
+    //  String tables.
+    meta_stringtable_t metastrings;
+    user_stringtable_t userstrings;
+
+    //  Maps table name to table manager.  Both static and dynamic.
+    typedef context_map_t<tablemap_traits> tables_t;
+    tables_t tables;
+    
+    //  Some statistics we compute in the first phase of serialization,
+    //  used during the second phase.
+    struct computed_packing_data
+    {
+        uint32 total_bytes;
+        uint32 tableset_size;
+        
+        //  We have to remap fixed entries.  In general this will be the 
+        //  size of the largest table with any deleted entries.  If the
+        //  table has no deleted entries, we have nothing to remap.
+        //  We build the map itself at the beginning of the serialization
+        //  phase of each table, and we do it nearly for free, as we are
+        //  serializing the rows themselves.  We then use it while serializing
+        //  the indices, and then rebuild/reuse it for the next table.
+        uint32 max_fixed_table_remap_size;
+        db_vector<row_idx_t> row_remap_vector;
+
+        computed_packing_data() : 
+        //  This only grows.  Should start at zero.
+        max_fixed_table_remap_size(0),
+            //  We'll reinit with a real allocator if we ever use this.
+            row_remap_vector(NULL)
+        {}
+    } packing;
+    
+    //  Current savepoint value.  Initializes to 1.  Never gets decremented,
+    //  even on rollback.
+    sp_marker_t _active_sp;
+    //  The sp that was active at the time the last rollback was initiated
+    //  (regardless of rollback target).
+    sp_marker_t _sp_active_last_rollback;
+    
+    sp_marker_t active_sp() const { return _active_sp; }
+    sp_marker_t new_sp() { 
+        //  We increment the value, but return the previous value.  That way
+        //  rolling back to the value returned yields the current state.
+        sp_marker_t x = _active_sp;
+        _active_sp.increment(); 
+        return x;
+    }
+
+    //  Each time any table schema changes (row/index create/drop or sp_rollback
+    //  of such action, the local count (in the table) gets updated, as does
+    //  this global count.  Sometimes the incement is more than one.  We just
+    //  guarantee that it changes.
+    uint32 last_schema_change;
+
+    static void rollback_null(table_ctx *table, sp_marker_t sp) {}
+
+    //  We rollback to a target sp.  Anything marked later than this sp gets
+    //  deleted.  rolling back to 0 deletes anything marked >= 1, which is
+    //  everything but the original data.
+    //
+    //  Rolling back to a savepoint never causes allocation.
+    void sp_rollback(sp_marker_t tsp)
+    {
+        //  Nothing within this block should raise.  The catch is just
+        //  to confirm that.
+        try {
+            _sp_active_last_rollback = _active_sp;
+        
+            //  We really don't have to do much when a table gets uncreated/
+            //  undropped.  The context map takes charge of destroying it
+            //  when the time comes.
+            tables.sp_rollback(
+                tsp, 
+                &rollback_null, 
+                &rollback_null, 
+                &rollback_null);
+            
+            //  Now rollback the tables themselves.  Even if the table itself
+            //  isn't being undropped/uncreated, it might still have a lot of
+            //  work to roll back.
+            typename tables_t::by_name_iterator bnend = tables.by_name().end();
+            typename tables_t::by_name_iterator bni = tables.by_name().begin();
+            for (;bni != bnend; ++bni) {
+                (*bni)->sp_rollback(tsp);
+            }
+        } catch (...) {
+            //  An exception here is either an internal bug or some sort of
+            //  heap corruption.
+            OINKY_ASSERT(false);
+            throw;
+        }
+    }
+    
+    //  This invalidates the handle.  Invoking this prior to destruction
+    //  can make destruction faster, since we stop tracking memory allocation,
+    //  etc. knowing we're about reclaim everything en masse.
+    void dismount() 
+    {
+        allocator.teardown();
+    }
+
+    void reset()
+    {
+        //  In general this king of thing is dangerous in C++, because any
+        //  constructor may encounter an allocation failure.  
+        //  In the worst case we can trigger double-destruction,
+        //  but all of our destructors are defensive against that.
+        //  So this is safe, as long as no-one is allowed to derive from 
+        //  this type.
+        this->~db_handle_internal();
+        ::new(this) db_handle_internal();
+    }
+    
+    db_handle_internal_t() :
+    allocator(),
+        metastrings(&allocator),
+        userstrings(&allocator),
+        tables(&allocator, &metastrings),
+        _active_sp(1),
+        _sp_active_last_rollback(0),
+        last_schema_change(0)
+    {}
+    
+    ~db_handle_internal_t()
+    {
+        dismount();
+        //  Member destructors do all the actual teardown work.  We simply
+        //  needed to invoke dismount to optimize (no-op) deallocation 
+        //  beforehand.
+    }
+    
+    uint32 mark_schema_change() { return last_schema_change += 1; }
+
+    //  This reports on modifications of data, not structure.  Creating a 
+    //  savepoint does not count as a modification, deleting a row does.
+    //  The question is, does the operation affect serialization?
+    //
+    //  This result is conservative.  It may return true even when nothing
+    //  has changed.  If we make a change and then roll it back, this will
+    //  still return true.  However, this state does not persist, unless the
+    //  sequence of events repeats.  If a new SP is created, and this is 
+    //  invoked with the new SP value, then it will return false, as it should.
+    bool modified_since(sp_marker_t sp) const { 
+        return (_sp_active_last_rollback > sp) ||
+            tables.modified_since(
+                sp,
+                boost::bind(&table_ctx::table_modified_since, _1, _2));
+    }
+
+    template<typename BASE>
+    class tables_accessor_spec : public BASE
+    {
+        static void make_table(table_ctx *target, db_handle_internal *db, const mstring_safe &name) {
+            target->reinit(db);
+            target->tablename = name;
+            db->mark_schema_change();
+        }
+    
+        db_handle_internal *db;
+    public:
+        tables_accessor_spec(db_handle_internal *_db) : 
+        BASE(_db->tables), 
+            db(_db) 
+        {}
+        
+        typedef typename BASE::iterator iterator;
+        
+        iterator create(const db_string &name) { 
+            return BASE::create(name, db->active_sp(), boost::bind(make_table, _1, db, _2)); 
+        }
+        
+        //  First field of the result-pair is true if the object is new.
+        std::pair<iterator, bool> create_if(const db_string &name) {
+            return BASE::create_if(name, db->active_sp(), boost::bind(make_table, _1, db, _2)); 
+        }
+        
+        void drop(iterator &itr) {
+            BASE::drop(itr, db->active_sp());
+        }
+    };
+
+    //  This is effectively the user interface to the DB.  You can get a
+    //  handle to a table/column/index, from which you can alter/insert/drop/
+    //  iterate, etc.
+    typedef typename table_ctx::table_handle table_handle;
+    typedef typename table_ctx::column_selector_t column_selector_t;
+    typedef typename table_ctx::row_iterator row_iterator;
+    typedef typename tables_t::template set_accessor_t<table_handle, tables_accessor_spec>::type tables_accessor_t;
+    tables_accessor_t tables_accessor() { return tables_accessor_t(this); }
+};
+
+//  The user gets one of these.
+template<typename ALLOCATOR_T>
+class db_instance_t
+{
+    typedef db_handle_internal_t<ALLOCATOR_T> db_handle_internal;
+    typedef db_instance_t<ALLOCATOR_T> db_instance;
+    
+    //  This has to be a member instead of a base because of the 
+    //  reset command.  Reset recreates a handle object, regardless of what
+    //  a user may have derived.
+    //
+    //  This way, a user can safely derive from db_instance_t, not that
+    //  there's any reason to.
+    db_handle_internal_t<ALLOCATOR_T> db;
+    
+public:
+    typedef typename db_handle_internal::column_selector_t column_selector_t;
+
+    typedef typename db_handle_internal::tables_accessor_t tables_accessor_t;
+    typedef typename db_handle_internal::table_handle table_handle;
+    typedef typename db_handle_internal::row_iterator row_iterator;
+    typedef typename tables_accessor_t::iterator table_itr;
+
+    typedef typename table_handle::columns_accessor_t columns_accessor_t;
+    typedef typename table_handle::column_handle column_handle;
+    typedef typename table_handle::cursor_handle table_cursor_handle;
+    typedef typename columns_accessor_t::iterator column_itr;
+    
+    typedef typename table_handle::indices_accessor_t indices_accessor_t;
+    typedef typename table_handle::index_handle index_handle;
+    typedef typename indices_accessor_t::iterator index_itr;
+    
+    typedef typename index_handle::iterator index_row_iterator;
+    typedef typename index_handle::cursor_handle index_cursor_handle;
+    typedef typename index_handle::column_defs_accessor index_columns_accessor_t;
+    typedef typename index_columns_accessor_t::column_itr index_column_itr;
+    
+    typedef Oinky::index_column_def index_column_def;
+    
+//    typedef typename index_handle::column_def index_column_def;
+//    typedef typename index_handle::column_iterator index_column_iterator;
+    
+    tables_accessor_t tables() { return db.tables_accessor(); }
+
+    //  This is on a private timeline.  The caller is only guranteed 
+    //  that it increases monotonically.
+    uint32 last_schema_change() const { return db.last_schema_change(); }
+
+    bool modified_since(sp_marker_t sp) const { 
+        return db.modified_since(sp); 
+    }
+    
+    //  Takes the const bytestring and mounts it as a database.  Returns a 
+    //  handle to the DB on successful mount.  Exception if mount fails.
+    //
+    //  The mount process does not exhaustively verify encoding.  Subsequent
+    //  accesses to the handle may throw an exception if they decode invalid
+    //  data.
+    //
+    //  The same flattened bytes can be mounted several times, which will 
+    //  produce unique handles.  Modifications to any handle will be invisible
+    //  to the others.
+    static void mount(db_instance *h, const char *buffer, uint32 buflen) {
+        Serialization::Serializer<
+            db_handle_internal,
+            Serialization::v1_sformat_tag
+            >::mount(&h->db, buffer, buflen);
+    }
+    
+    uint32 prepare_pack() {
+        return Serialization::Serializer<
+            db_handle_internal,
+            Serialization::v1_sformat_tag
+            >::prepare_pack(db);
+    }
+
+    uint32 complete_pack(char * target_buffer, uint32 buflen) {
+        char * end = target_buffer;
+        
+        Serialization::Serializer<
+            db_handle_internal,
+            Serialization::v1_sformat_tag
+            >::complete_pack(db, end, target_buffer + buflen);
+            
+        //  How many bytes were written
+        return end - target_buffer;
+    }
+    
+    sp_marker_t new_sp() { return db.new_sp(); }
+    void sp_rollback(sp_marker_t tsp) { db.sp_rollback(tsp); }
+    
+    //  The DB instance itself is noncopyable.  For that reason, 
+    //  we implement a custom reset() method.  This is equivalent to doing
+    //  db_t db;
+    //  ...<modify DB>
+    //  db = db_t();  //  <<---- This is the reset operation we simulate.
+    void reset() { db.reset(); }
+};
+
+} //namespace Internal
+
+typedef ::Oinky::Internal::db_instance_t<nky_allocator_t> db_t;
+
+} //namespace Oinky
+

data/ext/include/oinky/nky_index.hpp

@@ -0,0 +1,1038 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+namespace Oinky
+{
+namespace Internal
+{
+using namespace Oinky::Errors;
+using namespace Oinky::Utils;
+
+struct index_def_t
+{
+    //  Column width of index.
+    column_idx_t col_count;
+    
+    //  unique?
+    bool require_unique;
+    
+    const uint8 *cols_ascending;
+};
+
+template<typename TABLE_CTX>
+class idx_column_iterator_t : public
+    boost::iterator_facade<
+        idx_column_iterator_t<TABLE_CTX>,
+        idx_column_ctx,
+        std::random_access_iterator_tag,
+        const idx_column_ctx &
+        >
+{
+    typedef idx_column_iterator_t<TABLE_CTX> itr_t;
+    
+    const TABLE_CTX *table;
+    const index_def_t *defn;
+    const column_ctx *const* column_refs;
+    column_idx_t position;
+    
+    void set_value() {
+        OINKY_ASSERT(position <= defn->col_count);
+        if (position < defn->col_count) {
+            val.column = column_refs[position];
+            val.ascending = (defn->cols_ascending[position >> 3] & (1 << position & 7)) != 0;
+            val.index_pos = position;
+        }
+    }
+public:
+    typedef idx_column_ctx value_type;
+    
+    idx_column_iterator_t() : table(NULL), defn(NULL), position(0) {}
+    
+    idx_column_iterator_t(
+        const TABLE_CTX *_table,
+        const index_def_t *_defn,
+        const column_ctx *const *_column_refs,
+        column_idx_t _position) :
+    table(_table),
+        defn(_defn),
+        column_refs(_column_refs),
+        position(_position)
+    {
+        set_value();
+    }
+
+    index_column_def make_def() const {
+        index_column_def d;
+        d.column_name = dereference().column->colname;
+        d.sort_ascending = dereference().ascending;
+        return d;
+    }
+
+private:        
+    const value_type &dereference() const { 
+        OINKY_ASSERT(position < defn->col_count);
+        return val;
+    }
+    bool equal(const itr_t &other) const { 
+        OINKY_ASSERT(table == other.table);
+        return position == other.position; 
+    }
+    void increment() { ++position; set_value(); }
+    void decrement() { --position; set_value(); }
+    void advance(int x) { position += x; set_value(); }
+    int distance_to(const itr_t &other) const { 
+        OINKY_ASSERT(table == other.table);
+        return (int) position - other.position;
+    }
+
+    idx_column_ctx val;
+
+    friend class boost::iterator_core_access;
+};
+
+//  This manages a mixed index.  It provides a differential, mutable view 
+//  against a fixed serialized version of the index.
+template<typename TABLE_CTX>
+class index_ctx_t
+{   
+public:
+    typedef TABLE_CTX table_ctx;
+    typedef index_ctx_t<table_ctx> index_ctx;
+    typedef idx_column_iterator_t<table_ctx> idx_column_iterator;
+    typedef pending_row<table_ctx> pending_row_t;
+    typedef index_idx_t position_idx_t;
+    
+    mstring_safe indexname;
+
+    //  sp marker of the creation/deletion of this index.
+    ls_marker ls;
+
+    //  Parent table.
+    table_ctx *table;
+    
+    //  Which of the table's indexes is this?
+    index_idx_t index_idx;
+    
+    //  Head of the index tree.
+    row_index_hook head;
+
+    //  Definition of the index.
+    index_def_t defn;
+
+    //  We never access this.  It merely exists to free the memory 
+    //  when the index is destroyed. 
+    db_vector<uint8> __definition_data;
+
+    //  This specifies which columns are indexed, in which order.
+    const column_ctx *const *column_refs;
+
+    //  The serialized portion of the index.
+    typedef indexdata_fixed<table_ctx> fixed_t;
+    typedef typename fixed_t::iterator fixed_itr_t;
+    fixed_t fixed;
+
+    typedef column_selector_template<table_ctx> column_selector_t;
+
+    index_ctx_t(table_ctx *_table = NULL, index_idx_t _index_idx = (index_idx_t) 0) : 
+    table(_table),
+        index_idx(_index_idx),
+        __definition_data(_table ? &_table->db->allocator : NULL),
+        column_refs(NULL),
+        fixed(
+            tabular_t(),
+            _table,
+            this)
+    {
+        treealg::init_header(&head);
+    }
+    
+    ~index_ctx_t()
+    {
+        //  Clear the tree head.  Otherwise destructor will traverse and unlink 
+        //  everything, which is a waste of time.
+        treealg::init_header(&head);
+    }
+
+    void reinit_serial(table_ctx *_table, index_idx_t _index_idx)
+    {
+        this->~index_ctx();
+        ::new(this) index_ctx(_table, _index_idx);
+    }
+    
+    //  Allocate a vector and copy the values into it.  Index owns responsibility
+    //  for releasing.
+    template<typename ITR_T>
+    void reinit_dynamic(table_ctx *_table, index_idx_t _index_idx, index_uniqueness_spec_t uniq, ITR_T col_begin, ITR_T col_end)
+    {
+        BOOST_STATIC_ASSERT((boost::is_same<
+            typename boost::iterator_value<ITR_T>::type,
+            index_column_def>::value));
+            
+        uint32 colcount = std::distance(col_begin, col_end);
+    
+        reinit_serial(_table, _index_idx);
+
+        //  Allocate memory to hold our column arrays.  We use a bit per column 
+        //  for the cols_ascending value.
+        uint32 ascending_bytes = (colcount + 7) >> 3;
+        __definition_data.resize((sizeof(column_ctx *) * colcount) + ascending_bytes);
+        column_ctx **wcols = (column_ctx **) __definition_data.begin();
+        column_refs = wcols;
+        uint8 *cols_ascending = (uint8 *) (column_refs + colcount);
+        memset(cols_ascending, 0, ascending_bytes);
+
+        //  Fill definiton data from source iterators.
+        uint32 pos = 0;
+        for (ITR_T i = col_begin; i != col_end; ++i, ++pos) {
+            typename table_ctx::cols_by_name_itr_t col = table->columns.by_name().find(i->column_name);
+            if (col == table->columns.by_name().end()) {
+                //  The specified column does not exist.
+                throw_error(object_not_found());
+            }
+            if ((*col)->ctype == column_types::Variant) {
+                //  We do not index variant column types, because there is no natural
+                //  ordering among values of different types.  (We could obviously
+                //  declare one, but the declaration would be arbitrary, and arbitrary
+                //  semantics are better excluded.)
+                throw_error(column_not_indexable());
+            }
+            if (i->sort_ascending) {
+                cols_ascending[pos >> 3] |= 1 << (pos & 7);
+            }
+            wcols[pos] = *col;
+        }
+
+        defn.cols_ascending = cols_ascending;
+        defn.col_count = (column_idx_t) colcount;
+        defn.require_unique = (uniq == Oinky::Unique);
+        
+        //  The above setting of defn is enough to make coldefs_begin valid.
+        fixed = fixed_t(
+            tabular_t(),
+            _table,
+            this);
+    }
+
+    //  Compares two index nodes.
+    typedef index_row_compare_t<index_ctx, pending_row_t> index_row_compare;
+    struct hook_compare
+    {
+        index_row_compare irc;
+        
+        hook_compare(const index_ctx *_idx) : irc(_idx) {}
+        
+        const index_ctx *index() const { return irc.idx; }
+
+        template<typename ITR_T>
+        int compare(const row_index_hook *left_n, const simple_sequence<ITR_T> &right) const 
+        {
+            const pending_row_t *left = pending_row_t::from_index_hook(left_n,index());
+            return irc.compare(*left,right);
+        }
+        template<typename ITR_T>
+        int compare(const simple_sequence<ITR_T> &left, const row_index_hook *right_n) const
+        {
+            return -compare(right_n, left);
+        }
+        int compare(const row_index_hook *left_n, const row_index_hook *right_n) const
+        {
+            const pending_row_t *left = pending_row_t::from_index_hook(left_n,index());
+            const pending_row_t *right = pending_row_t::from_index_hook(right_n,index());
+            return irc.compare(*left,*right);
+        }
+    };
+    struct index_less
+    {
+        hook_compare c;
+        bool suborder_duplicates;
+        bool is_cursor_seek;
+        
+        index_less(const index_ctx *_idx, bool _suborder_duplicates, bool _is_cursor_seek = false) : 
+        c(_idx),
+            suborder_duplicates(_suborder_duplicates),
+            is_cursor_seek(_is_cursor_seek)
+        {}
+        
+        template<typename ITR_T>
+        bool operator()(const simple_sequence<ITR_T> &left, const row_index_hook *right_n) const 
+        {
+            //  Searching for sequences, we should never be subordering duplicates.  We only
+            //  do that on insert.
+            OINKY_ASSERT(!suborder_duplicates);
+            return c.compare(left,right_n) < 0;
+        }
+        template<typename ITR_T>
+        bool operator()(const row_index_hook *left_n, const simple_sequence<ITR_T> &right) const
+        {
+            OINKY_ASSERT(!suborder_duplicates);
+            return c.compare(left_n,right) < 0;
+        }
+
+        bool operator()(const row_index_hook *left_n, const row_index_hook *right_n) const 
+        {
+            if (left_n == right_n) {
+                return false;
+            }
+            const pending_row_t *left = pending_row_t::from_index_hook(left_n,c.index());
+            const pending_row_t *right = pending_row_t::from_index_hook(right_n,c.index());
+            int i = c.irc.compare(*left,*right);
+            if (i || !suborder_duplicates) return i < 0;
+            //  Secondary ordering is the sequence number.
+            OINKY_ASSERT((left->sequence_number != right->sequence_number) || is_cursor_seek);
+            return left->sequence_number < right->sequence_number;
+        }
+    };
+    
+    template<typename PITR>
+    struct merge_itr_compare
+    {
+        //  TBD: These symmetric comparisons should only be necessary for the
+        //  pending_iterator, not the cursor.  These are only used when
+        //  comparing two merge-iterators to each other.
+
+        //  Comparing identical iterator types is much faster than comparing values.
+        int operator()(const fixed_itr_t &left, const fixed_itr_t &right) const {
+            //  These are just row indices
+            return (int) *left - (int) *right;
+        }
+        int operator()(const PITR &left, const PITR &right) const {
+            //  Compare the two nodes.
+            const index_ctx *idx = left.index_ptr();
+            OINKY_ASSERT(idx == right.index_ptr());
+            const pending_row_t *lr = left.to_row();
+            const pending_row_t *rr = right.to_row();
+            int i = index_row_compare(idx).compare(*lr,*rr);
+            if (i || (lr == rr)) return i;
+            
+            //  Similar to below, where we must treat fixed and pending objects
+            //  differently, because this index may not be unique, we must 
+            //  consider that there may be multiple pending rows with the same
+            //  indexed values, and we need to treat them differently.  We differentiate
+            //  them by the node's sequence number.
+
+            //  See note below about this not being possible unless the index
+            //  is non-unique, or this is an expired iterator (or an internal bug).
+            OINKY_ASSERT(!idx->defn.require_unique);
+                
+            //  We should have generated unique values.
+            if (lr->sequence_number < rr->sequence_number) return -1;
+            if (lr->sequence_number > rr->sequence_number) return 1;
+
+            //  This should be impossible for pending_iterator.  Only the 
+            //  safe_cursor could be pointing to an old row object, which 
+            //  shares the sequence number and index position with its new 
+            //  version.
+            OINKY_ASSERT(!(boost::is_same<PITR, pending_iterator>::value));
+            return 0;
+        }
+        
+        //  Comparing different iterator types requires actual comparison.
+        int operator()(const PITR &left, const fixed_itr_t &right) const {
+            return -(*this)(right, left);
+        }
+        int operator()(const fixed_itr_t &left, const PITR &right) const {
+            //  The accessor comparison method doesn't work because
+            //  it is a value-based comparison.  It lacks the context
+            //  of knowing whether the columns are indexed in ascending
+            //  or decending order.
+            index_ctx *idx = right.index_ptr();
+            uint32 col_count = idx->defn.col_count;
+            variant_cv_t *lvals = (variant_cv_t *) alloca(sizeof(variant_cv_t) * col_count);
+            left.indexed_values().copy_to(lvals, col_count);
+            simple_sequence<const variant_cv_t *> lseq(lvals, lvals + col_count);
+            
+            int i = -index_row_compare(idx).compare(*right.to_row(), lseq);
+            //  Here we are enumerating two sets which we know to be disjoint in
+            //  terms of rowkeys.  It's possible that the index is non-unique,
+            //  in which case we might still have value overlap between the
+            //  two sets.  We need to be sure not to drop either value,
+            //  so in the event of a tie, the fixed iterator always wins.
+            //
+            //  Note that we do NOT do this in the comparison function that we
+            //  use for insertion, because that does need to discover duplicate
+            //  values in some cases.
+            //
+            //  Note also, however, that this IS the comparison function use to
+            //  compare two iterators.  If they refer to different storage 
+            //  (pending vs. fixed) we know they're not equivalent.
+            if (i == 0) {
+                //  Should not have been permitted to insert a pending node
+                //  that shadowed a fixed node.  Possibly this is a user
+                //  iterator that has since been invalidated by a savepoint
+                //  rollback.
+                OINKY_ASSERT(left.is_deleted() || !left.index()->defn.require_unique);
+                
+                //  Secondary sort by sequence number/rowkey.
+                if ((uint64) *left < right->sequence_number) {
+                    return -1;
+                }
+                if ((uint64) *left > right->sequence_number) {
+                    return 1;
+                }
+                //  Otherwise this is literally the same row.  
+                //  One MUST be deleted.
+                OINKY_ASSERT(left.is_deleted());
+                //  These must still be sequenced, so that we can skip over the
+                //  deleted item to reach the most current value (in the pending).
+                return -1;
+            }
+            return i;
+        }
+    };
+    
+    //  Nothing to do.  Indexes are merely projections of table data, and for
+    //  now their definitions are immutable.
+    void sp_rollback(sp_marker_t target_sp) {}
+    
+    void link_all()
+    {
+        typedef typename table_ctx::pitr_t pitr_t;
+
+        OINKY_ASSERT(treealg::size(&head) == 0);
+        pitr_t end = table->live_pending.end();
+        //  This amounts to a full sort.
+        for (
+            pitr_t i = table->live_pending.begin();
+            i != end;
+            ++i)
+        {
+            pending_row_t &row(*i);
+            OINKY_ASSERT( row.ls.delete_sp == 0 );
+            link_pending(&row, true);
+        }
+    }
+    
+    void unlink_all()
+    {
+        //  This is easy.  The member nodes will just be treated as 
+        //  uninitialized hooks.  No point unlinking them.
+        treealg::init_header(&head);
+    }
+    
+    void drop(sp_marker_t sp) 
+    {
+        unlink_all();
+    }
+
+    //  Uncreate the index itself.
+    void uncreate(sp_marker_t tsp)
+    {
+        //  Remove all remaining pending rows from this index.
+        unlink_all();
+    }
+    void undrop(sp_marker_t tsp)
+    {
+        //  Relink all rows.
+        link_all();
+    }
+
+    struct values_accessor_base
+    {
+        column_selector_t cs;
+        const pending_row_t *prow;
+        
+        values_accessor_base(
+            column_selector_t _cs,
+            const pending_row_t *_prow) :
+        cs(_cs),
+            prow(_prow)
+        {}
+        
+        uint32 column_count() const { return cs.column_count(); }
+
+        //  Parameters are a column definition and a value
+        template<typename FN>
+        void each_column_value(FN fn) const {
+            prow->each_column_value(cs, fn);
+        }
+    };
+    
+    typedef multicolumn_value_accessor<values_accessor_base> values_accessor_t;
+    
+    values_accessor_t indexed_values(pending_row_t *row) {
+        return values_accessor_t(index_columns(), row);
+    }
+
+    column_selector_t index_columns() { 
+        return column_selector_t::select_index(this);
+    }
+
+    //  Extract the indexed columns from the pending row and insert.
+    void link_pending(pending_row_t *pr, bool skip_fixed_check) 
+    {
+        row_index_hook *hook = pr->get_hook(index_idx);
+        
+        //  If we care about uniqueness, then check the fixed table first
+        if (!skip_fixed_check && defn.require_unique && !table->fixed_dropped()) {
+            //  Copy the relevant index values into sequence before searching.
+            variant_cv_t *values = (variant_cv_t *) alloca(sizeof(variant_cv_t) * defn.col_count);
+            indexed_values(pr).copy_to(values, defn.col_count);
+
+            fixed_itr_t fi = fixed.find(simple_sequence<const variant_cv_t *>(values, values + defn.col_count));
+            if ((fi != fixed.end()) && !fi.is_deleted()) {
+                throw_error(index_entry_not_unique());
+            }
+        }
+
+        //  Check whether the insertion would violate uniqueness, and if we care.
+        //  There's no additional overhead to checking, even if we're not 
+        //  enforcing uniqueness, since we have to insert anyway.
+        treealg::insert_commit_data commit_data;
+        std::pair<row_index_hook *, bool> result_pair = treealg::insert_unique_check(
+            &head, 
+            hook, 
+            //  second parameter is to add a secondary ordering (pointer) to 
+            //  duplicate entries.  This is because all orderings must be 
+            //  strict, even if they are identical, which can happen for 
+            //  non-unique indices.
+            index_less(this, !defn.require_unique),
+            commit_data);
+        if (!result_pair.second)    //  Entry exists in index.
+        {
+            //  If this is showing up as existing, then it must be using the unique check.
+            OINKY_ASSERT(defn.require_unique);
+            throw_error(index_entry_not_unique());
+        }
+
+        //  Otherwise do the insert.
+        treealg::insert_unique_commit(&head, hook, commit_data);
+    }
+    
+    //  Just extract from the tree.
+    void unlink_pending(pending_row_t *pr) 
+    {
+        row_index_hook *hook = pr->get_hook(index_idx);
+        treealg::erase(&head, hook);
+    }
+private:
+    struct pending_iterator_base
+    {
+        index_ctx *idx;
+        row_index_hook *hook;
+        
+        pending_iterator_base() : idx(NULL), hook(NULL) {}
+        
+        pending_iterator_base(index_ctx *_index, row_index_hook *_hook) : 
+        idx(_index),
+            hook(_hook)
+        {}
+        
+        values_accessor_t indexed_values() const {
+            return idx->indexed_values(to_row());
+        }
+        
+        pending_row_t *to_row() const {
+            if (hook == treealg::end_node(&idx->head)) {
+                return NULL;
+            }
+            return pending_row_t::from_index_hook(hook, idx);
+        }        
+        
+        index_ctx *index_ptr() const { return idx; }
+
+        int compare_to(const pending_iterator_base &other) const {
+            pending_row_t *l = to_row();
+            pending_row_t *r = other.to_row();
+            if (l == r) return 0;
+            if (l == NULL) return 1;
+            if (r == NULL) return -1;
+            return index_row_compare(idx).compare(*l,*r);
+        }
+
+        template<typename ITR_T>
+        int compare_to(const simple_sequence<ITR_T> &seq) const {
+            pending_row_t *l = to_row();
+            if (l == NULL) return 1;
+            return index_row_compare(idx).compare(*l,seq);
+        }
+    };
+    
+    //  Compared to the table iterator...
+    //  The index iterator must merge values...unlike the table iterator.
+    //  The index iterator must also check every fixed value against the 
+    //  delete set, since the delete set and index are not in the same order.
+    //
+    //  This iterator is bidirectional.
+    //  iterator over the pending items;
+    class pending_iterator : 
+        public pending_iterator_base, 
+        public boost::iterator_facade<
+            pending_iterator, 
+            const pending_row_t &,
+            boost::bidirectional_traversal_tag,
+            const pending_row_t &>
+    { 
+        friend class boost::iterator_core_access;
+        
+        void increment() {
+            hook = treealg::next_node(hook);
+        }
+        void decrement() {
+            hook = treealg::prev_node(hook);
+        }
+        const pending_row_t &dereference() const { return *to_row(); }
+        bool equal(const pending_iterator &other) const { 
+            return hook == other.hook;
+        }
+        
+    public:
+        using pending_iterator_base::idx;
+        using pending_iterator_base::hook;
+        using pending_iterator_base::to_row;
+        
+        pending_iterator() : pending_iterator_base() {}
+        
+        pending_iterator(index_ctx *_idx, row_index_hook *_hook) : 
+        pending_iterator_base(_idx,_hook)
+        {}
+        
+        pending_iterator(index_ctx *_idx, pending_row_t *_row) :
+        pending_iterator_base(_idx,_row->get_hook(_idx->index_idx))
+        {}
+
+        bool at_end() const { return hook == treealg::end_node(&idx->head); }
+        
+        static pending_iterator begin_from(index_ctx *idx) {
+            return pending_iterator(idx, treealg::begin_node(&idx->head));
+        }
+        static pending_iterator end_from(index_ctx *idx) {
+            return pending_iterator(idx, treealg::end_node(&idx->head));
+        }
+        template<typename SEQ_T>
+        static pending_iterator lower_bound_from(index_ctx *idx, const SEQ_T &key) {
+            //  The comparer we use will recognize equivalent entries as matches.
+            return pending_iterator(idx, treealg::lower_bound(&idx->head, key, index_less(idx, false)));
+        }
+        template<typename SEQ_T>
+        static pending_iterator upper_bound_from(index_ctx *idx, const SEQ_T &key) {
+            return pending_iterator(idx, treealg::upper_bound(&idx->head, key, index_less(idx, false)));
+        }
+        template<typename SEQ_T>
+        static pending_iterator find_from(index_ctx *idx, const SEQ_T &key) {
+            return pending_iterator(idx, treealg::find(&idx->head, key, index_less(idx, false)));
+        }
+    };
+    
+    pending_iterator pending_begin() { return pending_iterator::begin_from(this); }
+    pending_iterator pending_end() { return pending_iterator::end_from(this); }
+    
+public:    
+    idx_column_iterator coldefs_begin() const { 
+        return idx_column_iterator(table, &defn, column_refs, (column_idx_t) 0); 
+    }
+    idx_column_iterator coldefs_end() const { 
+        return idx_column_iterator(table, &defn, column_refs, defn.col_count); 
+    }
+
+    typedef symmetric_iterator<fixed_itr_t> fixed_range_t;
+    typedef symmetric_iterator<pending_iterator> pending_range_t;
+
+    struct pending_to_row {
+        template<typename PITR>
+        pending_row_t *operator()(const PITR &pitr) const {
+            return pitr.to_row();
+        }
+    };
+    
+    typedef typename iterator_builder<
+        table_ctx, 
+        fixed_itr_t, 
+        fixed_range_t, 
+        pending_iterator, 
+        pending_range_t,
+        merge_itr_compare<pending_iterator>,
+        pending_to_row>::iterator iterator;
+    
+    template<typename SEQ_T>
+    inline void check_key_type(const SEQ_T &key) const {
+        //  We support untyped value comparison internally, because we trust
+        //  the internals.  However, we require that the user-key be safely
+        //  typed.  The effects of passing a mistyped column value 
+        //  would be difficult to diagnose, and could potentially segfault.
+        BOOST_STATIC_ASSERT((boost::is_same<
+            typename boost::iterator_value<typename SEQ_T::itr_t>::type,
+            variant_cv_t>::value));
+    }
+    
+    //  Indices can be iterated forward and backward.
+    //  The position given describes the values of the indexed columns.  It
+    //  need not describe all the indexed columns.  A partial value specifies an 
+    //  exclusive lower bound on the resulting iterator.
+    template<typename SEQ_T>
+    iterator lower_bound(const SEQ_T &key) { 
+        check_key_type(key);
+        
+        fixed_itr_t fi = fixed.lower_bound(key);
+        pending_iterator pi = pending_iterator::lower_bound_from(this,key);
+        return iterator(
+            table,
+            fixed_range_t(fi, fixed.begin(), fixed.end()),
+            pending_range_t(pi, pending_begin(), pending_end()) 
+            );
+    }
+    template<typename SEQ_T>
+    iterator upper_bound(const SEQ_T &key) { 
+        check_key_type(key);
+        
+        fixed_itr_t fi = fixed.upper_bound(key);
+        pending_iterator pi = pending_iterator::upper_bound_from(this,key);
+        return iterator(
+            table,
+            fixed_range_t(fi, fixed.begin(), fixed.end()),
+            pending_range_t(pi, pending_begin(), pending_end()) 
+            );
+    }
+    template<typename SEQ_T>
+    iterator find(const SEQ_T &key) { 
+        check_key_type(key);
+        
+        fixed_itr_t fi = fixed.find(key);
+        pending_iterator pi = pending_iterator::find_from(this,key);
+        return iterator(
+            table,
+            fixed_range_t(fi, fixed.begin(), fixed.end()),
+            pending_range_t(pi, pending_begin(), pending_end()) 
+            );
+    }
+    iterator begin() { 
+        fixed_itr_t fi(fixed.begin());
+        pending_iterator pi(pending_begin());
+        return iterator(
+            table,
+            fixed_range_t(fi, fi, fixed.end()), 
+            pending_range_t(pi, pi, pending_end())
+            ); 
+    }
+    iterator end() { 
+        fixed_itr_t fi(fixed.end());
+        pending_iterator pi(pending_end());
+        return iterator(
+            table,
+            fixed_range_t(fi, fixed.begin(), fi), 
+            pending_range_t(pi, pending_begin(), pi)
+            ); 
+    }
+
+    struct safe_idx_cursor_host
+    {
+        index_ctx *idx;
+        safe_idx_cursor_host(index_ctx *_idx) : idx(_idx) {}
+        safe_idx_cursor_host() : idx(NULL) {}
+
+        bool operator==(const safe_idx_cursor_host &other) const {
+            return idx == other.idx;
+        }
+        bool operator!=(const safe_idx_cursor_host &other) const {
+            return idx != other.idx;
+        }
+        
+        typedef typename table_ctx::allocator_t allocator_t;
+        allocator_t &allocator() const { return idx->table->db->allocator; }
+        
+        table_ctx *table() const { return idx->table; }
+    
+        void check_state() const
+        {
+            if (idx->ls.is_deleted()) {
+                throw_error(object_deleted());
+            }
+        }
+
+        //  Alternatives are lower/upper bound.  Depends on fwd/reverse
+        pending_row_t *seek_pending(bool use_lower_bound, pending_row_t *row) const
+        {
+            OINKY_ASSERT(row->is_inactive());
+            row_index_hook *hook = row->get_hook(idx->index_idx);
+            
+            //  Our existing row isn't indexed, but we can use the familiar 
+            //  comparer to compute its insert position, which is where 
+            //  we should continue enumeration from.  This works whether
+            //  a matching entry is found or not.
+            //
+            //  We do this rather than just making a sequence out of the
+            //  positional values (using alloca/select/copy_to) because 
+            //  we also want to seek to a particular sequence number.
+            //
+            //  Suborder non-unique items..otehrwise we would always seek
+            //  to the first entry matching the indexed values.  There's
+            //  no harm doing this even when the index is unique.  We
+            //  specify too that we are in cursor_seek, which disables the
+            //  assert we would get from finding an existing node
+            //  matching another node in every way, including the sequence
+            //  number.  (It's not a problem because the node we're
+            //  searching with is not in the tree.
+            index_less comparer(idx, true, true);
+            
+            row_index_hook *h;
+            if (use_lower_bound) {
+                h = treealg::lower_bound(
+                    &idx->head, 
+                    hook,
+                    comparer);
+            } else {
+                h = treealg::upper_bound(
+                    &idx->head, 
+                    hook,
+                    comparer);
+            }
+            if (h == treealg::end_node(&idx->head)) {
+                return NULL;
+            }
+            return pending_row_t::from_index_hook(h, idx);
+        }
+        
+        pending_row_t *seek_prev(pending_row_t *row) const {
+            row_index_hook *h;
+            if (row == NULL) {
+                h = treealg::end_node(&idx->head);
+            } else {
+                h = row->get_hook(idx->index_idx);
+            }
+            if (h == treealg::begin_node(&idx->head)) {
+                return NULL;
+            } else {
+                return pending_row_t::from_index_hook(treealg::prev_node(h), idx);
+            }
+        }
+        pending_row_t *seek_next(pending_row_t *row) const {
+            row_index_hook *h;
+            if (row == NULL) {
+                h = treealg::begin_node(&idx->head);
+            } else {
+                h = row->get_hook(idx->index_idx);
+                h = treealg::next_node(h);
+            }
+            
+            if (h == treealg::end_node(&idx->head)) {
+                return NULL;
+            } else {
+                return pending_row_t::from_index_hook(h, idx);
+            }
+        }
+
+        //  Link/Unlink the internal cursor object from the host list.
+        template<typename INTERNAL>
+        void unlink_cursor(INTERNAL *i) {
+            idx->active_cursors.erase(index_ctx::active_cursors_t::s_iterator_to(*i));
+        }
+        template<typename INTERNAL>
+        void link_cursor(INTERNAL *i) {
+            idx->active_cursors.push_back(*i);
+        }
+
+        inline void assert_row_valid(pending_row_t *row) const {
+            OINKY_ASSERT(row && (row->table() == idx->table));
+        }
+    };
+    
+    class safe_pending_cursor : public safe_pending_cursor_t<pending_row_t, safe_idx_cursor_host>
+    {
+        typedef safe_pending_cursor_t<pending_row_t, safe_idx_cursor_host> base_t;
+    public:
+        using base_t::host;
+        using base_t::to_row;
+        
+        safe_pending_cursor() {}
+        
+        safe_pending_cursor(const safe_idx_cursor_host &_host, pending_row_t *_row, bool _bbegin) :
+        base_t(_host, _row, _bbegin)
+        {}
+        
+        const safe_pending_cursor &i() const { return *this; }
+        
+        values_accessor_t indexed_values() const {
+            return values_accessor_t(host.idx->index_columns(), to_row());
+        }
+        index_ctx *index_ptr() const { return host.idx; }
+    };
+
+    typedef typename iterator_builder<
+        table_ctx, 
+        fixed_itr_t, 
+        fixed_range_t, 
+        safe_pending_cursor,
+        safe_pending_cursor,
+        merge_itr_compare<safe_pending_cursor>,
+        pending_to_row>::iterator safe_itr_t;
+
+    typedef cursor_internal_t<safe_itr_t> cursor_internal;
+    typedef cursor_handle_t<cursor_internal, safe_idx_cursor_host> cursor_handle;
+
+    //
+    //  We keep track of all the active cursors on this table.  (excluding the 
+    //  indices).  We invalidate them when the index gets dropped or uncreated.
+    //
+    typedef boost::intrusive::list<cursor_internal> active_cursors_t;
+    active_cursors_t active_cursors;
+    
+    void invalidate_cursors()
+    {
+        while (active_cursors.begin() != active_cursors.end()) {
+            typename active_cursors_t::iterator i = active_cursors.begin();
+            i->invalidate();
+            active_cursors.erase(i);
+        }
+    }
+    
+    //  Invoked when the table is dropped from the DB, or when a create is rolled back via SP.
+    void on_deactivation() { 
+        invalidate_cursors();
+    }
+};
+
+template<typename INDEX_CTX>
+//  This is the public/user interface to the index_ctx.
+class index_handle_t
+{
+    typedef INDEX_CTX index_ctx;
+    typedef typename index_ctx::table_ctx table_ctx;
+    typedef typename index_ctx::safe_itr_t safe_itr_t;
+    typedef typename table_ctx::pending_row_t pending_row_t;
+    typedef typename index_ctx::safe_idx_cursor_host safe_idx_cursor_host;
+    typedef typename index_ctx::safe_pending_cursor safe_pending_cursor;
+    typedef typename index_ctx::idx_column_iterator idx_column_iterator;
+
+    index_ctx *index;
+public:
+    typedef typename index_ctx::cursor_handle cursor_handle;
+    typedef typename index_ctx::iterator iterator;
+    typedef typename table_ctx::column_selector_t column_selector_t;
+    
+    index_handle_t() : index(NULL) {}
+    index_handle_t(index_ctx *_index) : index(_index) {}
+    
+    //  This tells if the handle value is uninitialized.  If this returns false,
+    //  it does NOT mean the handle is valid.  It may still be an expired handle
+    //  or just invalid.  But it is a test as to whether it's initialized at all.
+    bool is_null() const { return index == NULL; }
+
+    //###############
+    //  For the C API.  The type is completely opaque, but an equivalent
+    //  handle object can be recovered from it.
+    class raw_handle_t {};
+    raw_handle_t *raw() const { return (raw_handle_t *) index; }
+    index_handle_t(raw_handle_t *_index) : index((index_ctx *) _index) {}
+
+    //###############
+    //  Handle OPS
+    db_string name() const { return index->indexname.as_string(); }
+    
+    bool is_unique() const { return index->defn.require_unique; }
+    
+    uint32 column_count() const { return (uint32) index->defn.col_count; }
+    
+    column_selector_t index_columns() const {
+        return index->index_columns();
+    }
+
+    class column_defs_accessor
+    {
+        index_ctx *index;
+    public:
+        column_defs_accessor() : index(NULL) {}
+        column_defs_accessor(index_ctx *_index) : index(_index) {}
+        
+        class column_itr : public boost::iterator_adaptor<
+            column_itr, 
+            idx_column_iterator,
+            index_column_def,
+            boost::bidirectional_traversal_tag,
+            index_column_def
+            >
+        {
+            typedef boost::iterator_adaptor<
+                column_itr, 
+                idx_column_iterator,
+                index_column_def,
+                boost::bidirectional_traversal_tag,
+                index_column_def
+                > base_t;
+    
+            friend class boost::iterator_core_access;
+        public:
+            column_itr() {}
+            column_itr(const idx_column_iterator &_i) : base_t(_i) {}
+            
+        private:
+            index_column_def dereference() const { return base_t::base().make_def(); }
+        };
+        
+        size_t column_count() const { return index->defn.col_count; }
+        column_itr begin() const { return column_itr(index->coldefs_begin()); }
+        column_itr end() const { return column_itr(index->coldefs_end()); }
+    };
+    
+    column_defs_accessor columns() const { return column_defs_accessor(index); }
+    
+    typename table_ctx::table_handle table() const { 
+        return typename table_ctx::table_handle(index->table);
+    }
+    
+    //  Indices can be iterated forward and backward.
+    //  The position given describes the values of the indexed columns.  It
+    //  need not describe all the indexed columns.  A partial value specifies an 
+    //  exclusive lower bound on the resulting iterator.
+    template<typename ITR_T>
+    iterator lower_bound(ITR_T key_begin, ITR_T key_end) const { 
+        return index->lower_bound(simple_sequence<ITR_T>(key_begin, key_end));
+    }
+    template<typename ITR_T>
+    iterator upper_bound(ITR_T key_begin, ITR_T key_end) const { 
+        return index->upper_bound(simple_sequence<ITR_T>(key_begin, key_end));
+    }
+    template<typename ITR_T>
+    iterator find(ITR_T key_begin, ITR_T key_end) const { 
+        return index->find(simple_sequence<ITR_T>(key_begin, key_end));
+    }
+    iterator begin() const { 
+        return index->begin();
+    }
+    iterator end() const { 
+        return index->end();
+    }
+    
+    cursor_handle new_cursor(const iterator &where) const {
+        if (index != where.pending().idx) {
+            //  Invalid iterator, or iterator over a different index.
+            throw_error(invalid_argument());
+        }
+
+        return cursor_handle(safe_idx_cursor_host(index), make_safe_itr(where));
+    }
+    void delete_cursor(cursor_handle &where) const {
+        where.free();
+    }
+    
+    //  Reset the cursor position to that defined by the given iterator.
+    //  This is more efficient than deleting and recreating a new cursor.
+    void move_cursor(cursor_handle &crs, const iterator &where) const {
+        if ((index != crs.host.idx) ||
+            (index != where.pending().idx)) 
+        {
+            //  Cursor/Iterator/IxHandle do not match.
+            throw_error(invalid_argument());
+        }
+
+        crs.iterator() = make_safe_itr(where);
+    }
+
+    uint32 row_count() const { 
+        return index->table->row_count(); 
+    }
+private:
+    safe_itr_t make_safe_itr(const iterator &where) const {
+        bool p_bbegin = where.pending_range().before_begin();
+        pending_row_t *p_row = where.pending_range().is_valid() ? where.pending().to_row() : NULL;
+        
+        return safe_itr_t(
+            index->table, 
+            where.fixed_range(),
+            safe_pending_cursor(safe_idx_cursor_host(index), p_row, p_bbegin));
+    }
+};
+
+} //namespace Internal
+} //namespace Oinky
+