oinky 0.1.0

data/ext/include/oinky.hpp

@@ -0,0 +1,63 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+//  This is the C++ header for Oinky.  It provides a header-only implementation
+//  of Oinky, requiring no library.
+
+#ifndef OINKY_HPP_INCLUDED
+#define OINKY_HPP_INCLUDED
+
+//stdc++
+#include <cstdio>
+#include <map>
+#include <memory>
+//TR1
+#include <boost/tr1/tr1/unordered_map>
+//  This is for shared_ptr
+#include <boost/tr1/tr1/memory>
+//#include <unordered_map>
+//BOOST
+#include <boost/bind.hpp>
+#include <boost/date_time.hpp>
+#include <boost/detail/endian.hpp>
+#include <boost/foreach.hpp>
+#include <boost/function.hpp>
+#include <boost/intrusive/list.hpp>
+#include <boost/intrusive/avl_set.hpp>
+#include <boost/intrusive/detail/rbtree_node.hpp>
+#include <boost/intrusive/rbtree_algorithms.hpp>
+#include <boost/intrusive/set.hpp>
+#include <boost/iterator/indirect_iterator.hpp>
+#include <boost/iterator/transform_iterator.hpp>
+#include <boost/iterator/filter_iterator.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/static_assert.hpp>
+#include <boost/strong_typedef.hpp>
+#include <boost/system/error_code.hpp>
+#include <boost/system/system_error.hpp>
+#include <boost/throw_exception.hpp>
+#include <boost/type_traits.hpp>
+#include <boost/utility.hpp>
+
+#include "oinky/nky_dialect.hpp"
+#include "oinky/nky_error.hpp"
+#include "oinky/nky_public.hpp"
+#include "oinky/nky_pool.hpp"
+#include "oinky/nky_merge_itr.hpp"
+#include "oinky/nky_base.hpp"
+#include "oinky/nky_strtable.hpp"
+#include "oinky/nky_core.hpp"
+#include "oinky/nky_fixed_table.hpp"
+#include "oinky/nky_cursor.hpp"
+#include "oinky/nky_index.hpp"
+#include "oinky/nky_table.hpp"
+#include "oinky/nky_handle.hpp"
+#include "oinky/nky_serializer.hpp"
+#include "oinky/nky_model.hpp"
+
+//#ifndef OINKY_HPP_INCLUDED
+#endif
+

data/ext/include/oinky/nky_base.hpp

@@ -0,0 +1,1116 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+//  This is a serialization engine for a SQL-like database.
+//
+//  A SQL database implementation is often separated between storage
+//  engine and query engine.  We further subdivide the storage engine into
+//  two layers: serialization and persistence.  This layering is not often
+//  always a good one for what I would call "online databases," which are 
+//  designed to support a large number of concurrent writes, while minimizing
+//  I/O.  Such implementations can benefit from unifying the serialization/
+//  persistence processes, using techniques like logging, etc.
+//
+//  The separation chosen here is designed for a more specific scenario:
+//      1.  Reads are optimized over writes.
+//      2.  Seek/addressing IO is optimized over raw byte IO throughput.
+//      3.  Extremely large dataset, with well understood partitioning 
+//          (suitable for sharding).
+//
+
+
+namespace Oinky
+{
+using namespace Oinky::Errors;
+using namespace Oinky::Utils;
+
+namespace Internal
+{
+
+template<typename TABLE_CTX>
+class column_selector_template;
+
+template<typename DB>
+class table_ctx_t;
+
+template<typename TABLE_CTX>
+class index_ctx_t;
+
+template<typename TABLE_CTX>
+class table_handle_t;
+
+//  No practical limit on the number of strings.
+typedef uint32 base_strtable_ref;
+
+//  There were a few reasons for separating the metastrings table and the
+//  userstrings table.  One was that the metastrings table would be modified
+//  less often (only on schema change) so could usually be serialized more
+//  efficiently, but it's so much smaller that I doubt it matters.  Another
+//  was that meta-string lookup time would be more important to performance,
+//  but I doubt that too.  At this point, I also don't see a strong argument
+//  for merging them into the same table, so separated they remain.
+//
+//  The table reference types are identical.  However, these typedefs keep
+//  us from accidentally assigning a reference from the the wrong table.
+BOOST_STRONG_TYPEDEF(base_strtable_ref, u_strtable_ref);
+BOOST_STRONG_TYPEDEF(base_strtable_ref, m_strtable_ref);
+
+//  This limits us to 65535 columns.
+BOOST_STRONG_TYPEDEF(uint16,column_idx_t);
+//  This limits us to 65535 indexes.
+BOOST_STRONG_TYPEDEF(uint16,index_idx_t);
+//  Actual indices are stored with variable width row indexes.
+BOOST_STRONG_TYPEDEF(uint32,row_idx_t);
+
+
+} //namespace Internal
+
+
+namespace Serialization
+{
+    struct v1_sformat_tag {};
+
+    template<typename TARGET, typename SCHEME>
+    class Serializer
+    {
+        BOOST_STATIC_ASSERT(sizeof(TARGET) != sizeof(TARGET));
+    };
+
+
+    template<typename TARGET>
+    class NativeSerializer
+    {
+    public:
+        //  Native types also support non-bounds-checked unpack.  The caller
+        //  has already checked the buffer type (for an array, perhaps),
+        //  and need not check it for every element.
+        inline static void pack_nbc(const TARGET &s, char *buffer_start)
+        {
+            *(TARGET *)buffer_start = s;
+        }
+        inline static void unpack_nbc(TARGET *t, const char *buffer)
+        {
+            *t = * (const TARGET *) buffer;
+        }
+    };
+
+    //  This is a dumb serializer concept.  It stands for non-bounds-checked.
+    //
+    //  It doe the actual work of pack/unpack after the bounds check has been
+    //  done, perhaps by a derived template, or by the owner of an array
+    //  of such types.
+    template<typename TARGET>
+    class NBCSerializer
+    {
+        //  This must be specialized for each type/platform.  It can be either
+        //  a NativeSerializer or a ByteSwapSerializer.
+        BOOST_STATIC_ASSERT(sizeof(TARGET) != sizeof(TARGET));
+    };
+
+#define OINKY_USE_NATIVE_NBC_SERIALIZER(target)                 \
+    template<> class NBCSerializer<target> :                    \
+    public NativeSerializer<target> {};
+
+    template<typename TARGET>
+    class FixedSerializer : public NBCSerializer<TARGET>
+    {
+    public:
+        using NBCSerializer<TARGET>::pack_nbc;
+        using NBCSerializer<TARGET>::unpack_nbc;
+        
+        //  Native serialization is independent of value.  Not all serializers are.
+        inline static uint32 virtual_pack() {
+            return sizeof(TARGET);
+        }
+        inline static uint32 virtual_pack(const TARGET &s) {
+            return sizeof(TARGET);
+        }
+        
+        inline static void pack(const TARGET &s, char *&buffer_start, const char *buffer_end)
+        {
+            if (buffer_end - buffer_start < sizeof(TARGET)) {
+                throw_error(buffer_overflow());
+            }
+            pack_nbc(s, buffer_start);
+            buffer_start += sizeof(TARGET);
+        }
+        
+        inline static void unpack(TARGET *t, const char *buffer, uint32 buflen, uint32 last_end, uint32 &this_end)
+        {
+            if (last_end + sizeof(TARGET) > buflen) {
+                throw_error(buffer_underflow());
+            }
+            unpack_nbc(t, buffer + last_end);
+            this_end = last_end + sizeof(TARGET);
+        }
+    };
+    
+    //  This only advances this_end if the string matches.
+    static bool check_bytes(const char *bytes, uint32 count, const char *buffer, uint32 buflen, uint32 last_end, uint32 &this_end)
+    {
+        if (count + last_end > buflen) {
+            throw_error(buffer_underflow());
+        }
+        if (0 == memcmp(bytes, buffer + last_end, count)) {
+            this_end = last_end + count;
+            return true;
+        }
+        return false;
+    }
+
+#define OINKY_ENABLE_FIXED_SERIALIZATION(type,scheme)               \
+    template<> class Serializer<type,scheme> :                      \
+    public FixedSerializer<type> {};    
+
+#define OINKY_INHERIT_NBC_SERIALIZER(derived,base)                      \
+    template<> class NBCSerializer<derived> {                           \
+    public:                                                             \
+        BOOST_STATIC_ASSERT(sizeof(base) == sizeof(derived));           \
+        inline static void pack_nbc(const derived &s, char *buffer) {   \
+            NBCSerializer<base>::pack_nbc(*(const base *)&s,buffer);      \
+        }                                                               \
+        inline static void unpack_nbc(derived *t, const char *buffer) { \
+            NBCSerializer<base>::unpack_nbc((base *)t,buffer);          \
+        }                                                               \
+    };
+
+#define OINKY_INHERIT_FIXED_SERIALIZER(derived,base,scheme)         \
+    OINKY_INHERIT_NBC_SERIALIZER(derived,base)                      \
+    OINKY_ENABLE_FIXED_SERIALIZATION(derived,scheme)                \
+
+    //  This is a simple helper which infers the target type.
+    template<typename SCHEME, typename T>
+    inline void unpack(T* target, const char *buffer, uint32 buflen, uint32 last_end, uint32 &this_end) {
+        Serializer<T,SCHEME>::unpack(target, buffer, buflen, last_end, this_end);
+    }
+    template<typename SCHEME, typename T>
+    inline void unpack_nbc(T* target, const char *buffer) {
+        Serializer<T,SCHEME>::unpack_nbc(target, buffer);
+    }
+    template<typename SCHEME, typename T>
+    inline void pack(const T&s, char *&buffer_start, const char *buffer_end) {
+        Serializer<T,SCHEME>::pack(s, buffer_start, buffer_end);
+    }
+    template<typename SCHEME, typename T>
+    inline void pack_nbc(const T&s, char *buffer_start) {
+        Serializer<T,SCHEME>::pack_nbc(s, buffer_start);
+    }
+    template<typename SCHEME, typename T>
+    inline uint32 virtual_pack(const T&s) {
+        return Serializer<T,SCHEME>::virtual_pack(s);
+    }
+
+    //  Bool is more specific than uint8
+    template<>
+    class NBCSerializer<bool>
+    {
+    public:
+        static inline void pack_nbc(const bool &s, char *buffer_start)
+        {
+            *(uint8 *)buffer_start = s ? 1 : 0;
+        }
+        static inline void unpack_nbc(bool *t, const char *buffer) {
+            int8 value = *buffer;
+            if (value == 1) {
+                *t = true;
+            } else if (value == 0) {
+                *t = false;
+            } else {
+                throw_error(bad_encoding());
+            }
+        }
+    };
+
+
+
+//  ##################
+//  Enable for testing.  This isn't a great test, but it will catch any
+//  cases where you forget an odd number of times to reverse bytes.
+//#define OINKY_REVERSE_NATIVE_ENDIAN_TEST
+//  ##################
+
+
+//  we use little endian encoding unless we're testing
+#if defined(BOOST_BIG_ENDIAN)
+
+#ifndef OINKY_REVERSE_NATIVE_ENDIAN_TEST
+#define OINKY_REVERSE_ENDIAN
+#define OINKY_LITTLE_ENDIAN_ENCODING
+#else
+#define OINKY_BIG_ENDIAN_ENCODING
+#endif
+
+//#ifdef BOOST_BIG_ENDIAN
+#elif defined(BOOST_LITTLE_ENDIAN)
+
+#ifdef OINKY_REVERSE_NATIVE_ENDIAN_TEST
+#define OINKY_REVERSE_ENDIAN
+#define OINKY_BIG_ENDIAN_ENCODING
+#else
+#define OINKY_LITTLE_ENDIAN_ENCODING
+#endif
+
+//#ifdef BOOST_BIG_ENDIAN / #elif defined(BOOST_LITTLE_ENDIAN)
+#else
+#error "Unrecognized architecture.  Cannot define serializer."
+#endif
+
+//  universal
+OINKY_USE_NATIVE_NBC_SERIALIZER(uint8)
+
+#ifndef OINKY_REVERSE_ENDIAN
+OINKY_USE_NATIVE_NBC_SERIALIZER(uint64)
+OINKY_USE_NATIVE_NBC_SERIALIZER(uint32)
+OINKY_USE_NATIVE_NBC_SERIALIZER(uint16)
+#else
+    template<> class NBCSerializer<uint16> {
+    public:
+        static inline void pack_nbc(const uint16 &s, char *buffer_start) {
+            *(uint16 *)buffer_start = (s >> 8) | (s << 8);
+        }
+        static inline void unpack_nbc(uint16 *t, const char *buffer) {
+            uint16 x = *(const uint16 *)buffer;
+            *t = (x >> 8) | (x << 8);
+        }
+    };
+    template<> class NBCSerializer<uint32> {
+    public:
+        static inline void pack_nbc(const uint32 &s, char *buffer_start) {
+            *(int32 *)buffer_start = __builtin_bswap32((int32)s);
+        }
+        static inline void unpack_nbc(uint32 *t, const char *buffer) {
+            *t = (uint32) __builtin_bswap32(*(int32 *)buffer);
+        }
+    };
+    template<> class NBCSerializer<uint64> {
+    public:
+        static inline void pack_nbc(const uint64 &s, char *buffer_start) {
+            *(int64 *)buffer_start = __builtin_bswap64((int64)s);
+        }
+        static inline void unpack_nbc(uint64 *t, const char *buffer) {
+            *t = (uint64) __builtin_bswap64(*(int64 *)buffer);
+        }
+    };
+#endif
+
+//  We use the fixed serializer for all of these.
+OINKY_ENABLE_FIXED_SERIALIZATION(uint64,v1_sformat_tag)
+OINKY_ENABLE_FIXED_SERIALIZATION(uint32,v1_sformat_tag)
+OINKY_ENABLE_FIXED_SERIALIZATION(uint16,v1_sformat_tag)
+OINKY_ENABLE_FIXED_SERIALIZATION(uint8,v1_sformat_tag)
+
+//  All of these types derive from the 4 basic NBC serializers.
+OINKY_INHERIT_FIXED_SERIALIZER(int64,uint64,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(int32,uint32,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(int16,uint16,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(int8,uint8,v1_sformat_tag)
+
+//  Bool has its own NBC serializer, defined above.
+OINKY_ENABLE_FIXED_SERIALIZATION(bool,v1_sformat_tag)
+
+//  Now the fake types (identical to above)
+using namespace Oinky::Internal;
+OINKY_INHERIT_FIXED_SERIALIZER(datetime_t,int64,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(u_strtable_ref,uint32,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(m_strtable_ref,uint32,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(column_idx_t,uint16,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(index_idx_t,uint16,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(row_idx_t,uint32,v1_sformat_tag)
+
+OINKY_INHERIT_FIXED_SERIALIZER(float32,int32,v1_sformat_tag)
+OINKY_INHERIT_FIXED_SERIALIZER(float64,int64,v1_sformat_tag)
+
+} //namespace Serialization
+
+
+namespace Internal
+{
+
+typedef boost::intrusive::avl_set_base_hook<> strmap_hook;
+
+template<typename REF_T>
+class indirect_string_header_t : public strmap_hook, boost::noncopyable
+{
+    typedef indirect_string_header_t header_t;
+    friend class stringtable_t<REF_T>;
+
+    //  This is meaningless at first, but the serializer later uses this
+    //  to cache the reference value that will be used in serialization.
+    //  Only during the serialization process is this field used.
+    REF_T _ref;
+    uint32 _length;
+            
+    indirect_string_header_t(const char *str, uint32 length) : _ref(0), _length(length) {
+        memcpy(((char *)this) + sizeof(header_t), str, length);
+    }
+
+    struct compare_header {
+        bool operator()(const header_t &left, const header_t &right) const {
+            return left.as_string() < right.as_string();
+        }
+        bool operator()(const db_string &left, const header_t &right) const {
+            return left < right.as_string();
+        }
+        bool operator()(const header_t &left, const db_string &right) const {
+            return left.as_string() < right;
+        }
+    };
+    
+public:
+    db_string as_string() const { return db_string(begin(), _length); }
+
+    const char *begin() const { return ((const char *) this) + sizeof(header_t); }
+    const char *end() const { return begin() + _length; }
+    uint32 length() const { return _length; }
+
+    //  The live contexts.
+    typedef boost::intrusive::avl_set<
+        header_t, 
+        boost::intrusive::compare<compare_header> > strmap_t;
+    typedef typename strmap_t::iterator strmap_itr_t;
+
+    //  Given a tree and a string, return a pre-existing entry from the tree
+    //  or create/insert a new one.  Always returns a valid entry.
+    template<typename ALLOC>
+    static header_t *prepare(ALLOC &alloc, strmap_t &treehead, const db_string &str) {
+        //  Search/prepare insert in one op.
+        typename strmap_t::insert_commit_data commit;
+        std::pair<strmap_itr_t,bool> res = treehead.insert_check(
+            str,
+            compare_header(),
+            commit);
+            
+        //  res.second tells us if a value CAN be inserted.  False indicates that
+        //  the iterator is valid.  It is synonymous with the PRE-condition of map.insert
+        if (!res.second) {
+            OINKY_ASSERT(res.first->as_string() == str);
+            return &(*res.first);
+        }
+
+        header_t *x = (header_t *) alloc->malloc(sizeof(header_t) + str.length());
+        //  Init
+        ::new(x) header_t(str.begin(), str.length());
+        //  Insert
+        treehead.insert_commit(*x, commit);
+        return x;
+    }
+    
+    static void clear_reference_counts(strmap_t &map)
+    {
+        strmap_itr_t i = map.begin();
+        strmap_itr_t end = map.end();
+        for (;i != end; ++i) {
+            i->_ref = 0;
+        }
+    }
+};
+
+//  Once a string is inserted into the database, we allocate space for it in the
+//  private heap, and do not reference it thereafter.  Thus, this has no reference.
+//  The string's lifetime is equivalent to that of the DB instance.
+//
+//  This structure can point to strings in the serialized string table, or to
+//  newly inserted strings, which have not yet been given a position in the
+//  table.
+//
+//  This is identical to the db_string, except a string of this type
+//  can be trusted to be stored in the DB heap before it is constructed.  
+//  Therefore, any method which may save the string value (such as table-insert)
+//  must demand this parameter type, while methods such as find() may require
+//  only the unsafe, db_string type.
+//
+template<typename REF_T> 
+class u_string_value_safe
+{
+    friend class stringtable_t<REF_T>;
+    friend class safe_cv_t;
+    typedef u_string_value_safe<REF_T> safestr_t;
+    typedef indirect_string_header_t<REF_T> header_t;
+
+    //  If this is a new string, this will be non-null.  If it's from the 
+    //  stringtable, this will be NULL, and the db_string's reference will be 
+    //  valid.
+    header_t *newstr;
+    db_string str;
+
+public:
+    //  Possibly it will be neither, if it is uninitialized.
+    bool is_new() const { return newstr; }
+    bool is_old() const { return str.xref != (REF_T) 0; }
+
+    REF_T sref() const { 
+        OINKY_ASSERT(is_old());
+        return (REF_T) str.xref;
+    }
+    
+    const db_string &as_string() const {
+        return str;
+    }
+    operator const db_string &() const {
+        return str;
+    }
+    const db_string *operator->() const { 
+        return &str; 
+    }
+
+    u_string_value_safe() : newstr(NULL) {}
+  
+    template<typename T>
+    bool operator==(const T &other) const { return compare_to(other) == 0; }
+    template<typename T>
+    bool operator!=(const T &other) const { return compare_to(other) != 0; }
+    template<typename T>
+    bool operator<(const T &other) const { return compare_to(other) < 0; }
+    template<typename T>
+    bool operator<=(const T &other) const { return compare_to(other) <= 0; }
+    template<typename T>
+    bool operator>(const T &other) const { return compare_to(other) > 0; }
+    template<typename T>
+    bool operator>=(const T &other) const { return compare_to(other) >= 0; }
+    
+    int compare_to(const db_string &other) const {
+        return str.compare_to(other);
+    }
+
+    int compare_to(const safestr_t &other) const {
+        if (is_old() && other.is_old()) {
+            if (sref() < other.sref()) return -1;
+            if (sref() > other.sref()) return 1;
+            return 0;
+        }
+        return str.compare_to(other.str);
+    }
+};
+
+typedef u_string_value_safe<u_strtable_ref> ustring_safe;
+typedef u_string_value_safe<m_strtable_ref> mstring_safe;
+
+//  Safe column value just means localizing any string values.  Everything
+//  else is a straightforward copy.
+class safe_cv_t
+{
+    friend class stringtable_t<u_strtable_ref>;
+    
+    //  Column values are always user strings.
+    typedef indirect_string_header_t<u_strtable_ref> header_t;
+
+    //  This will only be valid (non-null) if the value is a string, and
+    //  it is new.  (Highly similar to u_string_value_safe;
+    header_t *newstr;
+    variant_cv_t _value;
+public:
+    safe_cv_t() : newstr(NULL) {}
+    
+    safe_cv_t(const ustring_safe &str) : newstr(str.newstr), _value(str) {}
+
+    safe_cv_t(bool v) : newstr(NULL), _value(v) {}
+    safe_cv_t(int8 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(int16 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(int32 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(int64 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(uint8 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(uint16 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(uint32 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(uint64 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(float32 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(float64 v) : newstr(NULL), _value(v) {}
+    safe_cv_t(datetime_t v) : newstr(NULL), _value(v) {}
+
+    inline void check_valid() const {
+        OINKY_ASSERT(_value.type() && (_value.type() <= OINKY_VALUE_TYPE_MAX));
+        OINKY_ASSERT((_value.type() != value_types::String) || 
+            (memcmp(string_value().begin(), string_value().begin(), string_value().length()) == 0));
+    }
+    
+    template<typename STRTABLE>
+    safe_cv_t(column_type_code_t ct, const variant_cv_t &__val, STRTABLE *strtable) :
+    newstr(NULL),
+        _value(__val)
+    {
+        //  First coerce the value.  This will raise an exception if the
+        //  column and value types are incompatible.
+        _value.coerce(ct);
+        //  Now if the value is a string, we need to internalize it.
+        if (_value.is_string()) {
+            ustring_safe us = strtable->make_safestring(__val.string_value());
+            
+            _value.string_value() = us;
+            newstr = us.newstr;
+        }
+    }
+
+    //  This is to be called when we are converting a serialized value to a 
+    //  safe value.  Such transformations are "safe" because we know the 
+    //  indirect value is already stored, and our reference should be valid.
+    //  Thus, we know that we need not store an indirect pointer.  It should
+    //  obviously not be called except when unpacking fixed values.
+    template<typename STRTABLE>
+    static safe_cv_t from_fixed(const variant_cv_t &val, const STRTABLE *strtable) {
+        safe_cv_t r;
+        r._value = val;
+        r.newstr = NULL;
+        //  Can assert validity.  But should have been checked by caller.
+        //  We know the string isn't new.
+        if (r._value.type() == value_types::String) {
+            const db_string &str(r._value.string_value());
+            OINKY_ASSERT( str.xref && (str == strtable->from_untrusted_ref((u_strtable_ref) str.xref)) );
+        }
+        return r;
+    }
+
+    void coerce(column_type_code_t ct) {
+        //  Only integers get coerced, so this is orthogonal to 
+        //  newstr, so we can just pass it through.
+        _value.coerce(ct);
+    }
+
+    //  This could change it to something unsafe.
+    //db_string &string_value() { return _value.string_value(); }
+    const db_string &string_value() const { return _value.string_value(); }
+    
+    datetime_t &dt_value() { return _value.dt_value(); }
+    const datetime_t &dt_value() const { return _value.dt_value(); }
+    
+    int64 &int_value() { return _value.int_value(); }
+    const int64 &int_value() const { return _value.int_value(); }
+    
+    uint64 &uint_value() { return _value.uint_value(); }
+    const uint64 &uint_value() const { return _value.uint_value(); }
+    
+    float64 &f64_value() { return _value.f64_value(); }
+    const float64 &f64_value() const { return _value.f64_value(); }
+    
+    float32 &f32_value() { return _value.f32_value(); }
+    const float32 &f32_value() const { return _value.f32_value(); }
+    
+    bool &bit_value() { return _value.bit_value(); }
+    bool bit_value() const { return _value.bit_value(); }
+    
+    bool is_string() const { return _value.is_string(); }
+    bool is_date() const { return _value.is_date(); }
+    bool is_int() const { return _value.is_int(); }    
+    bool is_uint() const { return _value.is_uint(); }    
+    
+    value_type_code_t type() const { return _value.type(); }
+    
+    const variant_cv_t &value() const { return _value; }
+    operator const variant_cv_t &() const { return _value; }
+    
+    int compare_to(const safe_cv_t &other) const {
+        return _value.compare_to(other.value());
+    }
+    int compare_to(const variant_cv_t &other) const {
+        return _value.compare_to(other);
+    }
+};
+
+//
+//  A pair of savepoint markers defines the lifespan of the object.  The
+//  creation stamp and deletion stamp.
+//
+struct ls_marker
+{
+    ls_marker() : insert_sp(0), delete_sp(0) {}
+    
+    //  The top SP marker at the time the object was created.  0 if the
+    //  object was deserialized (created by a previous instance).
+    //
+    //  If/when this object is un-created by a savepoint rollback, we will
+    //  destroy/unlink it, and it will no longer be discoverable.
+    //
+    //  If the object gets deleted in the same SP in which it is inserted
+    //  (a new SP isn't created in the interim) them we can delete it
+    //  immediately.  Thus, the primary savepoint overhead (zombie pending_row
+    //  objects) is only paid in the case that savepoints are actually being
+    //  used.  This restricts the paranoid behavior scenario to the one
+    //  where the same rows are being repeatedly updated (deleted/inserted) 
+    //  AND savepoints are being set between each update.
+    sp_marker_t insert_sp;
+    
+    //  Has this row been deleted?  If so, this will be nonzero, equal to the
+    //  value of the savepoint marker at the time of delete.
+    //
+    //  If/when this object is un-deleted by a savepoint rollback, this will
+    //  be reset to zero.
+    sp_marker_t delete_sp;
+    
+    bool is_deleted() const { return delete_sp != 0; }
+};
+
+struct row_offset_accumulator_t {
+    //  Non-bit columns will not update the byte_offset.
+    //  Bit columns will increment the bit count.
+    uint32 byte_offset;
+    uint16 bit_count;
+    
+    uint32 full_width() const { return ((bit_count + 7) >> 3) + byte_offset; }
+    
+    row_offset_accumulator_t() : byte_offset(0), bit_count(0) {}
+};
+
+//  This is the dynamic column context.  In serialization, the column definition
+//  vector and the column ordering in the serialized rows are equivalent.  However,
+//  if we alter a table, these will change.
+//
+//  Explode is the first step of alter-table.  So the serialized values are 
+//  not necessarily interesting, except if we've added rows in the interim, we
+//  have pending rows which are stored in the same order.  Rather than swap
+//  all column values each time we insert a column (which is O(N) expensive
+//  for each column insert) we just keep this mapping.  The overhead of this
+//  amounts to a single small allocation per table on mount.
+//
+//  The index definitions reference columns by their index.  Thus, we maintain 
+//  a mapping from index to column contexts.
+class column_ctx
+{
+public:
+    mstring_safe colname;
+    safe_cv_t default_value;
+    column_type_code_t ctype;
+    
+    //  Column create/drop
+    ls_marker ls;
+    
+    typedef column_idx_t position_idx_t;
+    
+    //  When we serialize the column set, we assign each column a new position,
+    //  restoring the density and sorted-column order, relative to what we 
+    //  maintain in memory.
+    column_idx_t new_position;
+    
+    //  Bytes from the beginning of the row data that this value is
+    //  stored.  The fixed and pending storage ranges are different because
+    //  certain types (string/variant) are larger in pending-rows than they
+    //  are in fixed rows.
+    
+    //  --pending
+    uint32 prow_byte_offset;
+    uint8 prow_bit_offset;
+    //  --fixed
+    uint32 frow_byte_offset;
+    uint8 frow_bit_offset;
+    
+    //  When we assign the new_position, we also recompute what bytes (bits)
+    //  in the fixed row this column will consume.
+    uint32 new_row_offset_bytes;
+    //  We compute this in two passes.  The first pass counts the number of 
+    //  single-bit columns, and thus needs to have that full range.  We
+    //  will do a second-pass to shift all but the low 3 bits into 
+    //  new_row_offset_bytes, after we know the non-bit serialized row length.
+    uint32 new_bit_offset;
+
+    inline column_type_code_t type() const { return ctype; }
+    inline bool is_bit() const { return ctype == column_types::Bit; }
+
+public:
+    //  Take the user's value and make it internal.  This involves two steps:
+    //  1) coercion to column's type (if possible...exception otherwise).
+    //  2) internalizing any indirect data and getting a reference to the header.
+    template<typename STRINGTABLE_T>
+    safe_cv_t internalize(const variant_cv_t &val, STRINGTABLE_T *strtable) const {
+        return safe_cv_t(type(), val, strtable);
+    }
+
+    //  This is the public/user interface to the index_ctx.
+    class column_handle : public table_column_def
+    {
+        const column_ctx *col;
+        friend class column_ctx;
+        
+    public:
+        column_handle() {}
+        
+        column_handle(const column_ctx *_col) : 
+        table_column_def(
+            _col->colname.as_string(), 
+            _col->type(),
+            variant_cv_t(_col->default_value.value())),
+            col(_col)
+        {}
+    };
+    
+    static const column_ctx *from_handle(const column_handle &h) {
+        return h.col;
+    }
+};
+
+//  Column descriptor for index columns.
+struct idx_column_ctx
+{
+    const column_ctx *column;
+    bool ascending;
+    //  This is the pos'th column in the index.  This is equal to
+    //  itr - index_def.begin().  Ranges from 0 to (index.width-1)
+    column_idx_t index_pos;
+    
+    operator const column_ctx &() const {
+        return *column;
+    }
+};
+
+//  This is a concept, not a class.  It can be used to enumerate/extract/compare
+//  multicolumn values.  It also permits examination of column definitions
+//  (types/names), column count, etc.  It does this for both serialized and
+//  dynamic rows, as well as both tables and indexes.
+/*
+struct multicolumn_value_accessor_concept
+{
+    uint32 column_count();
+
+    //  Parameters are a column definition and a value
+    //  bool (*fn)(COL_DEF coldef, const variant_cv_t &val);
+    //
+    //  COL_DEF depends on whether an index or a 
+    template<typename FN>
+    void each_column_value(FN fn) const;
+};*/
+
+//  This adds funtionality to the above methods.
+template<typename BASE>
+class multicolumn_value_accessor : public BASE
+{
+public:
+    multicolumn_value_accessor() : BASE() {}
+    template<typename T1>
+    multicolumn_value_accessor(T1 t1) : BASE(t1) {}
+    template<typename T1, typename T2>
+    multicolumn_value_accessor(T1 t1, T2 t2) : BASE(t1, t2) {}
+    template<typename T1, typename T2, typename T3>
+    multicolumn_value_accessor(T1 t1, T2 t2, T3 t3) : BASE(t1, t2, t3) {}
+    template<typename T1, typename T2, typename T3, typename T4>
+    multicolumn_value_accessor(T1 t1, T2 t2, T3 t3, T4 t4) : BASE(t1, t2, t3, t4) {}
+
+private:    
+    //  This is an enumerator, which can be passed to a row_iterators column enumerator, 
+    //  which writes each element to a vector.
+    template<typename ITR>
+    static bool copy_row_cb_limit(ITR *i, uint32 *limit, const variant_cv_t &val)
+    {
+        **i = val;
+        ++(*i);
+        --(*limit);
+        //  We can only keep going if we have space remaining in the target.
+        return *limit > 0;
+    }
+    
+    template<typename ITR>
+    static bool copy_row_cb(ITR *i, const ITR *end, const variant_cv_t &val)
+    {
+        **i = val;
+        ++(*i);
+        //  We can only keep going if we have space remaining in the target.
+        return *i != *end;
+    }
+
+    template<typename OSTREAM>
+    static bool format_cb(const variant_cv_t &value, OSTREAM *os, uint32 *count) {
+        if (*count) {
+            (*os) << ", ";
+        }
+        (*os) << value;
+        ++(*count);
+        return true;
+    }
+public:
+    template<typename OSTREAM>
+    void format(OSTREAM &os) const {
+        uint32 count = 0;
+        BASE::each_column_value(boost::bind(&format_cb<OSTREAM>, _2, &os, &count));
+    }
+
+    typedef multicolumn_value_accessor<BASE> this_t;
+    
+    //  Returns the end iterator of the new column sequence.
+    template<typename OUT_ITR>
+    OUT_ITR copy_to(OUT_ITR i, uint32 limit) const {
+        BASE::each_column_value(
+            boost::bind(&copy_row_cb_limit<OUT_ITR>, &i, &limit, _2)
+            );
+        return i;
+    }
+    
+    //  Returns the end iterator of the new column sequence.
+    template<typename OUT_ITR>
+    OUT_ITR copy_to(OUT_ITR i, OUT_ITR end) const {
+        BASE::each_column_value(
+            boost::bind(&copy_row_cb<OUT_ITR>, &i, &end, _2)
+            );
+        return i;
+    }
+
+    //  Parameter is a variant_cv_t
+    template<typename FN>
+    void each_value(FN fn) const
+    {
+        //  
+        check_function_concept<bool(const variant_cv_t &cv)>(fn); 
+        
+        struct v_only
+        {
+            FN *fn;
+            v_only(FN *_fn) : fn(_fn) {}
+            
+            bool operator()(const table_column_def &col, const variant_cv_t &val) { 
+                return (*fn)(val);
+            }
+        };
+        v_only cb(&fn);
+        BASE::each_column_value(fn);
+    }
+    
+private:
+    template<typename UCV_ITR>
+    struct _cmp
+    {
+        struct ctx_t {
+            UCV_ITR i;
+            const UCV_ITR &pos_end;
+            int result;
+            
+            ctx_t(const UCV_ITR &_pos_begin, const UCV_ITR &_pos_end) :
+            i(_pos_begin), pos_end(_pos_end), result(0)
+            {}
+            
+            bool fn(bool reverse, const variant_cv_t &val)
+            {
+                //  Verify that enumeration is stopping when we ask it to.
+                OINKY_ASSERT(result == 0);
+                
+                //  The right position specifier is identical but shorter
+                //  than the left.  The left is therefore greater.
+                //
+                //  NOTE: We define the ordering for partial matches regardless
+                //  of the ascending/descending nature of unspecified columns.
+                if (i == pos_end) {
+                    result = 1;
+                    return false;
+                }
+                result = val.compare_to(*i);
+                //  Handle the reverse case.
+                if (reverse) {
+                    result = -result;
+                }
+                ++i;
+                //  If identical, then continue enumeration.
+                if (!result) return true;
+                //  Otherwise, stop.  We are done.
+                return false;
+            }
+        };
+        
+        ctx_t &ctx;
+        
+        _cmp(ctx_t &_ctx) : ctx(_ctx) {}
+                
+        //  Provide separate implementations, since this is used both for
+        //  tables and indexes.
+        //
+        //  NOTE: We provide a comparison function for tables even though
+        //  tables aren't naturally sorted.  Someone may still want to 
+        //  enumerate over a table and filter according to some comparison
+        //  function.
+        bool operator()(const column_ctx &idef, const variant_cv_t &val) const
+        {
+            return ctx.fn(false, val);
+        }
+    };
+    
+public:
+    //  Compares this row to an array of values.
+    template<typename UCV_ITR>
+    int compare_to(const UCV_ITR &pos_begin, const UCV_ITR &pos_end) const {
+        typename _cmp<UCV_ITR>::ctx_t c(pos_begin, pos_end);
+        _cmp<UCV_ITR> comparer(c);
+        BASE::each_column_value(comparer);
+        
+        //  If we didn't reach the end of the specifier, but matched the entire
+        //  iterator, then the specifier (right) is beyond the iterator(left).
+        if ((c.result == 0) && (c.i != pos_end)) {
+            return -1;
+        }
+        return c.result;
+    }
+
+    //  Compares this row to another row via its accessor.
+    template<typename OTHERBASE>
+    int compare_to(const multicolumn_value_accessor<OTHERBASE> &other) const {
+        //  Just stack-alloc a vector to hold the other's values, then 
+        //  compare self to the vector.
+        uint32 othersize = other.column_count();
+        variant_cv_t *othera = (variant_cv_t *) alloca(sizeof(variant_cv_t) * othersize);
+        other.copy_to(othera, othera + othersize);
+        return compare_to(othera, othera + othersize);
+    }
+};
+
+template<typename SELECTOR_T>
+struct column_selector_accessor
+{
+    typedef SELECTOR_T selector_t;
+    typedef typename selector_t::table_ctx table_ctx;
+    
+    const selector_t &cs;
+    
+    column_selector_accessor(const selector_t &_cs) : cs(_cs) {}
+    
+    const table_ctx *table() const { return cs.table; }
+    const column_ctx *const *colrefs() const { return cs.colrefs; }
+    uint32 colcount() const { return cs.colcount; }
+    template<typename TBL>
+    void check_valid(TBL *table) const { cs.check_valid(table); }
+};
+
+template<typename ITR_T>
+class simple_sequence
+{
+    ITR_T _begin, _end;
+    uint32 _count;
+
+public:
+    typedef ITR_T itr_t;
+
+    simple_sequence(const ITR_T &__begin, const ITR_T &__end) :
+    _begin(__begin), _end(__end), _count(std::distance(_begin, _end)) 
+    {}
+    
+    itr_t begin() const { return _begin; }
+    itr_t end() const { return _end; }
+    uint32 size() const { return _count; }
+};
+
+inline uint8 compute_serialized_width(column_type_code_t ct) 
+{
+    if (ct > OINKY_VALUE_TYPE_MAX) {
+        throw_error(bad_encoding());
+    }
+    //  This is less than a byte, which we have no way of returning.
+    OINKY_ASSERT(ct != column_types::Bit);
+    
+    //  User string refs are always less than 8 bytes, so the variant width
+    //  is max() + 1 = 9
+    static const uint8 widths[] = { 9, 0, 1, 2, 4, 8, 8, sizeof(u_strtable_ref), 1, 2, 4, 8, 4, 8 };
+    BOOST_STATIC_ASSERT(OINKY_VALUE_TYPE_MAX + 1== sizeof(widths));
+    return widths[ct];
+}
+inline uint8 compute_pending_width(column_type_code_t ct) 
+{
+    //  This is less than a byte, which we have no way of returning.
+    OINKY_ASSERT(ct != column_types::Bit);
+    OINKY_ASSERT(ct <= OINKY_VALUE_TYPE_MAX);
+    
+    //  String and variant require safe_cv_t.  Everything else is identical to serialized.
+    static const uint8 widths[] = { sizeof(safe_cv_t), 0, 1, 2, 4, 8, 8, sizeof(safe_cv_t), 1, 2, 4, 8, 4, 8 };
+    BOOST_STATIC_ASSERT(OINKY_VALUE_TYPE_MAX + 1== sizeof(widths));
+    return widths[ct];
+}
+
+struct index_rowkey_xform
+{
+    uint8 dr_width;
+        
+    //  The xform process depends on machine architecture, not on target layout.
+    //  That's why we key off BOOST_BIN_ENDIAN not OINKY_REVERSE_ENDIAN
+    uint8 dr_shift;
+    uint32 dr_mask;
+
+    inline static uint8 compute_row_index_line_width(row_idx_t row_count) 
+    {
+        //  This case doesn't matter.  We won't serialize any.
+        if (row_count == 0) return 1;
+        //  Otherwise, the max idx is row_count-1.
+        uint32 rc = (uint32) row_count - 1;
+        if (rc & 0xff<<24) {
+            //  32 bit indices
+            return 4;
+        } else if (rc & 0xff<<16) {
+            //  24 bit indices
+            return 3;
+        } else if (rc & 0xff<<8) {
+            //  16 bit indices
+            return 2;
+        } else {
+            //  8 bit indices
+            return 1;
+        }
+    }
+
+    index_rowkey_xform() : dr_width(0) {}
+    index_rowkey_xform(row_idx_t row_count)
+    {
+        dr_width = compute_row_index_line_width(row_count);
+
+        const uint8 shift[] = {24, 16, 8, 0};
+        dr_shift = shift[dr_width-1];
+
+        const uint32 mask[] = {0xff, 0xffff, 0xffffff, 0xffffffff};
+        dr_mask = mask[dr_width-1];
+    }
+    
+    template<typename scheme>
+    inline row_idx_t unpack(row_idx_t position, const char *base) const {
+        OINKY_ASSERT(dr_width);
+        
+        BOOST_STATIC_ASSERT(sizeof(row_idx_t) == 4);
+        uint32 value;
+        uint32 s_value = * (uint32 *)(base + (position * dr_width));
+        Serialization::unpack_nbc<scheme,uint32>(&value, (const char *)&s_value);
+
+#if defined(OINKY_BIG_ENDIAN_ENCODING)
+        return (row_idx_t)(value >> dr_shift);
+#elif defined(OINKY_LITTLE_ENDIAN_ENCODING)
+        return (row_idx_t)(value & dr_mask);
+#else
+#error "OINKY ENDIAN ENCODING UNDEFINED"
+#endif
+    }
+
+    template<typename scheme>
+    inline uint32 make_svalue(row_idx_t value) const {
+        uint32 s_value;
+#if defined(OINKY_BIG_ENDIAN_ENCODING)
+        value <<= dr_shift;
+#elif defined(OINKY_LITTLE_ENDIAN_ENCODING)
+        OINKY_ASSERT((value & ~dr_mask) == 0);
+#else
+#error "OINKY ENDIAN ENCODING UNDEFINED"
+#endif
+        Serialization::pack_nbc<scheme,row_idx_t>(value, (char *)&s_value);
+        return s_value;
+    }
+    
+    template<typename scheme>
+    inline void pack_nbc_destructive(char *target, row_idx_t value) const {
+        uint32 s_value = make_svalue<scheme>(value);
+        *(uint32 *)target = (uint32) s_value;
+    }
+    
+    template<typename scheme>
+    inline void pack_nbc_orbits(char *target, row_idx_t value) const {
+        uint32 s_value = make_svalue<scheme>(value);
+        *(uint32 *)target |= (uint32) s_value;
+    }
+};
+
+
+} //namespace Internal
+} //namespace Oinky
+
+
+namespace std
+{
+
+template<class _Traits, class BASE> 
+inline std::basic_ostream<char, _Traits>& 
+operator<<(
+    std::basic_ostream<char, _Traits> &os, 
+    const Oinky::Internal::multicolumn_value_accessor<BASE> &accessor
+    )
+{
+    accessor.format(os);
+    return os;
+}
+
+}//namespace std
+