oinky 0.1.0

data/ext/include/oinky/nky_strtable.hpp

@@ -0,0 +1,504 @@
+//  This source is distributed under the terms of the MIT License.  Refer 
+//  to the 'LICENSE' file for details.
+//  
+//  Copyright (c) Jacob Lacouture, 2012
+
+namespace Oinky
+{
+namespace Internal
+{
+using namespace Oinky::Errors;
+
+
+//
+//  The stringtable is a single-instance store for all strings.  String 
+//  references are just indexes into the table.  The table is sorted so that
+//  string references collate identically to the strings themselves.  This
+//  means queries operate on the references only.  We only use the string table
+//  when converting to/from external interface values (queries/results).
+//
+
+template<typename REF_T>
+class stringtable_t : boost::noncopyable
+{
+    typedef stringtable_t<REF_T> stable_t;
+    
+    //  For now these are uint32s.  We could probably limit the stringtable
+    //  size to 16MB and use uint24s, but the savings would be minimal.
+    
+    //  These are offsets into the values buffer.  The length of the string is
+    //  determined by the difference between one offset and the next.
+    //
+    //  The validity of these offsets must be checked by the deserializer.  They
+    //  should all be less than vbuf_length.
+    nky_allocator_t *allocator;
+    const uint32 *offsets;
+    uint32 vcount;
+    const char *values;
+    uint32 vbuf_length;
+    
+    typedef u_string_value_safe<REF_T> safestr;
+    typedef typename safestr::header_t safe_header_t;
+    typedef typename safe_header_t::strmap_t strmap_t;
+    typedef typename strmap_t::iterator strmap_itr_t;
+    
+    strmap_t strmap;
+    db_vector<REF_T> offset_xform;
+    
+    //  Count of strings
+    //  Combined length of all string bytes.
+    //  Total length of stringtable in serialized form.
+    uint32 prepare_packed_string_count;
+    uint32 prepare_packed_stringslength;
+    uint32 prepare_packed_total_length;
+
+public:    
+    //  Default stringtable is empty.
+    stringtable_t(nky_allocator_t *_alloc) : 
+    allocator(_alloc),
+        offsets(NULL), 
+        vcount(0), 
+        values(NULL), 
+        vbuf_length(0),
+        offset_xform(_alloc) 
+    {}
+
+    stringtable_t(
+        nky_allocator_t *_alloc,
+        uint32 _vcount,
+        const uint32 *_offsets,
+        uint32 _vbuf_length,
+        const char *_vbuf
+        ) : 
+    allocator(_alloc),
+        offsets(_offsets), 
+        vcount(_vcount), 
+        values(_vbuf), 
+        vbuf_length(_vbuf_length),
+        offset_xform(_alloc) 
+    {}
+
+private:
+    template<bool is_pooled_alloc>
+    void strtable_teardown()
+    {
+        if (is_pooled_alloc) {
+            //  The default destructor walks the entire tree and tears it down. 
+            //  We don't need to do that, since we're about to dealloc the entire
+            //  pool anyway.  This clears the tree without walking anything.
+            ::new(&strmap) strmap_t();
+        } else {
+            //  This will free every object, but will not call the 
+            //  object's destructor.  This is most equivalent to what 
+            //  happens in the above case, with the pooled allocator.
+            strmap.clear_and_dispose(boost::bind(&nky_allocator_t::free, allocator, _1));
+        }
+    }
+    
+public:    
+    ~stringtable_t() 
+    {
+        strtable_teardown<is_pooled_allocator<nky_allocator_t>::value >();
+    }
+        
+    class iterator
+    {
+        uint32 offset;
+        const stable_t *host;
+    public:
+        iterator() : offset(0), host(NULL) {}
+        
+        iterator(const stable_t *_host, uint32 _offset) : 
+        offset(_offset), host(_host) 
+        {}
+        
+        iterator &operator++() {
+            ++offset;
+            return *this;
+        }
+        iterator &operator--() {
+            --offset;
+            return *this;
+        }
+        iterator &operator+=(int x) {
+            offset += x;
+            return *this;
+        }
+        iterator &operator-=(int x) {
+            offset -= x;
+            return *this;
+        }
+        
+        bool operator==(const iterator &other) const {
+            OINKY_ASSERT(host == other.host);
+            return offset == other.offset;
+        }
+        bool operator!=(const iterator &other) const {
+            OINKY_ASSERT(host == other.host);
+            return offset != other.offset;
+        }
+
+        int operator-(const iterator &other) const {
+            OINKY_ASSERT(host == other.host);
+            return (int) offset - (int) other.offset;
+        }
+        
+        u_string_value_safe<REF_T> to_string() const { 
+            return host->from_trusted_ref(value());
+        }
+        REF_T operator*() const { return value(); }
+        //  The REF is 1 + the offset into the table
+        REF_T value() const { return (REF_T) (offset + 1); }
+        
+        uint32 _offset() const { return offset; }
+    };
+
+    //  This does not do a search.  This is an array-lookup.  This is the 
+    //  retrieval method to convert column values to real strings.
+    //iterator map(REF_T ref) const {
+    //    return iterator(this, ref);
+    //}
+
+    //  Map but skipping the iterator.
+    safestr from_untrusted_ref(REF_T ref) const { 
+        //  This is an internal error.  The should never see the REF_T type.
+        //  OINKY_ASSERT(ref && (ref <= vcount));
+        //
+        //  This will map bad values (both above and below the range) to zero,
+        //  which we satisfy with the NULL string.
+        uint32 offset = map_into_range((uint32) ref, 1, vcount, 0);
+        if (offset == 0) {
+            //  The only place we get bad refs is from serialized form.  
+            //
+            //  TBD:
+            //  That we get this exception lazily (after mount completes) might
+            //  be confusing.
+            throw_error(bad_encoding());
+        }
+        return from_trusted_ref(ref);
+    }
+        
+    static inline uint32 unpack_offset(const uint32 *src) {
+        uint32 val;
+        Serialization::unpack_nbc<Serialization::v1_sformat_tag, uint32>(&val, (const char *)src);
+        return val;
+    }
+    
+    inline safestr from_trusted_ref(REF_T ref) const { 
+        uint32 index = (uint32) ref - 1;
+        uint32 begin = unpack_offset(offsets + index);
+        uint32 end = (index == vcount-1) ? 
+            vbuf_length :
+            unpack_offset(offsets + index + 1);
+        //  Zero length is permissible.
+        //  TBD:  We checked this earlier, so we can probably skip it here.
+        if ((end < begin) || (end > vbuf_length)) {
+            throw_error(bad_encoding());
+        }
+        safestr ss;
+        ss.newstr = NULL;
+        ss.str = db_string(values + begin, end - begin);
+        ss.str.xref = (uint32) ref;
+        return ss;
+    }
+
+    iterator begin() const { return iterator(this, 0); }
+    iterator end() const { return iterator(this, vcount); }
+    iterator find(const db_string &key) const {
+        return std::find(key, begin(), end());
+    }
+    iterator lower_bound(const db_string &key) const {
+        return std::lower_bound(key, begin(), end());
+    }
+    iterator upper_bound(const db_string &key) const {
+        return std::upper_bound(key, begin(), end());
+    }
+
+public:
+    void reinit(
+        nky_allocator_t *_alloc,
+        uint32 _vcount,
+        const uint32 *_offsets,
+        uint32 _vbuf_length,
+        const char *_vbuf
+        )
+    {
+        this->~stringtable_t();
+        ::new(this) stable_t(_alloc, _vcount, _offsets, _vbuf_length, _vbuf);
+    }
+    
+    //  This should use the DB allocator and a new-string map to 
+    //  ensure that we don't grow the DB heap pointlessly.
+    safestr make_safestring(const db_string &str)
+    {
+        //  This check is fast.
+        if (str.xref && (str.xref < vcount)) {
+            safestr s = from_trusted_ref((REF_T) str.xref);
+            if (s == str) {
+                return s;
+            }
+        }
+        
+        //  If we don't know the location in the fixed table, we will not even 
+        //  search for it there.  We will search/insert in the dynamic table.
+        //
+        //  Q: Why?
+        //
+        //  A: The assumption here is that the fixed stringtable size is MUCH
+        //  larger than the working stringtable size.  We would prefer to
+        //  search the smaller table.  Even though we make the dynamic table
+        //  larger my adding more strings to it than we have to, we presume the
+        //  the resulting table will STILL be much smaller than the fixed table.
+        //  Also, there should be more redundancy among strings inserted in
+        //  the same session.  We can easily strip the duplicates in the
+        //  serialization phase.
+        
+        safestr x;
+        x.str.xref = 0;
+        x.newstr = safe_header_t::prepare(allocator, strmap, str);
+        x.str = x.newstr->as_string();
+        return x;
+    }
+    
+    //###########################################################
+    //      Serialization
+    //
+    void clear_references()
+    {
+        //  Clear the fixed reference counts.
+        offset_xform.resize(vcount);
+        memset(offset_xform.begin(), 0, sizeof(REF_T) * vcount);
+        
+        //  Clear the dynamic reference counts.
+        safe_header_t::clear_reference_counts(strmap);
+    }
+    
+    //  Mark the string (either fixed or new, but not necessarily both) 
+    //  as in-use, so it will be stored when the DB is serialized.
+    void mark(const safestr &str)
+    {
+        mark(str.newstr, (REF_T) str.str.xref);
+    }
+    void mark(const safe_cv_t &cv)
+    {
+        OINKY_ASSERT(cv.is_string());
+        mark(cv.newstr, (REF_T) cv.string_value().xref);
+    }
+
+private:
+    void mark(safe_header_t *header, REF_T ref)
+    {
+        //  This check is fast.
+        if (ref) {
+            OINKY_ASSERT(offset_xform.size() > ref - 1);
+            //  (re-)mark the bin
+            offset_xform[ref - 1] = 1;
+        } else if (header) {
+            //  (re-)mark the header
+            header->_ref = (REF_T) 1;
+        } else {
+            //  This string obviously was not safe.
+            OINKY_ASSERT(false);
+            throw_error(implementation_bug());
+        }
+    }
+    
+    struct compare_strings
+    {
+        int operator()(const iterator &l, const strmap_itr_t &r) const {
+            return l.to_string().compare_to(r->as_string());
+        }
+    };
+    
+    typedef symmetric_iterator<iterator> fixed_range_t;
+    typedef symmetric_iterator<strmap_itr_t> dynamic_range_t;
+    
+    class s_itr : public merge_iterator_template<
+        s_itr,
+        int,
+        fixed_range_t, //index over offsets.
+        dynamic_range_t,
+        compare_strings>
+    {
+        typedef merge_iterator_template<
+            s_itr,
+            int,
+            fixed_range_t, //index over offsets.
+            symmetric_iterator<strmap_itr_t>,
+            compare_strings> base_t;
+        friend class boost::iterator_core_access;
+        
+        int dereference() const { return 0; }
+        
+    public:
+        s_itr(const fixed_range_t &f, const dynamic_range_t &d) :
+        base_t(f, d)
+        {}
+        
+    };
+    
+public:
+    //  This does two things.  
+    //      1) It determines which strings are reachable, and assigns each
+    //          reachable string an index, which corresponds to the string's
+    //          position in the new sorted stringtable.  The first index is 1.
+    //      2) It computes the total number of bytes required to store the new
+    //          stringtable, including the header data.
+    //  This should be called AFTER all database parts depending on the 
+    //  stringtable have been prepared.  That is how we compute reachability.
+    uint32 compute_indexes_and_length()
+    {
+        s_itr i(
+            fixed_range_t(begin(), begin(), end()), 
+            dynamic_range_t(strmap.begin(), strmap.begin(), strmap.end())
+            );
+
+        //  Indexes start at 1.  0 means the string is being dropped.
+        uint32 target_index = 1;
+        uint32 total_string_bytes = 0;
+        while (!i.at_end()) {
+            uint32 this_index = 0;
+            if (i.left_active()) {
+                const iterator &l(i.left().i());
+                int x = l - begin();
+                //  If this is nonzero, then it means the fixed string is still
+                //  referenced.
+                if (offset_xform[x]) {
+                    offset_xform[x] = this_index = target_index;
+                    target_index++;
+                    total_string_bytes += l.to_string().as_string().length();
+                }
+            }
+            if (i.right_active()) {
+                const strmap_itr_t &r(i.right().i());
+                typename safestr::header_t &hdr(*r);
+                if (r->_ref) {
+                    if (!this_index) {
+                        this_index = target_index;
+                        target_index++;
+                        total_string_bytes += r->length();
+                    } else {
+                        //  these are the same string!
+                        OINKY_ASSERT(i.left().i().to_string().as_string().length() == r->length());
+                    }
+                    r->_ref = this_index;
+                }
+            }
+            ++i;
+        }
+
+        //  Total stringtable length is array of offsets plus string bytes
+        //  plus 4-byte offset indicating first-string-offset.
+        uint32 string_count = target_index - 1;
+        uint32 byte_count = 8 + (string_count * 4) + total_string_bytes;
+        
+        //  Save these for later.  We need to know before we start iterating.
+        prepare_packed_string_count = string_count;
+        prepare_packed_stringslength = total_string_bytes;
+        prepare_packed_total_length = byte_count;
+        
+        return byte_count;
+    }
+
+private:
+    inline REF_T xform_ref(safe_header_t *header, REF_T ref) const {
+        if (header) {
+            OINKY_ASSERT(header->_ref);
+            return header->_ref;
+        } else {
+            REF_T newref = offset_xform[ref_to_index(ref)];
+            OINKY_ASSERT(newref);
+            return newref;
+        }
+    }
+public:
+    REF_T xform_ref(const safestr &str) const {
+        return xform_ref(str.newstr, (REF_T) str.str.xref);
+    }
+    REF_T xform_ref(const safe_cv_t &cv) const {
+        return xform_ref(cv.newstr, (REF_T) cv.string_value().xref);
+    }
+    REF_T xform_ref(const REF_T &oldref) const {
+        return xform_ref(NULL, oldref);
+    }
+    
+    void mark_ref(const REF_T &oldref) {
+        offset_xform[ref_to_index(oldref)] = 1;
+    }
+    
+    uint32 ref_to_index(REF_T oldref) const {
+        OINKY_ASSERT(oldref);
+        OINKY_ASSERT((uint32) oldref <= offset_xform.size());
+        //  In case asserts aren't active, we want to map into range.
+        return map_into_range((uint32) oldref, 1, offset_xform.size(), 1)-1;
+    }
+    
+    template<typename SCHEME>
+    void pack(char *&buf, const char *end_buf)
+    {
+        char *table_base = buf;
+        //  This is the only bounds check we do.  As long as the packing
+        //  protocol is being followed correctly (clear/count/pack) then
+        //  this will work out.  We ASSERT as much.  But we don't presume
+        //  a caller is malicious, since they could do worse much more easily.
+        if (buf + prepare_packed_total_length > end_buf) {
+            throw_error(buffer_overflow());
+        }
+        
+        Serialization::pack<SCHEME, uint32>(prepare_packed_total_length, buf, end_buf);
+        Serialization::pack<SCHEME, uint32>(prepare_packed_string_count, buf, end_buf);
+        
+        char *offset_buf = buf;
+        char *strings_buf = buf + (4 * prepare_packed_string_count);
+        char *strings_base = strings_buf;
+        
+        s_itr i(
+            fixed_range_t(begin(), begin(), end()), 
+            dynamic_range_t(strmap.begin(), strmap.begin(), strmap.end())
+            );
+
+        //  Indexes start at 1.  0 means the string is being dropped.
+        while (!i.at_end()) {
+            bool saved = false;
+            if (i.left_active()) {
+                const iterator &l(i.left().i());
+                //  We only write if it's referenced.
+                if (offset_xform[l - begin()]) {
+                    //  Pack offset
+                    Serialization::pack<SCHEME, uint32>(strings_buf - table_base, offset_buf, end_buf);
+                    //  Pack bytes
+                    const db_string &str(l.to_string().as_string());
+                    memcpy(strings_buf, str.begin(), str.length());
+                    strings_buf += str.length();
+                    saved = true;
+                }
+            }
+            //  We only need to pack once.
+            if (!saved && i.right_active()) {
+                const strmap_itr_t &r(i.right().i());
+                //  Again..
+                if (r->_ref) {
+                    //  Pack offset
+                    Serialization::pack<SCHEME, uint32>(strings_buf - table_base, offset_buf, end_buf);
+                    //  Pack bytes
+                    const db_string &str(r->as_string());
+                    memcpy(strings_buf, str.begin(), str.length());
+                    strings_buf += str.length();
+                    saved = true;
+                }
+            }
+            ++i;
+        }
+        OINKY_ASSERT(table_base + prepare_packed_total_length == strings_buf);
+        OINKY_ASSERT(offset_buf == strings_base);
+        OINKY_ASSERT(strings_base + prepare_packed_stringslength == strings_buf);
+        buf = strings_buf;
+    }
+};
+
+typedef stringtable_t<m_strtable_ref> meta_stringtable_t;
+typedef stringtable_t<u_strtable_ref> user_stringtable_t;
+
+} //namespace Internal
+} //namespace Oinky
+