ruby_odeum 0.2.1

data/ext/odeum_index/odeum_index.c

@@ -0,0 +1,991 @@
+/*************************************************************************************************
+ * Implementation of Curia for Ruby
+ *                                                      Copyright (C) 2000-2005 Mikio Hirabayashi
+ * This file is part of QDBM, Quick Database Manager.
+ * QDBM is free software; you can redistribute it and/or modify it under the terms of the GNU
+ * Lesser General Public License as published by the Free Software Foundation; either version
+ * 2.1 of the License or any later version.  QDBM is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+ * details.
+ * You should have received a copy of the GNU Lesser General Public License along with QDBM; if
+ * not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ * 02111-1307 USA.
+ *************************************************************************************************/
+
+
+#include "ruby.h"
+#include <odeum.h>
+
+#define FALSE 0
+
+static VALUE mOdeum;
+static VALUE cIndex;
+static VALUE cDocument;
+static VALUE cPair;
+
+
+/* A union used to do nothing more than wrap the ODEUM, ODDOC, and ODPAIR structures
+ * so that we can get at them in an alloc method before they need to be initialized.
+ */
+
+typedef union OdeumWrapper {
+    ODEUM *odeum;
+    ODDOC *oddoc;
+} OdeumWrapper;
+
+/** Convenience macors to get at the different types we store in T_DATA stuff. */
+#define RAISE_NOT_NULL(T) if(T == NULL) rb_raise(rb_eStandardError, "NULL found for " # T " when shouldn't be.");
+#define DATA_GET_WRAPPER(from,name) OdeumWrapper *name = NULL; Data_Get_Struct(from, OdeumWrapper, name); RAISE_NOT_NULL(name);
+#define DATA_GET_ODEUM(from,name) DATA_GET_WRAPPER(from,name##wrapper); RAISE_NOT_NULL(name##wrapper); ODEUM *name = name##wrapper->odeum; RAISE_NOT_NULL(name);
+#define DATA_GET_ODDOC(from,name) DATA_GET_WRAPPER(from,name##wrapper); RAISE_NOT_NULL(name##wrapper); ODDOC *name = name##wrapper->oddoc; RAISE_NOT_NULL(name);
+
+#define REQUIRE_TYPE(V, T) if(TYPE(V) != T) rb_raise(rb_eTypeError, "Wrong argument type for " # V " required " # T);
+
+
+/* Converts from a CBLIST of char * strings to a Ruby Array of Strings. */
+VALUE CBLIST_2_array(const CBLIST *list) 
+{
+    int count = cblistnum(list);
+    int i = 0;
+    VALUE ary = rb_ary_new();
+    
+    for(i = 0; i < count;  i++) {
+        int sp = 0;
+        const char *val = cblistval(list, i, &sp);
+        rb_ary_push(ary, rb_str_new(val, sp));
+    }
+    
+    return ary;
+}
+
+/** Converts an array of strings to a CBLIST. */
+CBLIST *array_2_CBLIST(VALUE ary) 
+{
+    long i = 0;
+    CBLIST *result = cblistopen();
+    VALUE str;
+    
+    for(i = 0; (str = rb_ary_entry(ary, i)) != Qnil; i++) {
+        cblistpush(result, RSTRING(str)->ptr, RSTRING(str)->len);
+    }
+    
+    return result;
+}
+
+/** Converts a CBMAP of char * strings into a Ruby Hash of Strings. */
+VALUE CBMAP_2_hash(CBMAP *map) 
+{
+    int key_size = 0;
+    int val_size = 0;
+    const char *map_key = NULL;
+    const char *map_val = NULL;
+    VALUE hash = rb_hash_new();
+    VALUE key;
+    VALUE val;
+    
+    cbmapiterinit(map);
+    while((map_key = cbmapiternext(map, &key_size)) != NULL) {
+        map_val = cbmapget(map, map_key, key_size, &val_size);
+        
+        key = rb_str_new(map_key, key_size);
+        val = rb_str_new(map_val, val_size);
+        
+        rb_hash_aset(hash, key, val);
+    }
+    
+    return hash;
+}
+
+/** Builds a new document from a created ODDOC. This is needed since the Index_* functions
+ * will return an ODDOC pointer, but the only function to construct a Document normally is
+ * with oddocopen which requires a URI.  This solves the problem by using the "naked"
+ * Document_initialize, and then attaches the doc to it.
+ */
+VALUE Document_from_ODDOC(ODDOC *doc) 
+{
+    VALUE uri[1]; 
+    VALUE new_doc;
+    
+    uri[0] = Qnil;
+    new_doc = rb_class_new_instance(1, uri, cDocument);
+    
+    DATA_GET_WRAPPER(new_doc, wrapper);    
+    wrapper->oddoc = doc;
+    
+    return new_doc;
+}
+
+
+/** 
+ * Frees the ODEUM struct contained in the wrapper, closing it if it is not already
+ * NULL.  This is the best we can do for automatically closing the ODEUM if it wasn't
+ * done explicitly with Odeum.close.
+ */
+void Index_free(void *ptr)
+{
+    OdeumWrapper *wrapper = (OdeumWrapper *)ptr;
+    if(wrapper->odeum) {
+        // didn't explicitly close, so do it for them
+        odclose(wrapper->odeum);
+        wrapper->odeum = NULL;
+    }
+    // now safe to free it
+    free(wrapper);
+}
+
+
+/** Allocates only the OdeumWrapper we use to wrap the pointers we need for internal
+ * operation.  This is needed because we can't create the ODEUM struct until we know
+ * the name of the catalog, so we have to "defer" until Odeum.initialize.
+ */
+VALUE Index_alloc(VALUE klass) 
+{
+    OdeumWrapper *wrapper = NULL;
+    VALUE obj = Data_Make_Struct(klass, OdeumWrapper, NULL, Index_free, wrapper);
+    return obj;
+}
+
+
+/**
+ * call-seq:
+ *   Index.new(name, mode) -> Index
+ * 
+ * Creates an Index with the given name according to mode.  The name will be used
+ * as the basis for a local directory which will contain the database for the documents.
+ * 
+ * Possible modes might be:
+ * 
+ * - Odeum::OWRITER -- Opens as a writer.
+ * - Odeum::OREADER -- Read-only.
+ * - Odeum::OCREAT -- Or'd in to OWRITER to indicate that you want it created if not existing.
+ * - Odeum::ONOLOCK -- Opens without locking on the directory.
+ *
+ * Opening as OWRITER creates an exclusive lock on the database dir, but OREADER
+ * opens with a shared lock.  A thread will block until the lock is achieved, but
+ * none of this has been tested in Ruby with Ruby's in-process threads.
+ */
+VALUE Index_initialize(VALUE self, VALUE name, VALUE mode) 
+{
+    DATA_GET_WRAPPER(self, wrapper);
+    REQUIRE_TYPE(name, T_STRING);
+    REQUIRE_TYPE(mode, T_FIXNUM);
+    
+    wrapper->odeum = odopen(RSTRING(name)->ptr, FIX2INT(mode));
+    if(wrapper->odeum == NULL) {
+        // there was an error, find out what it was
+        rb_raise(rb_eStandardError, "Failed to open requested database.");
+    }
+    
+    return self;
+}
+
+/**
+ * call-seq:
+ *    Index.close -> true/false
+ *
+ * Closes the Index explicitly.  It will be closed by the GC when Index_free
+ * is finally called, but don't rely on this as it is not reliable enough.
+ * Don't use the index after this, it will throw an exception or possibly crash.
+ */
+VALUE Index_close(VALUE self) 
+{
+    int result = 0;
+    DATA_GET_WRAPPER(self, wrapper);
+    
+    // must set the wrapper->odeum to NULL so that Index_free does not try to close it again
+    result = odclose(wrapper->odeum);
+    wrapper->odeum = NULL;
+    
+    return result == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    index.put(doc, wmax, over) -> true/false
+ *
+ * Puts the Document doc into the Index, and indexes a maximum of wmax
+ * words in the document.  If over is true than the document is overwritten
+ * in the database.  Otherwise, if the document already exists in the 
+ * database and over== nil/false then the method will return false as
+ * an error.
+ */
+VALUE Index_put(VALUE self, VALUE doc, VALUE wmax, VALUE over) 
+{
+    int res = 0;
+    DATA_GET_ODEUM(self, odeum);
+    DATA_GET_ODDOC(doc, oddoc);  
+    
+    REQUIRE_TYPE(wmax, T_FIXNUM);
+    
+    res = odput(odeum, oddoc, FIX2INT(wmax), !(over == Qnil || over == Qfalse));
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *   index.delete(uri) -> true/false
+ *
+ * Deletes the document given by the uri.  The Index must be opened
+ * as a writer, and the call will return false if no such document exists.
+ */
+VALUE Index_delete(VALUE self, VALUE uri) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(uri, T_STRING);
+    
+    int res = odout(odeum, RSTRING(uri)->ptr);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *   index.delete_by_id(id) -> true/false
+ *
+ * Deletes a document based on its id.
+ */
+VALUE Index_delete_by_id(VALUE self, VALUE id) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(id, T_FIXNUM);
+    
+    int res = odoutbyid(odeum, FIX2INT(id));
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *   index.get(uri) -> Document
+ *
+ * Gets a Document based on the uri, or returns nil.
+ */
+VALUE Index_get(VALUE self, VALUE uri) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(uri, T_STRING);
+    
+    ODDOC *oddoc = odget(odeum, RSTRING(uri)->ptr);
+    if(oddoc == NULL)
+        return Qnil;
+    else
+        return Document_from_ODDOC(oddoc);
+}
+
+
+/**
+ * call-seq:
+ *   index.get_by_id(id) -> Document
+ *
+ * Gets a Document based on its id, or nil if that document isn't there.
+ */
+VALUE Index_get_by_id(VALUE self, VALUE id) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(id, T_FIXNUM);
+    
+    ODDOC *oddoc = odgetbyid(odeum, FIX2INT(id));
+
+    if(oddoc == NULL)
+        return Qnil;
+    else
+        return Document_from_ODDOC(oddoc);
+}
+
+
+/**
+ * call-seq:
+ *   index.get_id_by_uri(id)
+ *
+ * Returns just the id of the document with the given uri.
+ */
+VALUE Index_get_id_by_uri(VALUE self, VALUE uri) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(uri, T_STRING);
+    
+    int res = odgetidbyuri(odeum, RSTRING(uri)->ptr);
+    return INT2FIX(res);
+}
+
+
+/**
+ * call-seq:
+ *   index.check(id)
+ *
+ * Checks if a document with the given id is in the database.
+ */
+VALUE Index_check(VALUE self, VALUE id) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(id, T_FIXNUM);
+    
+    int res = odcheck(odeum, FIX2INT(id));
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    index.search(word, max) -> [[id,score], ... ]
+ *
+ * The big payoff method which actually searches for the documents
+ * that have the given word mentioned.  The result of the search is
+ * an array of "tuples" with [id, score] and sorted in score order
+ * from descending order of their scores.  If there are no results
+ * then it returns an empty array.
+ *
+ * If the search attempt fails for some reason then an exception is thrown,
+ * but an empty result is NOT a failure (that returns an empty hash).
+ */
+VALUE Index_search(VALUE self, VALUE word, VALUE max) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(word, T_STRING);
+    REQUIRE_TYPE(max, T_FIXNUM);
+    
+    int num_returned = 0;
+    int i = 0;
+    VALUE results;
+    
+    ODPAIR *pairs = odsearch(odeum, RSTRING(word)->ptr, FIX2INT(max), &num_returned);
+    if(pairs == NULL) {
+        // nothing found
+        rb_raise(rb_eStandardError, "Search failure.");
+    }
+    
+    results = rb_ary_new();
+    
+    // convert the array of ODPAIR structs into an array of result tuples
+    for(i = 0; i < num_returned; i++) {
+        VALUE ary = rb_ary_new();
+        rb_ary_push(ary, INT2FIX(pairs[i].id));
+        rb_ary_push(ary, INT2FIX(pairs[i].score));
+        rb_ary_push(results, ary);
+    }
+    
+    free(pairs);
+    return results;
+}
+
+/**
+ * call-seq:
+ *    index.search_doc_count(word) -> Fixnum
+ *
+ * Returns the number of document matching the given word.  If the word
+ * does not match anything then it returns -1.
+ */
+VALUE Index_search_doc_count(VALUE self, VALUE word) {
+    DATA_GET_ODEUM(self, odeum);
+    REQUIRE_TYPE(word, T_STRING);
+    
+    int res = odsearchdnum(odeum, RSTRING(word)->ptr);
+    return INT2FIX(res);
+}
+
+
+/**
+ * call-seq:
+ *    index.iterator -> true/false
+ *
+ * Begins an iterator loop to process documents in the system.
+ * An iterator/next pattern is used due to the difficulty of getting
+ * memory collection correct inside an each/block design.
+ */
+VALUE Index_iterator(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = oditerinit(odeum);
+    
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+/**
+ * call-seq:
+ *   index.next -> Document
+ *
+ * Returns the next document or nil if there was an error.  Must call
+ * Index.iterator first.
+ */
+VALUE Index_next(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    
+    ODDOC *doc = oditernext(odeum);
+    if(doc == NULL)
+        return Qnil;
+    
+    VALUE doc_obj = Document_from_ODDOC(doc);
+    
+    return doc_obj;
+}
+
+/**
+ * call-seq:
+ *   index.sync -> true/false
+ *
+ * Synchronizes any changes you have made with the database.  If you
+ * don't do this every once in a while then the memory load will get
+ * to great.  I found that every 1000 documents or so is a good trade-off.
+ *
+ * Returns true if everything worked, or false otherwise.
+ */
+VALUE Index_sync(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odsync(odeum);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *   index.optimize -> true/false
+ *
+ * Purges deleted documents from the index.  I found that if you 
+ * call this while you are updating documents then it stops adding
+ * documents after the optimize call.
+ */
+VALUE Index_optimize(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odoptimize(odeum);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    index.name -> String
+ */
+VALUE Index_name(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    char *name = odname(odeum);
+    VALUE result = rb_str_new2(name);
+    free(name);
+    return result;
+}
+
+/**
+ * call-seq:
+ *   index.size -> Fixnum
+ *
+ * Returns the size of the database files or -1 if there's a failure.
+ */
+VALUE Index_size(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    double res = odfsiz(odeum);
+    return rb_float_new(res);
+}
+
+
+/**
+ * call-seq:
+ *   index.bucket_count -> Fixnum
+ *
+ * Returns the total number of elements of the bucket arrays, or -1 on failure.
+ */
+VALUE Index_bucket_count(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odbnum(odeum);
+    return INT2FIX(res);
+}
+
+/**
+ * call-seq:
+ *   index.buckets_used -> Fixnum
+ *
+ * The total number of used elements of the bucket arrays, or -1 if failure.
+ */
+VALUE Index_buckets_used(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odbusenum(odeum);
+    return INT2FIX(res);
+}
+
+
+/**
+ * call-seq:
+ *     index.doc_count -> Fixnum
+ *
+ * Number of documents stored in the database, or -1 on failure.
+ */
+VALUE Index_doc_count(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = oddnum(odeum);
+    return INT2FIX(res);
+}
+
+
+/**
+ * call-seq:
+ *   index.word_count -> Fixnum
+ */
+VALUE Index_wnum(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odwnum(odeum);
+    return INT2FIX(res);
+}
+
+
+/**
+ * call-seq:
+ *   index.writable -> true/false
+ */
+VALUE Index_writable(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int res = odwritable(odeum);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    index.fatal_error -> Fixnum
+ *
+ * Returns true if there's a fatal error or false otherwise.
+ */
+VALUE Index_fatal_error(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int err = odfatalerror(odeum);
+    return err == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    index.inode -> Fixnum
+ *
+ * The inode number of the database directory.
+ */
+VALUE Index_inode(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int inode = odinode(odeum);
+    return INT2FIX(inode);
+}
+
+
+/**
+ * call-seq:
+ *   index.mtime -> Fixnum
+ * 
+ * The mtime of the database directory.
+ */
+VALUE Index_mtime(VALUE self) {
+    DATA_GET_ODEUM(self, odeum);
+    int mtime = odmtime(odeum);
+    return INT2FIX(mtime);
+}
+
+
+/**
+ * call-seq:
+ *    Odeum::merge(new_name, other_databases) -> true/false
+ *
+ * Merges the databases listed in other_databases (Array of Strings)
+ * into the new database new_name.
+ * If two or more documents have the same URI then the first one is
+ * adopted and the others are ignored.
+ */
+VALUE Odeum_merge(VALUE self, VALUE name, VALUE elemnames) {
+    REQUIRE_TYPE(name, T_STRING);
+    REQUIRE_TYPE(elemnames, T_ARRAY);
+    
+    CBLIST *elems = array_2_CBLIST(elemnames);
+    int res = odmerge(RSTRING(name)->ptr, elems);
+    cblistclose(elems);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *   Odeum::remove(name) -> true/false
+ * 
+ * Removes the database directory and everything in it.
+ */
+VALUE Odeum_remove(VALUE self, VALUE name) {
+    REQUIRE_TYPE(name, T_STRING);
+    
+    int res = odremove(RSTRING(name)->ptr);
+    return res == FALSE ? Qfalse : Qtrue;
+}
+
+
+/**
+ * call-seq:
+ *    Odeum::breaktext(test) -> [word1, word2, word3]
+ *
+ * Breaks a string into an array of words that are separated by
+ * space characters and such delimiters as period, commaa, etc.
+ * You should also check out StringScanner as a more flexible
+ * alternative.  This function must do a lot of data copying and
+ * other things in order to convert from Odeum internal types to Ruby
+ * types.
+ */
+VALUE Odeum_breaktext(VALUE self, VALUE text) {
+    REQUIRE_TYPE(text, T_STRING);
+    
+    CBLIST *result = odbreaktext(RSTRING(text)->ptr);
+    VALUE list = CBLIST_2_array(result);
+    cblistclose(result);
+    return list;
+}
+
+
+/**
+ * call-seq:
+ *   Odeum::normalizeword(asis) -> normal
+ *
+ * Given a word from breaktext (which is considered "as-is") 
+ * it will "normalize" it in a consistent way which is suitable
+ * for searching.  The normalization effectively strips puntuation
+ * and spacing, and then lowercases the word.  If there is nothing
+ * but "removed" chars in the asis string then the return is empty.
+ * Check for this so you don't try to search for nothing.
+ */
+VALUE Odeum_normalizeword(VALUE self, VALUE asis) {
+    REQUIRE_TYPE(asis, T_STRING);
+    
+    char *result = odnormalizeword(RSTRING(asis)->ptr);
+	VALUE res_str = rb_str_new2(result);
+	free(result);
+    return res_str;
+}
+
+
+/**
+ * call-seq:
+ *   Odeum::settuning(ibnum, idnum, cbnum, csiz) -> nil
+ *
+ * ibnum=32749: Number of buckets for inverted indexes. 
+ * idnum=7: Division number of inverted index. 
+ * cbnum=262139:  Number of buckets for dirty buffers. 
+ * csiz=8388608: Maximum bytes to use memory for dirty buffers.
+ *
+ * This is set globally for all Indexes.  Not sure what would happen
+ * if you changed this mid-stream, so don't.  Make sure everything is closed.
+ */
+VALUE Odeum_settuning(VALUE self, VALUE ibnum, VALUE idnum, VALUE cbnum, VALUE csiz) {
+    REQUIRE_TYPE(ibnum, T_FIXNUM);
+    REQUIRE_TYPE(idnum, T_FIXNUM);
+    REQUIRE_TYPE(cbnum, T_FIXNUM);
+    REQUIRE_TYPE(csiz, T_FIXNUM);
+    
+    odsettuning(FIX2INT(ibnum), FIX2INT(idnum), FIX2INT(cbnum), FIX2INT(csiz));
+    return Qnil;
+}
+
+
+
+/**
+ * Frees the internal wrapper and properly cleans up the ODDOC.
+ */
+void Document_free(void *ptr) {
+    OdeumWrapper *wrapper = (OdeumWrapper *)ptr;
+    
+    if(wrapper->oddoc) {
+        // didn't explicitly close, so do it for them
+        oddocclose(wrapper->oddoc);
+        wrapper->oddoc = NULL;
+    }
+    // now safe to free it
+    free(wrapper);
+}
+
+
+/**
+ * Allocates the wrapper only, leaving the actual allocation for Document_initialize.
+ */
+VALUE Document_alloc(VALUE klass) {
+    OdeumWrapper *wrapper = NULL;
+    VALUE obj = Data_Make_Struct(klass, OdeumWrapper, NULL, Document_free, wrapper);
+    return obj;
+}
+
+/**
+ * call-seq:
+ *   Document.new uri -> Document
+ *
+ * The uri should be specified if you're calling this.  Internally the
+ * Ruby/Odeum library kind of "cheats" and passes a Qnil for the uri 
+ * so that the ODDOC can be assigned externally.  You should not
+ * (and probably cannot) do this from Ruby.
+ */
+VALUE Document_initialize(VALUE self, VALUE uri) {
+    DATA_GET_WRAPPER(self, wrapper);
+    
+    if(NIL_P(uri)) {
+        wrapper->oddoc = NULL;
+    } else {
+        REQUIRE_TYPE(uri, T_STRING);
+        wrapper->oddoc = oddocopen(RSTRING(uri)->ptr);
+    }
+    
+    return self;
+}
+
+
+/**
+ * call-seq:
+ *   doc.close -> nil
+ *
+ * Explicitly closes a document.  Because of what I can only decide is a bug
+ * in how an each iterator works, you must explicitly close a document
+ * if you are not storing it and you are in an each.  There are probably
+ * subtle things about Ruby memory management I'm missing, but my tests 
+ * show that all Document objects created with Index.get do not get 
+ * garbage collected until they exit a block.
+ */
+VALUE Document_close(VALUE self) {
+    DATA_GET_WRAPPER(self, wrapper);
+    oddocclose(wrapper->oddoc);
+    wrapper->oddoc = NULL;  // must set to null to prevent double free
+    return Qnil;
+}
+    
+
+/**
+ * call-seq:
+ *   doc[attr] = value
+ *
+ * Adds meta-data to the document.  They should be Strings only.
+ */
+VALUE Document_addattr(VALUE self, VALUE name, VALUE value) {    
+    DATA_GET_ODDOC(self, doc);
+    REQUIRE_TYPE(name, T_STRING);
+    REQUIRE_TYPE(value, T_STRING);
+    
+    oddocaddattr(doc, RSTRING(name)->ptr, RSTRING(value)->ptr);
+    return self;
+}
+
+
+/**
+ * call-seq:
+ *   document.add_content(content) -> document
+ *
+ * Takes the contents, breaks the words up, and then puts them in the document
+ * in normalized form.  This is the common pattern that people use a Document
+ * with.  You may also use Document.addword to add one word a time, and
+ * Document.add_word_list to add a list of words.
+ *
+ * It uses the default Odeum::breaktext method to break up the text,
+ * so don't use this if you need specialized stuffs.
+ *
+ * If the process of normalizing a word creates an empty word, then it
+ * is not added to the document's words.  This usually happens for
+ * punctation that isn't usualy searched for anyway.
+ */
+VALUE Document_add_content(VALUE self, VALUE content) {
+    DATA_GET_ODDOC(self, doc);
+    REQUIRE_TYPE(content, T_STRING);
+    
+    CBLIST *words = odbreaktext(RSTRING(content)->ptr);
+    
+    // go through words and add them
+    int count = cblistnum(words);
+    int i = 0;
+    
+    for(i = 0; i < count;  i++) {
+        int sp = 0;
+        const char *asis = cblistval(words, i, &sp);
+        char *norm = odnormalizeword(asis);
+        // only add words that normalize to some content
+        if(strlen(norm) > 0) {
+            oddocaddword(doc, norm, asis);
+        }
+        free(norm);
+    }
+    
+    cblistclose(words);
+    
+    return self;
+}
+
+/**
+ * call-seq:
+ *   document.add_word_list(asis) -> document
+ *
+ * Takes an array of "as-is" words, normalizes them, and puts them in the document.
+ * It assumes that the array is composed of asis words and normalizes them
+ * before putting them in the document.
+ */
+VALUE Document_add_word_list(VALUE self, VALUE asis) {
+    DATA_GET_ODDOC(self, doc);
+    REQUIRE_TYPE(asis, T_ARRAY);
+    VALUE str;
+    int i = 0;
+    
+    for(i = 0; (str = rb_ary_entry(asis, i)) != Qnil; i++) {
+        char *result = odnormalizeword(RSTRING(str)->ptr);
+        oddocaddword(doc, result, RSTRING(str)->ptr);
+        free(result);
+    }
+    
+    return self;
+}
+
+
+/**
+ * call-seq:
+ *   document.addword(normal, asis)
+ *
+ * The basic call to add a normal and asis version of a word to the 
+ * document for indexing.
+ */
+VALUE Document_addword(VALUE self, VALUE normal, VALUE asis) {
+    DATA_GET_ODDOC(self, doc);
+    REQUIRE_TYPE(normal, T_STRING);
+    REQUIRE_TYPE(asis, T_STRING);
+    
+    oddocaddword(doc, RSTRING(normal)->ptr, RSTRING(asis)->ptr);
+    return self;
+}
+
+
+/**
+ * call-seq:
+ *   document.id -> Fixnum
+ *
+ * Gives you the Odeum::Index id used to for the document.
+ */
+VALUE Document_id(VALUE self) {
+    DATA_GET_ODDOC(self, doc);
+    int id = oddocid(doc);
+    return INT2FIX(id);
+}
+
+
+/**
+ * call-seq:
+ *   document.uri -> String
+ *
+ * Gets the uri that this document represents.
+ */
+VALUE Document_uri(VALUE self) {
+    DATA_GET_ODDOC(self, doc);
+    VALUE uri = rb_str_new2(oddocuri(doc));
+    return uri;
+}
+
+
+/**
+ * call-seq:
+ *   document[name] -> String
+ *
+ * Gets the meta-data attribute for the given name.  The name must
+ * be a String.
+ */
+VALUE Document_getattr(VALUE self, VALUE name) {
+    DATA_GET_ODDOC(self, doc);
+    REQUIRE_TYPE(name, T_STRING);
+    
+    VALUE attr = rb_str_new2(oddocgetattr(doc, RSTRING(name)->ptr));
+    return attr;
+}
+
+
+/**
+ * call-seq:
+ *    document.normal_words -> [word1, word2, ... ]
+ *
+ * Returns the list of "normal" words in this document.
+ */
+VALUE Document_normal_words(VALUE self) {
+    DATA_GET_ODDOC(self, doc);
+    const CBLIST *list = oddocnwords(doc);
+    return CBLIST_2_array(list);
+}
+
+
+/**
+ * call-seq:
+ *   document.asis_words -> [word1, word2, ...]
+ *
+ * Returns all of the asis or "appearance form" words in the document.
+ */
+VALUE Document_asis_words(VALUE self) {
+    DATA_GET_ODDOC(self, doc);
+    const CBLIST *list = oddocawords(doc);
+    return CBLIST_2_array(list);
+}
+
+
+/**
+ * call-seq:
+ *   document.scores(max, index) -> { word => score, word => score, ...}
+ *
+ * Get the normalized words and their scores in the document.  The
+ * strange thing is that the scores are returned as Strings, but they
+ * are decimal strings.
+ */
+VALUE Document_scores(VALUE self, VALUE max, VALUE odeum_obj) {
+    DATA_GET_ODDOC(self, doc);
+    DATA_GET_ODEUM(odeum_obj, odeum);
+    REQUIRE_TYPE(max, T_FIXNUM);
+    
+    CBMAP *scores = oddocscores(doc, FIX2INT(max), odeum);
+    VALUE map = CBMAP_2_hash(scores);
+    cbmapclose(scores);
+    return map;
+}
+
+
+
+Init_odeum_index(){
+    // Module Odeum
+    mOdeum = rb_define_module("Odeum");
+    
+    // module Odeum functions
+    rb_define_singleton_method(mOdeum, "merge", Odeum_merge, 2);
+    rb_define_singleton_method(mOdeum, "remove", Odeum_remove, 1);
+    rb_define_singleton_method(mOdeum, "breaktext", Odeum_breaktext, 1);
+    rb_define_singleton_method(mOdeum, "normalizeword", Odeum_normalizeword, 1);
+    rb_define_singleton_method(mOdeum, "settuning", Odeum_settuning, 4);
+    
+    rb_define_const(mOdeum, "OREADER", INT2FIX(OD_OREADER));
+    rb_define_const(mOdeum, "OWRITER", INT2FIX(OD_OWRITER));
+    rb_define_const(mOdeum, "OCREAT", INT2FIX(OD_OCREAT));
+    rb_define_const(mOdeum, "OTRUNC", INT2FIX(OD_OTRUNC));
+    rb_define_const(mOdeum, "ONOLCK", INT2FIX(OD_ONOLCK));
+    
+    cIndex = rb_define_class_under(mOdeum, "Index", rb_cObject);
+    
+    rb_define_alloc_func(cIndex, Index_alloc);
+    rb_define_method(cIndex, "initialize",Index_initialize, 2);
+    rb_define_method(cIndex, "close",Index_close, 0);
+    rb_define_method(cIndex, "put",Index_put, 3);
+    rb_define_method(cIndex, "delete",Index_delete, 1);
+    rb_define_method(cIndex, "delete_by_id",Index_delete_by_id, 1);
+    rb_define_method(cIndex, "get",Index_get, 1);
+    rb_define_method(cIndex, "get_by_id",Index_get_by_id, 1);
+    rb_define_method(cIndex, "get_id_by_uri",Index_get_id_by_uri, 1);
+    rb_define_method(cIndex, "check",Index_check, 1);
+    rb_define_method(cIndex, "search",Index_search, 2);
+    rb_define_method(cIndex, "search_doc_count",Index_search_doc_count, 1);
+    rb_define_method(cIndex, "iterator",Index_iterator, 0);
+    rb_define_method(cIndex, "next",Index_next, 0);
+    rb_define_method(cIndex, "sync",Index_sync, 0);
+    rb_define_method(cIndex, "optimize",Index_optimize, 0);
+    rb_define_method(cIndex, "name",Index_name, 0);
+    rb_define_method(cIndex, "size",Index_size, 0);
+    rb_define_method(cIndex, "bucket_count",Index_bucket_count, 0);
+    rb_define_method(cIndex, "buckets_used",Index_buckets_used, 0);
+    rb_define_method(cIndex, "doc_count",Index_doc_count, 0);
+    rb_define_method(cIndex, "word_count",Index_wnum, 0);
+    rb_define_method(cIndex, "writable",Index_writable, 0);
+    rb_define_method(cIndex, "fatal_error",Index_fatal_error, 0);
+    rb_define_method(cIndex, "inode",Index_inode, 0);
+    rb_define_method(cIndex, "mtime",Index_mtime, 0);
+    
+    // define class Document
+    cDocument = rb_define_class_under(mOdeum, "Document", rb_cObject);
+    rb_define_alloc_func(cDocument, Document_alloc);
+    
+    rb_define_method(cDocument, "initialize",Document_initialize, 1);
+    rb_define_method(cDocument, "[]=",Document_addattr, 2);
+    rb_define_method(cDocument, "[]",Document_getattr, 1);
+    
+    rb_define_method(cDocument, "addword",Document_addword, 2);
+    rb_define_method(cDocument, "add_word_list",Document_add_word_list, 1);
+    rb_define_method(cDocument, "add_content",Document_add_content, 1);
+    rb_define_method(cDocument, "id",Document_id, 0);
+    rb_define_method(cDocument, "uri",Document_uri, 0);
+    rb_define_method(cDocument, "normal_words",Document_normal_words, 0);
+    rb_define_method(cDocument, "asis_words",Document_asis_words, 0);
+    rb_define_method(cDocument, "scores",Document_scores, 2);
+    rb_define_method(cDocument, "close", Document_close, 0);
+}