ruby_odeum 0.2.1

data/README

@@ -0,0 +1,50 @@
+== Ruby/Odeum
+
+Ruby/Odeum is a simple full text reverse indexer that lets you index a set of
+files and then search through them very quickly.  It is similar to Java's Lucene
+but at a lower level since it does not provide a query language or a lexical parser
+for documents.  The extension is based on Mikio Hirabayashi's QDBM library and 
+includes a full distribution needed to use the extension right out of the box.
+
+The library is very simple to use and has full documentation in ruby docs.
+Take a look at the test/test_odeum.rb and the bin/odeum_mgr files for examples
+of using the library in a simple way.
+
+== Features
+
+Pretty much the same features that you'd get from Odeum and QDBM, but available
+in an idiomatic Ruby package.  The big list is:
+
+1.  Fast as hell.  QDBM is one of the fastest libraries out there for this
+    kind of thing.
+2.  Simple interface involving two classes and maybe one small set of module functions.
+3.  Indexes documents of any type, with arbitrary names and unlimited (well, sort of)
+    meta-data.
+4.  Searching by normalized words with returned "scores" for weighting.
+5.  Locking at the thread level for the OS.
+
+Mikio states that it is probably not suitable for document stores that are larger
+than about 1 million documents in size.
+
+== Building
+
+Developers who want to work on the project should put Ruby files in lib and
+modifications to the extension in ext/odeum_index.  You should then use the
+Rakefile and rake to build the application and run tests (build docs, etc.)
+
+== Installing
+
+There's a setup.rb you can use to compile and install the extension and odeum_mgr
+script for regular users.
+
+Anyone feel like making a gem out of this?  :-)
+
+== Contact
+
+The Ruby extension is entirely my fault.  Please do not contact Mikio about it
+unless it's to say thanks for doing such a cool job on QDBM and Odeum.  If you 
+have problems with the extension then let me know.  I'll work with Mikio or 
+fix them myself depending on what needs fixing.
+
+You can contact me at zedshaw at zedshaw dot com.
+

data/bin/odeum_mgr

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require 'odeum_index'
+require 'find'
+
+
+# Does a higher level job of managing an inverted index of documents.
+class Manager
+    attr_reader :name
+    include Odeum
+    
+    def initialize(name)
+        @name = name
+    end
+    
+    def setup_new_doc(file)
+        doc = Document.new file
+        contents = File.read(file)
+                        
+        doc.add_content(contents)
+        doc["Date"] = File.mtime(file).to_s
+        
+        return doc
+    end
+    
+    def load(dir)
+        odeum = Index.new name, OWRITER | OCREAT
+        
+        i = 0
+        Dir.chdir(dir) do
+            Find.find("./") do |file|
+                if File.file? file
+                    print "file: #{file}"
+                    
+                    doc = odeum.get(file)
+                    if not odeum.get(file) or doc["Date"] != File.mtime(file).to_s
+                        doc = setup_new_doc(file)
+                        odeum.put(doc, -1, 1)
+                        doc.close
+                        puts "...Added."
+                        
+                        if i % 1000 == 0
+                            odeum.sync
+                        end
+                        
+                        i += 1
+                    else
+                        puts "...skipped."
+                    end        
+                end
+            end
+        end
+        
+        odeum.close
+    end
+    
+    
+    def search(words)
+        odeum = Index.new name, OREADER
+        
+        words.each do |word|
+            docs = odeum.search(Odeum::normalizeword(word), -1)
+            puts "(#{word}) Found #{docs.length} docs."
+            docs.each do |id, score|
+                doc = odeum.get_by_id(id)
+                puts "#{score}: #{doc.uri}"
+                doc.close
+            end
+        end
+        
+        puts "Searched #{odeum.doc_count} documents."
+    end
+end
+
+
+cmd = ARGV.shift
+case cmd
+when "update"
+    mgr = Manager.new ARGV.shift
+    mgr.load(ARGV.shift)
+when "search"
+    mgr = Manager.new ARGV.shift
+    mgr.search(ARGV)
+    puts "Done."
+when "merge"
+    Odeum.merge(ARGV.shift, ARGV)
+when "remove"
+    Odeum.remove(ARGV.shift)
+when "optimize"
+    odeum = Odeum::Index.new ARGV.shift, Odeum::OWRITER
+    odeum.optimize
+when "stats"
+    odeum = Odeum::Index.new ARGV.shift, Odeum::OREADER
+    puts "Database stats for #{odeum.name}"
+    puts "
+    Number of Stored Documents: #{odeum.doc_count}
+    Size of Stored Documents: #{odeum.size}
+    Average Document Size: #{odeum.size / odeum.doc_count}
+    Modified Time: #{Time.at(odeum.mtime)}
+    Bucket Count: #{odeum.bucket_count}
+    Buckets Used: #{odeum.buckets_used}
+    Database Inode: #{odeum.inode}
+    "
+else 
+	puts "Commands:  update, search, merge, remove, optimize, stats"
+end

data/doc/rdoc/classes/Odeum.html

@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Module: Odeum</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="Content-Script-Type" content="text/javascript" />
+  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
+  <script type="text/javascript">
+  // <![CDATA[
+
+  function popupCode( url ) {
+    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
+  }
+
+  function toggleCode( id ) {
+    if ( document.getElementById )
+      elem = document.getElementById( id );
+    else if ( document.all )
+      elem = eval( "document.all." + id );
+    else
+      return false;
+
+    elemStyle = elem.style;
+    
+    if ( elemStyle.display != "block" ) {
+      elemStyle.display = "block"
+    } else {
+      elemStyle.display = "none"
+    }
+
+    return true;
+  }
+  
+  // Make codeblocks hidden by default
+  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
+  
+  // ]]>
+  </script>
+
+</head>
+<body>
+
+
+
+    <div id="classHeader">
+        <table class="header-table">
+        <tr class="top-aligned-row">
+          <td><strong>Module</strong></td>
+          <td class="class-name-in-header">Odeum</td>
+        </tr>
+        <tr class="top-aligned-row">
+            <td><strong>In:</strong></td>
+            <td>
+                <a href="../files/ext/odeum_index/odeum_index_c.html">
+                ext/odeum_index/odeum_index.c
+                </a>
+        <br />
+            </td>
+        </tr>
+
+        </table>
+    </div>
+  <!-- banner header -->
+
+  <div id="bodyContent">
+
+
+
+  <div id="contextContent">
+
+
+
+   </div>
+
+    <div id="method-list">
+      <h3 class="section-bar">Methods</h3>
+
+      <div class="name-list">
+      <a href="#M000012">breaktext</a>&nbsp;&nbsp;
+      <a href="#M000010">merge</a>&nbsp;&nbsp;
+      <a href="#M000013">normalizeword</a>&nbsp;&nbsp;
+      <a href="#M000011">remove</a>&nbsp;&nbsp;
+      <a href="#M000014">settuning</a>&nbsp;&nbsp;
+      </div>
+    </div>
+
+  </div>
+
+
+    <!-- if includes -->
+
+    <div id="section">
+
+    <div id="class-list">
+      <h3 class="section-bar">Classes and Modules</h3>
+
+      Class <a href="Odeum/Document.html" class="link">Odeum::Document</a><br />
+Class <a href="Odeum/Index.html" class="link">Odeum::Index</a><br />
+
+    </div>
+
+
+
+
+      
+
+
+    <!-- if method_list -->
+    <div id="methods">
+      <h3 class="section-bar">Public Class methods</h3>
+
+      <div id="method-M000012" class="method-detail">
+        <a name="M000012"></a>
+
+        <div class="method-heading">
+          <a href="Odeum.src/M000012.html" target="Code" class="method-signature"
+            onclick="popupCode('Odeum.src/M000012.html');return false;">
+          <span class="method-name">Odeum::breaktext(test) &rarr; [word1, word2, word3]<br />
+</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Breaks a string into an array of words that are separated by space
+characters and such delimiters as period, commaa, etc. You should also
+check out StringScanner as a more flexible alternative. This function must
+do a lot of data copying and other things in order to convert from <a
+href="Odeum.html">Odeum</a> internal types to Ruby types.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000010" class="method-detail">
+        <a name="M000010"></a>
+
+        <div class="method-heading">
+          <a href="Odeum.src/M000010.html" target="Code" class="method-signature"
+            onclick="popupCode('Odeum.src/M000010.html');return false;">
+          <span class="method-name">Odeum::merge(new_name, other_databases) &rarr; true/false<br />
+</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Merges the databases listed in other_databases (Array of Strings) into the
+new database new_name. If two or more documents have the same URI then the
+first one is adopted and the others are ignored.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000013" class="method-detail">
+        <a name="M000013"></a>
+
+        <div class="method-heading">
+          <a href="Odeum.src/M000013.html" target="Code" class="method-signature"
+            onclick="popupCode('Odeum.src/M000013.html');return false;">
+          <span class="method-name">Odeum::normalizeword(asis) &rarr; normal<br />
+</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Given a word from breaktext (which is considered &quot;as-is&quot;) it will
+&quot;normalize&quot; it in a consistent way which is suitable for
+searching. The normalization effectively strips puntuation and spacing, and
+then lowercases the word. If there is nothing but &quot;removed&quot; chars
+in the asis string then the return is empty. Check for this so you
+don&#8217;t try to search for nothing.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000011" class="method-detail">
+        <a name="M000011"></a>
+
+        <div class="method-heading">
+          <a href="Odeum.src/M000011.html" target="Code" class="method-signature"
+            onclick="popupCode('Odeum.src/M000011.html');return false;">
+          <span class="method-name">Odeum::remove(name) &rarr; true/false<br />
+</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+Removes the database directory and everything in it.
+</p>
+        </div>
+      </div>
+
+      <div id="method-M000014" class="method-detail">
+        <a name="M000014"></a>
+
+        <div class="method-heading">
+          <a href="Odeum.src/M000014.html" target="Code" class="method-signature"
+            onclick="popupCode('Odeum.src/M000014.html');return false;">
+          <span class="method-name">Odeum::settuning(ibnum, idnum, cbnum, csiz) &rarr; nil<br />
+</span>
+          </a>
+        </div>
+      
+        <div class="method-description">
+          <p>
+ibnum=32749: Number of buckets for inverted indexes. idnum=7: Division
+number of inverted index. cbnum=262139: Number of buckets for dirty
+buffers. csiz=8388608: Maximum bytes to use memory for dirty buffers.
+</p>
+<p>
+This is set globally for all Indexes. Not sure what would happen if you
+changed this mid-stream, so don&#8217;t. Make sure everything is closed.
+</p>
+        </div>
+      </div>
+
+
+    </div>
+
+
+  </div>
+
+
+<div id="validator-badges">
+  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
+</div>
+
+</body>
+</html>

data/doc/rdoc/classes/Odeum.src/M000010.html

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html>
+<head>
+  <title>merge (Odeum)</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+</head>
+<body class="standalone-code">
+  <pre>/**
+ * call-seq:
+ *    Odeum::merge(new_name, other_databases) -&gt; true/false
+ *
+ * Merges the databases listed in other_databases (Array of Strings)
+ * into the new database new_name.
+ * If two or more documents have the same URI then the first one is
+ * adopted and the others are ignored.
+ */
+VALUE Odeum_merge(VALUE self, VALUE name, VALUE elemnames) {
+</pre>
+</body>
+</html>

data/doc/rdoc/classes/Odeum.src/M000011.html

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html>
+<head>
+  <title>remove (Odeum)</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+</head>
+<body class="standalone-code">
+  <pre>/**
+ * call-seq:
+ *   Odeum::remove(name) -&gt; true/false
+ * 
+ * Removes the database directory and everything in it.
+ */
+VALUE Odeum_remove(VALUE self, VALUE name) {
+</pre>
+</body>
+</html>

data/doc/rdoc/classes/Odeum.src/M000012.html

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE html 
+     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+<html>
+<head>
+  <title>breaktext (Odeum)</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
+</head>
+<body class="standalone-code">
+  <pre>/**
+ * call-seq:
+ *    Odeum::breaktext(test) -&gt; [word1, word2, word3]
+ *
+ * Breaks a string into an array of words that are separated by
+ * space characters and such delimiters as period, commaa, etc.
+ * You should also check out StringScanner as a more flexible
+ * alternative.  This function must do a lot of data copying and
+ * other things in order to convert from Odeum internal types to Ruby
+ * types.
+ */
+VALUE Odeum_breaktext(VALUE self, VALUE text) {
+</pre>
+</body>
+</html>