RubyGems - ankusa - Versions diffs - 0.0.13 → 0.0.14 - Mend

ankusa 0.0.13 → 0.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (57) hide show

data/Gemfile +4 -0
data/Gemfile.lock +16 -0
data/README.rdoc +5 -3
data/Rakefile +5 -5
data/lib/ankusa/naive_bayes.rb +3 -3
data/lib/ankusa/version.rb +1 -1
metadata +36 -100
data/docs/Ankusa.html +0 -229
data/docs/Ankusa/CassandraStorage.html +0 -801
data/docs/Ankusa/Classifier.html +0 -440
data/docs/Ankusa/FileSystemStorage.html +0 -376
data/docs/Ankusa/HBaseStorage.html +0 -845
data/docs/Ankusa/KLDivergenceClassifier.html +0 -265
data/docs/Ankusa/MemoryStorage.html +0 -672
data/docs/Ankusa/NaiveBayesClassifier.html +0 -313
data/docs/Ankusa/TextHash.html +0 -390
data/docs/README_rdoc.html +0 -268
data/docs/String.html +0 -241
data/docs/created.rid +0 -14
data/docs/images/brick.png +0 -0
data/docs/images/brick_link.png +0 -0
data/docs/images/bug.png +0 -0
data/docs/images/bullet_black.png +0 -0
data/docs/images/bullet_toggle_minus.png +0 -0
data/docs/images/bullet_toggle_plus.png +0 -0
data/docs/images/date.png +0 -0
data/docs/images/find.png +0 -0
data/docs/images/loadingAnimation.gif +0 -0
data/docs/images/macFFBgHack.png +0 -0
data/docs/images/package.png +0 -0
data/docs/images/page_green.png +0 -0
data/docs/images/page_white_text.png +0 -0
data/docs/images/page_white_width.png +0 -0
data/docs/images/plugin.png +0 -0
data/docs/images/ruby.png +0 -0
data/docs/images/tag_green.png +0 -0
data/docs/images/wrench.png +0 -0
data/docs/images/wrench_orange.png +0 -0
data/docs/images/zoom.png +0 -0
data/docs/index.html +0 -212
data/docs/js/darkfish.js +0 -116
data/docs/js/jquery.js +0 -32
data/docs/js/quicksearch.js +0 -114
data/docs/js/thickbox-compressed.js +0 -10
data/docs/lib/ankusa/cassandra_storage_rb.html +0 -54
data/docs/lib/ankusa/classifier_rb.html +0 -52
data/docs/lib/ankusa/extensions_rb.html +0 -54
data/docs/lib/ankusa/file_system_storage_rb.html +0 -54
data/docs/lib/ankusa/hasher_rb.html +0 -56
data/docs/lib/ankusa/hbase_storage_rb.html +0 -54
data/docs/lib/ankusa/kl_divergence_rb.html +0 -52
data/docs/lib/ankusa/memory_storage_rb.html +0 -52
data/docs/lib/ankusa/naive_bayes_rb.html +0 -52
data/docs/lib/ankusa/stopwords_rb.html +0 -52
data/docs/lib/ankusa/version_rb.html +0 -52
data/docs/lib/ankusa_rb.html +0 -64
data/docs/rdoc.css +0 -759

data/docs/README_rdoc.html DELETED

@@ -1,268 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-  <title>File: README.rdoc [Ankusa - Naive Bayes classifier with big data storage]</title>
-  <link type="text/css" media="screen" href="./rdoc.css" rel="stylesheet" />
-  <script src="./js/jquery.js" type="text/javascript"
-    charset="utf-8"></script>
-  <script src="./js/thickbox-compressed.js" type="text/javascript"
-    charset="utf-8"></script>
-  <script src="./js/quicksearch.js" type="text/javascript"
-    charset="utf-8"></script>
-  <script src="./js/darkfish.js" type="text/javascript"
-    charset="utf-8"></script>
-</head>
-<body class="file">
-  <div id="metadata">
-    <div id="home-metadata">
-      <div id="home-section" class="section">
-        <h3 class="section-header">
-          <a href="./index.html">Home</a>
-          <a href="./index.html#classes">Classes</a>
-          <a href="./index.html#methods">Methods</a>
-        </h3>
-      </div>
-    </div>
-    <div id="project-metadata">
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-          <li class="file"><a href="./README_rdoc.html">README.rdoc</a></li>
-        </ul>
-      </div>
-      <div id="classindex-section" class="section project-section">
-        <h3 class="section-header">Class Index
-          <span class="search-toggle"><img src="./images/find.png"
-            height="16" width="16" alt="[+]"
-            title="show/hide quicksearch" /></span></h3>
-        <form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
-        <fieldset>
-          <legend>Quicksearch</legend>
-          <input type="text" name="quicksearch" value=""
-            class="quicksearch-field" />
-        </fieldset>
-        </form>
-        <ul class="link-list">
-          <li><a href="./Ankusa.html">Ankusa</a></li>
-          <li><a href="./Ankusa/CassandraStorage.html">Ankusa::CassandraStorage</a></li>
-          <li><a href="./Ankusa/Classifier.html">Ankusa::Classifier</a></li>
-          <li><a href="./Ankusa/FileSystemStorage.html">Ankusa::FileSystemStorage</a></li>
-          <li><a href="./Ankusa/HBaseStorage.html">Ankusa::HBaseStorage</a></li>
-          <li><a href="./Ankusa/KLDivergenceClassifier.html">Ankusa::KLDivergenceClassifier</a></li>
-          <li><a href="./Ankusa/MemoryStorage.html">Ankusa::MemoryStorage</a></li>
-          <li><a href="./Ankusa/NaiveBayesClassifier.html">Ankusa::NaiveBayesClassifier</a></li>
-          <li><a href="./Ankusa/TextHash.html">Ankusa::TextHash</a></li>
-          <li><a href="./String.html">String</a></li>
-        </ul>
-        <div id="no-class-search-results" style="display: none;">No matching classes.</div>
-      </div>
-    </div>
-  </div>
-  <div id="documentation">
-<h1>ankusa</h1>
-<p><a href="Ankusa.html">Ankusa</a> is a text classifier in Ruby that can use
-either Hadoop’s HBase or Cassandra for storage.  Because it uses HBase or
-Cassandra as a backend, the training corpus can be many terabytes in size
-(though additional memory and single file storage abilities also exist for
-smaller corpora).</p>
-<p><a href="Ankusa.html">Ankusa</a> currently provides both a Naive Bayes and
-Kullback-Leibler divergence classifier.  It ignores common words (a.k.a,
-stop words) and stems all others.  Additionally, it uses Laplacian
-smoothing in both classification methods.</p>
-<h2>Installation</h2>
-<p>First, install HBase/Hadoop or Cassandra (&gt;= 0.7.0-rc2).  Then, install
-the appropriate gem:</p>
-<pre>gem install hbaserb
-# or
-gem install cassandra</pre>
-<p>If you’re using HBase, make sure the HBase Thrift interface has been
-started as well.  Then:</p>
-<pre>gem install ankusa</pre>
-<h2>Basic Usage</h2>
-<p>Using the naive Bayes classifier:</p>
-<pre>require 'rubygems'
-require 'ankusa'
-require 'ankusa/hbase_storage'
-# connect to HBase.  Alternatively, just for this test, use in memory storage with
-# storage = Ankusa::MemoryStorage.new
-storage = Ankusa::HBaseStorage.new 'localhost'
-c = Ankusa::NaiveBayesClassifier.new storage
-# Each of these calls will return a bag-of-words
-# has with stemmed words as keys and counts as values
-c.train :spam, &quot;This is some spammy text&quot;
-c.train :good, &quot;This is not the bad stuff&quot;
-# This will return the most likely class (as symbol)
-puts c.classify &quot;This is some spammy text&quot;
-# This will return Hash with classes as keys and
-# membership probability as values
-puts c.classifications &quot;This is some spammy text&quot;
-# If you have a large corpus, the probabilities will
-# likely all be 0.  In that case, you must use log
-# likelihood values
-puts c.log_likelihoods &quot;This is some spammy text&quot;
-# get a list of all classes
-puts c.classnames
-# close connection
-storage.close</pre>
-<h2>KL Diverence Classifier</h2>
-<p>There is a Kullback–Leibler divergence classifier as well.  KL divergence
-is a distance measure (though not a true metric because it does not satisfy
-the triangle inequality).  The KL classifier simply measures the relative
-entropy between the text you want to classify and each of the classes.  The
-class with the shortest “distance” is the best class.  You may find
-that for a especially large corpus it may be slightly faster to use this
-classifier (since prior probablities are never calculated, only
-likelihoods).</p>
-<p>The API is the same as the NaiveBayesClassifier, except rather than calling
-“classifications” if you want actual numbers you call “distances”.</p>
-<pre>require 'rubygems'
-require 'ankusa'
-require 'ankusa/hbase_storage'
-# connect to HBase
-storage = Ankusa::HBaseStorage.new 'localhost'
-c = Ankusa::KLDivergenceClassifier.new storage
-# Each of these calls will return a bag-of-words
-# has with stemmed words as keys and counts as values
-c.train :spam, &quot;This is some spammy text&quot;
-c.train :good, &quot;This is not the bad stuff&quot;
-# This will return the most likely class (as symbol)
-puts c.classify &quot;This is some spammy text&quot;
-# This will return Hash with classes as keys and
-# distances &gt;= 0 as values
-puts c.distances &quot;This is some spammy text&quot;
-# get a list of all classes
-puts c.classnames
-# close connection
-storage.close</pre>
-<h2>Storage Methods</h2>
-<p><a href="Ankusa.html">Ankusa</a> has a generalized storage interface that
-has been implemented for HBase, Cassandra, single file, and in-memory
-storage.</p>
-<p>Memory storage can be used when you have a very small corpora</p>
-<pre>require 'ankusa/memory_storage'
-storage = Ankusa::MemoryStorage.new</pre>
-<p>FileSystem storage can be used when you have a very small corpora and want
-to persist the classification results.</p>
-<pre>require 'ankusa/file_system_storage'
-storage = Ankusa::FileSystemStorage.new '/path/to/file'
-# Do classification ...
-storage.save</pre>
-<p>The FileSystem storage does NOT save to the filesystem automatically, the
-#save method must be invoked to save and persist the results</p>
-<p>HBase storage:</p>
-<pre>require 'ankusa/hbase_storage'
-# defaults: host='localhost', port=9090, frequency_tablename=&quot;ankusa_word_frequencies&quot;, summary_tablename=&quot;ankusa_summary&quot;
-storage = Ankusa::HBaseStorage.new host, port, frequency_tablename, summary_tablename</pre>
-<p>For Cassandra storage:</p>
-<ul><li>
-<p>You will need Cassandra version 0.7.0-rc2 or greater.</p>
-</li><li>
-<p>You will need to set a max number classes since current implementation of
-the Ruby Cassandra client doesn’t support table scans.</p>
-</li><li>
-<p>Prior to using the Cassandra storage you will need to run the following
-command from the cassandra-cli: “create keyspace ankusa with
-replication_factor = 1”.  This should be fixed with a new release
-candidate for Cassandra.</p>
-</li></ul>
-<p>To use the Cassandra storage class:</p>
-<pre>require 'ankusa/cassandra_storage'
-# defaults: host='127.0.0.1', port=9160, keyspace = 'ankusa', max_classes = 100
-storage = Ankusa::HBaseStorage.new host, port, keyspace, max_classes</pre>
-<h2>Running Tests</h2>
-<p>You can run the tests for any of the four storage methods.  For instance,
-for memory storage:</p>
-<pre>rake test_memory</pre>
-<p>For the other methods you will need to edit the file test/config.yml and
-set the configuration params.  Then:</p>
-<pre>rake test_hbase
-# or
-rake test_cassandra
-# or
-rake test_filesystem</pre>
-  </div>
-  <div id="validator-badges">
-    <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-    <p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
-      Rdoc Generator</a> 2</small>.</p>
-  </div>
-</body>
-</html>

data/docs/String.html DELETED

@@ -1,241 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
-  <title>Class: String</title>
-  <link rel="stylesheet" href="./rdoc.css" type="text/css" media="screen" />
-  <script src="./js/jquery.js" type="text/javascript" charset="utf-8"></script>
-  <script src="./js/thickbox-compressed.js" type="text/javascript" charset="utf-8"></script>
-  <script src="./js/quicksearch.js" type="text/javascript" charset="utf-8"></script>
-  <script src="./js/darkfish.js" type="text/javascript" charset="utf-8"></script>
-</head>
-<body id="top" class="class">
-  <div id="metadata">
-    <div id="home-metadata">
-      <div id="home-section" class="section">
-        <h3 class="section-header">
-          <a href="./index.html">Home</a>
-          <a href="./index.html#classes">Classes</a>
-          <a href="./index.html#methods">Methods</a>
-        </h3>
-      </div>
-    </div>
-    <div id="file-metadata">
-      <div id="file-list-section" class="section">
-        <h3 class="section-header">In Files</h3>
-        <div class="section-body">
-          <ul>
-            <li><a href="./lib/ankusa/extensions_rb.html?TB_iframe=true&amp;height=550&amp;width=785"
-              class="thickbox" title="lib/ankusa/extensions.rb">lib/ankusa/extensions.rb</a></li>
-          </ul>
-        </div>
-      </div>
-    </div>
-    <div id="class-metadata">
-      <!-- Parent Class -->
-      <div id="parent-class-section" class="section">
-        <h3 class="section-header">Parent</h3>
-        <p class="link">Object</p>
-      </div>
-      <!-- Method Quickref -->
-      <div id="method-list-section" class="section">
-        <h3 class="section-header">Methods</h3>
-        <ul class="link-list">
-          <li><a href="#method-i-numeric-3F">#numeric?</a></li>
-          <li><a href="#method-i-to_ascii">#to_ascii</a></li>
-        </ul>
-      </div>
-    </div>
-    <div id="project-metadata">
-      <div id="fileindex-section" class="section project-section">
-        <h3 class="section-header">Files</h3>
-        <ul>
-          <li class="file"><a href="./README_rdoc.html">README.rdoc</a></li>
-        </ul>
-      </div>
-      <div id="classindex-section" class="section project-section">
-        <h3 class="section-header">Class/Module Index
-          <span class="search-toggle"><img src="./images/find.png"
-            height="16" width="16" alt="[+]"
-            title="show/hide quicksearch" /></span></h3>
-        <form action="#" method="get" accept-charset="utf-8" class="initially-hidden">
-        <fieldset>
-          <legend>Quicksearch</legend>
-          <input type="text" name="quicksearch" value=""
-            class="quicksearch-field" />
-        </fieldset>
-        </form>
-        <ul class="link-list">
-          <li><a href="./Ankusa.html">Ankusa</a></li>
-          <li><a href="./Ankusa/CassandraStorage.html">Ankusa::CassandraStorage</a></li>
-          <li><a href="./Ankusa/Classifier.html">Ankusa::Classifier</a></li>
-          <li><a href="./Ankusa/FileSystemStorage.html">Ankusa::FileSystemStorage</a></li>
-          <li><a href="./Ankusa/HBaseStorage.html">Ankusa::HBaseStorage</a></li>
-          <li><a href="./Ankusa/KLDivergenceClassifier.html">Ankusa::KLDivergenceClassifier</a></li>
-          <li><a href="./Ankusa/MemoryStorage.html">Ankusa::MemoryStorage</a></li>
-          <li><a href="./Ankusa/NaiveBayesClassifier.html">Ankusa::NaiveBayesClassifier</a></li>
-          <li><a href="./Ankusa/TextHash.html">Ankusa::TextHash</a></li>
-          <li><a href="./String.html">String</a></li>
-        </ul>
-        <div id="no-class-search-results" style="display: none;">No matching classes.</div>
-      </div>
-    </div>
-  </div>
-  <div id="documentation">
-    <h1 class="class">String</h1>
-    <div id="description" class="description">
-    </div><!-- description -->
-    <div id="5Buntitled-5D" class="documentation-section">
-      <!-- Methods -->
-      <div id="public-instance-method-details" class="method-section section">
-        <h3 class="section-header">Public Instance Methods</h3>
-        <div id="numeric-3F-method" class="method-detail ">
-          <a name="method-i-numeric-3F"></a>
-          <div class="method-heading">
-            <span class="method-name">numeric?</span><span
-              class="method-args">()</span>
-            <span class="method-click-advice">click to toggle source</span>
-          </div>
-          <div class="method-description">
-            <div class="method-source-code" id="numeric-3F-source">
-<pre>
-<span class="ruby-comment"># File lib/ankusa/extensions.rb, line 4</span>
-<span class="ruby-keyword">def</span> <span class="ruby-identifier">numeric?</span>
-  <span class="ruby-keyword">true</span> <span class="ruby-keyword">if</span> <span class="ruby-constant">Float</span>(<span class="ruby-keyword">self</span>) <span class="ruby-keyword">rescue</span> <span class="ruby-keyword">false</span>
-<span class="ruby-keyword">end</span></pre>
-            </div><!-- numeric-3F-source -->
-          </div>
-        </div><!-- numeric-3F-method -->
-        <div id="to_ascii-method" class="method-detail ">
-          <a name="method-i-to_ascii"></a>
-          <div class="method-heading">
-            <span class="method-name">to_ascii</span><span
-              class="method-args">()</span>
-            <span class="method-click-advice">click to toggle source</span>
-          </div>
-          <div class="method-description">
-            <div class="method-source-code" id="to_ascii-source">
-<pre>
-<span class="ruby-comment"># File lib/ankusa/extensions.rb, line 8</span>
-<span class="ruby-keyword">def</span> <span class="ruby-identifier">to_ascii</span>
-  <span class="ruby-comment"># from http://www.jroller.com/obie/tags/unicode</span>
-  <span class="ruby-identifier">converter</span> = <span class="ruby-constant">Iconv</span>.<span class="ruby-identifier">new</span>(<span class="ruby-string">'ASCII//IGNORE//TRANSLIT'</span>, <span class="ruby-string">'UTF-8'</span>)
-  <span class="ruby-identifier">converter</span>.<span class="ruby-identifier">iconv</span>(<span class="ruby-keyword">self</span>).<span class="ruby-identifier">unpack</span>(<span class="ruby-string">'U*'</span>).<span class="ruby-identifier">select</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">cp</span><span class="ruby-operator">|</span> <span class="ruby-identifier">cp</span> <span class="ruby-operator">&lt;</span> <span class="ruby-value">127</span> }.<span class="ruby-identifier">pack</span>(<span class="ruby-string">'U*'</span>) <span class="ruby-keyword">rescue</span> <span class="ruby-string">&quot;&quot;</span>
-<span class="ruby-keyword">end</span></pre>
-            </div><!-- to_ascii-source -->
-          </div>
-        </div><!-- to_ascii-method -->
-      </div><!-- public-instance-method-details -->
-    </div><!-- 5Buntitled-5D -->
-  </div><!-- documentation -->
-  <div id="validator-badges">
-    <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-    <p><small>Generated with the <a href="http://deveiate.org/projects/Darkfish-Rdoc/">Darkfish
-      Rdoc Generator</a> 2</small>.</p>
-  </div>
-</body>
-</html>