ankusa 0.0.12 → 0.0.13

data/README.rdoc

@@ -110,6 +110,13 @@ To use the Cassandra storage class:
   # defaults: host='127.0.0.1', port=9160, keyspace = 'ankusa', max_classes = 100
   storage = Ankusa::HBaseStorage.new host, port, keyspace, max_classes
 
+For MongoDB storage:
+  require 'ankusa/mongo_db_storage'
+  storage = Ankusa::MongoDbStorage.new :host => "localhost", :port => 27017, :db => "ankusa"
+  # defaults: :host => "localhost", :port => 27017, :db => "ankusa"
+  # no default username or password
+  # tou can also use frequency_tablename and summary_tablename options
+
 
 == Running Tests
 You can run the tests for any of the four storage methods.  For instance, for memory storage:
@@ -121,6 +128,8 @@ For the other methods you will need to edit the file test/config.yml and set the
   rake test_cassandra
   # or
   rake test_filesystem
+  #or
+  rake test_mongo_db
 
 
 

data/Rakefile

@@ -40,3 +40,10 @@ Rake::TestTask.new("test_filesystem") { |t|
   t.test_files = FileList['test/hasher_test.rb', 'test/file_system_classifier_test.rb']
   t.verbose = true
 }
+
+desc "Run all unit tests with MongoDb storage"
+Rake::TestTask.new("test_mongo_db") { |t|
+  t.libs << "lib"
+  t.test_files = FileList['test/hasher_test.rb', 'test/mongo_db_classifier_test.rb']
+  t.verbose = true
+}

data/lib/ankusa/mongo_db_storage.rb

@@ -0,0 +1,127 @@
+require 'mongo'
+#require 'bson_ext'
+
+module Ankusa
+  class MongoDbStorage
+
+    def initialize(opts={})
+      options = { :host => "localhost", :port => 27017, :db => "ankusa",
+                  :frequency_tablename => "word_frequencies", :summary_tablename => "summary"
+                }.merge(opts)
+
+      @db = Mongo::Connection.new(options[:host], options[:port]).db(options[:db])
+      @db.authenticate(options[:username], options[:password]) if options[:password]
+
+      @ftablename = options[:frequency_tablename]
+      @stablename = options[:summary_tablename]
+
+      @klass_word_counts = {}
+      @klass_doc_counts = {}
+
+      init_tables
+    end
+
+    def init_tables
+      @db.create_collection(@ftablename) unless @db.collection_names.include?(@ftablename)
+      freq_table.create_index('word')
+      @db.create_collection(@stablename) unless @db.collection_names.include?(@stablename)
+      summary_table.create_index('klass')
+    end
+
+    def drop_tables
+      @db.drop_collection(@ftablename)
+      @db.drop_collection(@stablename)
+    end
+
+    def classnames
+      summary_table.distinct('klass')
+    end
+
+    def reset
+      drop_tables
+      init_tables
+    end
+
+    def incr_word_count(klass, word, count)
+      freq_table.update({:word => word}, { '$inc' => {klass => count} }, :upsert => true)
+
+      #update vocabulary size
+      word_doc = freq_table.find_one({:word => word})
+      if word_doc[klass.to_s] == count
+        increment_summary_klass(klass, 'vocabulary_size', 1)
+      elsif word_doc[klass.to_s] == 0
+        increment_summary_klass(klass, 'vocabulary_size', -1)
+      end
+      word_doc[klass.to_s]
+    end
+
+    def incr_total_word_count(klass, count)
+      increment_summary_klass(klass, 'word_count', count)
+    end
+
+    def incr_doc_count(klass, count)
+      increment_summary_klass(klass, 'doc_count', count)
+    end
+
+    def get_word_counts(word)
+      counts = Hash.new(0)
+
+      word_doc = freq_table.find_one({:word => word})
+      if word_doc
+        word_doc.delete("_id")
+        word_doc.delete("word")
+        #convert keys to symbols
+        counts.merge!(word_doc.inject({}){|h, (k, v)| h[(k.to_sym rescue k) || k] = v; h}) 
+      end
+
+      counts
+    end
+
+    def get_total_word_count(klass)
+      klass_doc = summary_table.find_one(:klass => klass)
+      klass_doc ? klass_doc['word_count'].to_f : 0.0
+    end
+
+    def doc_count_totals
+      count = Hash.new(0)
+
+      summary_table.find.each do |doc|
+        count[ doc['klass'] ] = doc['doc_count']
+      end
+
+      count
+    end
+
+    def get_vocabulary_sizes
+      count = Hash.new(0)
+
+      summary_table.find.each do |doc|
+        count[ doc['klass'] ] = doc['vocabulary_size']
+      end
+
+      count
+    end
+
+    def get_doc_count(klass)
+      klass_doc = summary_table.find_one(:klass => klass) 
+      klass_doc ? klass_doc['doc_count'].to_f : 0.0
+    end
+
+    def close
+    end
+
+    private
+    def summary_table
+      @stable ||= @db[@stablename]
+    end
+
+    def freq_table
+      @ftable ||= @db[@ftablename]
+    end
+
+    def increment_summary_klass(klass, field, count)
+      summary_table.update({:klass => klass}, { '$inc' => {field => count} }, :upsert => true)
+    end
+
+  end
+end

data/lib/ankusa/version.rb

@@ -1,3 +1,3 @@
 module Ankusa
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ankusa
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: 
   segments: 
   - 0
   - 0
-  - 12
-  version: 0.0.12
+  - 13
+  version: 0.0.13
 platform: ruby
 authors: 
 - Brian Muller
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-10 00:00:00 Z
+date: 2012-04-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fast-stemmer
@@ -50,6 +50,7 @@ files:
 - lib/ankusa/hbase_storage.rb
 - lib/ankusa/kl_divergence.rb
 - lib/ankusa/memory_storage.rb
+- lib/ankusa/mongo_db_storage.rb
 - lib/ankusa/naive_bayes.rb
 - lib/ankusa/stopwords.rb
 - lib/ankusa/version.rb