yanbi-ml 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd991aa7bffd5dd520fc71f40954b3603c8390a0
-  data.tar.gz: ff8d4ae0a916964c018d325f5c07ca75bcd5dc3f
+  metadata.gz: c5a2754ccc8b5d063fe67e47a3a1794195ba6e2b
+  data.tar.gz: 23a09668b5f69f61a9889eb0b59d285aa8ddf680
 SHA512:
-  metadata.gz: 8ee84e2f9a7c28dba34ab09af791f5be93b019317b2839597f40a55b3e88a1795e0e8474e6615a9c8dcfb708c3d72fac64920136c495e24c5c0a575233770b84
-  data.tar.gz: 2f874ba50d8f5538b3941be0f70cedfc5a0936a810d2cba0ac0a598e86b4dc220a9bc5c561affd701c4d020fd80a3a41cfc3ac1338bcfaa9dc573a233544227f
+  metadata.gz: 41c3ab772fd3bb90de3860d6833bac4a08c91bd4f0b5ae43e8db39a6ed281cd1b5e379ad74271346e9dfee3ae76d95e01fcfae27640d0dfe5d0804904a4a8e64
+  data.tar.gz: a93add553c7928a0eb8f34f25fedd6309479a3981dbd8e6a838708a96af824ee8b436879950223ce27d14a05be339a172fc10e1b0e570acecf8487d778c2aa23

data/README.md

@@ -181,6 +181,33 @@ docs.each_doc do |d|
 end
 ```
 
+There's also a single, global bag of words that contains all of the words seen in every document in the corpus.  This is accessed (surprisingly) through the 'all' attribute.
+
+```ruby
+# Non unique global list of words
+docs.all.words
+
+# Unique global list of words
+docs.all.words.uniq
+```
+
+Note that this global word bag is updated whenever you remove words through by iterating through documents with each_doc.
+
+
+## Dictionaries
+
+Speaking of a global list of words, the corpus class also allows you to capture a snapshot of the unique list of words in a set of documents as a dictionary object.  This object can then be used to encode strings as integer arrays of indices:
+
+```ruby
+my_dictionary = docs.to_index
+
+# Get an integer mapping of the words in this string
+indices = my_dictionary.to_idx('the quick brown fox')
+```
+
+Words not present in the dictionary will be returned as nils.  This is useful for working with other types of classifiers that might not be capable of accepting straight text.
+ 
+
 ## Feature thresholds 
 
 A method on the classifier is provided to prune infrequently seen features.  This is often one of the first things recommended for improving the accuracy of a classifier in real world applications.  Note that when you prune features, there's no un-pruning afterwards - so be sure you actually want to do it!

data/lib/corpus.rb

@@ -21,6 +21,7 @@ module Yanbi
   
     def initialize(klass=WordBag)
       @all = klass.new
+      @index = nil
       @docs = []
       @bags = []
     end
@@ -52,14 +53,39 @@ module Yanbi
         @bags << @all.class.new(doc)
         @all.add_text doc
         @docs << doc
+        @index = nil
       end
     end
   
     def each_doc
+      before = 0
+      after = 0
+
+      @bags.each do |bag, doc|
+        before += bag.words.count
+        yield bag, doc
+        after += bag.words.count
+      end
+
+      rebuild_all if before != after
+    end
+
+    def to_index
+      if @index.nil?
+        w = all.words.uniq
+        @index = Yanbi::Dictionary.new(w, @all.class)
+      end
+
+      @index
+    end
+
+    private
+
+    def rebuild_all
+      @all = @all.class.new
       @bags.each do |bag|
-        yield bag
+        @all.add_text bag.words.join(' ')
       end
     end
   end
-
 end

data/lib/dictionary.rb

@@ -0,0 +1,40 @@
+# Author::    Robert Dormer (mailto:rdormer@gmail.com)
+# Copyright:: Copyright (c) 2016 Robert Dormer
+# License::   MIT
+
+# Class for storing word dictionaries created from wordbags
+# and corpuses.  Includes methods to encode strings as integer
+# index arrays into the dictionary.
+
+$: << File.dirname(__FILE__)
+require 'yaml'
+
+module Yanbi
+  class Dictionary
+    attr_accessor :bag_class
+  
+    def initialize(w, klass)
+      @index = {}
+      @klass = klass
+      i = (0..w.size).to_a
+      w.zip(i).each { |x| @index[x.first] = x.last }
+    end
+  
+    def to_idx(doc)
+      bag = @klass.new(doc)
+      bag.words.map { |w| @index[w] }
+    end
+  
+    def self.load(fname)
+      c = YAML.load(File.read(fname + '.yml'))
+      raise LoadError unless c.is_a? self
+      c
+    end
+  
+    def save(name)
+      File.open(name + '.yml', 'w') do |out|
+        YAML.dump(self, out)
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -3,5 +3,5 @@
 # License::   MIT
 
 module Yanbi
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

data/lib/yanbi.rb

@@ -13,5 +13,7 @@ Dir[base + "/bayes/**/*.rb"].each do |c|
   require c
 end
 
+require 'dictionary'
 require 'corpus'
 require 'version'
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yanbi-ml
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Robert Dormer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-31 00:00:00.000000000 Z
+date: 2017-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -82,6 +82,7 @@ files:
 - lib/bayes/bayes.rb
 - lib/bayes/fisher.rb
 - lib/corpus.rb
+- lib/dictionary.rb
 - lib/version.rb
 - lib/wordbags/diadbag.rb
 - lib/wordbags/stembag.rb