vector_embed 0.3.3 → 0.4.0

data/CHANGELOG

@@ -1,3 +1,13 @@
+0.4.0 / 2013-07-01
+
+* Breaking changes
+
+  * Treat 0/1 and '0'/'1' as true/false (per @ihough)
+
+* Enhancements
+
+  * Add VectorEmbed#stats_report, which lists types and cardinalities of features (thanks @ihough)
+
 0.3.3 / 2013-06-14
 
 * Enhancement

data/lib/vector_embed.rb

@@ -21,15 +21,15 @@ class VectorEmbed
   LABEL_MAKERS =   [Maker::Boolean, Maker::Number]
   FEATURE_MAKERS = [Maker::Boolean, Maker::Date, Maker::Number, Maker::Ngram, Maker::Phrase]
 
-  attr_reader :options
   attr_accessor :logger
   attr_reader :dict
+  attr_reader :options
 
   def initialize(options = {})
     @options = options.dup
     @mutex = Mutex.new
     @feature_makers = {}
-    @logger = options[:logger] || (l = Logger.new($stderr); l.level = Logger::INFO; l)
+    @logger = options[:logger] || (l = Logger.new($stderr); l.level = (ENV['VERBOSE'] == 'true') ? Logger::DEBUG : Logger::INFO; l)
     if dict = @options.delete(:dict)
       @dict = dict.dup
     end
@@ -59,15 +59,43 @@ class VectorEmbed
   end
 
   def index(parts)
-    k = parts.join NULL_BYTE
+    sig = parts.join NULL_BYTE
     if dict
-      k = Digest::MD5.digest k
-      dict[k] || @mutex.synchronize do
-        dict[k] ||= dict.length + 1
+      sig = Digest::MD5.digest sig
+      dict[sig] || @mutex.synchronize do
+        dict[sig] ||= begin
+          k = parts[0]
+          @feature_makers[k].cardinality += 1
+          dict[sig] = dict.length + 1
+        end
       end
     else
-      MurmurHash3::V32.str_hash(k).to_s[0..6].to_i
+      MurmurHash3::V32.str_hash(sig).to_s[0..6].to_i
+    end
+  end
+
+  def stats_report
+    report = @feature_makers.map do |feature, maker|
+      [feature, maker.class, maker.cardinality]
+    end
+    total_cardinality = report.inject(0) { |sum, row| sum += row[2]; sum }
+
+    report.unshift %w{ Feature Class Cardinality }
+    feature_width     = report.map { |row| row[0].to_s.length }.max
+    class_width       = report.map { |row| row[1].to_s.length }.max
+    cardinality_width = report.map { |row| row[2].to_s.length }.max
+
+    report = report.map do |row|
+      [
+        row[0].to_s.ljust(feature_width),
+        row[1].to_s.ljust(class_width),
+        row[2].to_s.rjust(cardinality_width),
+      ].join(' | ')
     end
+    total_width = report.first.length
+    report.insert(1, ''.ljust(total_width, '-'))
+    report.push(total_cardinality.to_s.rjust(total_width))
+    report.push('').join("\n")
   end
 
   private

data/lib/vector_embed/maker.rb

@@ -20,12 +20,14 @@ class VectorEmbed
       end
     end
 
+    attr_accessor :cardinality
     attr_reader :parent
     attr_reader :k
 
     def initialize(k, parent)
       @k = k
       @parent = parent
+      @cardinality = 0
     end
 
     def pairs(v)
@@ -42,7 +44,7 @@ class VectorEmbed
         if (vv = value(v)).nil?
           []
         else
-          [ [ parent.index([k]), value(v) ] ]
+          [ [ parent.index([k]), vv ] ]
         end
       end
     end

data/lib/vector_embed/maker/boolean.rb

@@ -27,9 +27,9 @@ class VectorEmbed
 
       def pairs(v)
         case v
-        when TrueClass, TRUE, T
+        when TrueClass, TRUE, T, 1, '1'
           [ [ parent.index([k, 'true']), 1 ] ]
-        when FalseClass, FALSE, F
+        when FalseClass, FALSE, F, 0, '0'
           [ [ parent.index([k, 'false']), 1 ] ]
         when NilClass, NULL, SLASH_N, BLANK
           [ [ parent.index([k, 'null']), 1 ] ]

data/lib/vector_embed/version.rb

@@ -1,3 +1,3 @@
 class VectorEmbed
-  VERSION = '0.3.3'
+  VERSION = '0.4.0'
 end

data/spec/vector_embed_spec.rb

@@ -101,20 +101,28 @@ describe VectorEmbed do
     describe 'in boolean attributes' do
       it "stores true/false/nil as (1,0,0)/(0,1,0)/(0,0,1)" do
         v = VectorEmbed.new
-        v.line(1, 1 => true).should == "1 #{l_h("1\x00true")}:1"
+        v.line(1, 1 => true).should   == "1 #{l_h("1\x00true")}:1"
         v.line(1, 1 => 'true').should == "1 #{l_h("1\x00true")}:1"
         v.line(1, 1 => 'TRUE').should == "1 #{l_h("1\x00true")}:1"
-        v.line(1, 1 => 't').should == "1 #{l_h("1\x00true")}:1"
-        v.line(1, 1 => 'T').should == "1 #{l_h("1\x00true")}:1"
-        v.line(1, 1 => false).should == "1 #{l_h("1\x00false")}:1"
+        v.line(1, 1 => 't').should    == "1 #{l_h("1\x00true")}:1"
+        v.line(1, 1 => 'T').should    == "1 #{l_h("1\x00true")}:1"
+        v.line(1, 1 => 1).should      == "1 #{l_h("1\x00true")}:1"
+        v.line(1, 1 => '1').should    == "1 #{l_h("1\x00true")}:1"
+
+        v.line(1, 1 => false).should   == "1 #{l_h("1\x00false")}:1"
         v.line(1, 1 => 'false').should == "1 #{l_h("1\x00false")}:1"
         v.line(1, 1 => 'FALSE').should == "1 #{l_h("1\x00false")}:1"
-        v.line(1, 1 => 'f').should == "1 #{l_h("1\x00false")}:1"
-        v.line(1, 1 => 'F').should == "1 #{l_h("1\x00false")}:1"
-        v.line(1, 1 => nil).should == "1 #{l_h("1\x00null")}:1"
+        v.line(1, 1 => 'f').should     == "1 #{l_h("1\x00false")}:1"
+        v.line(1, 1 => 'F').should     == "1 #{l_h("1\x00false")}:1"
+        v.line(1, 1 => 0).should       == "1 #{l_h("1\x00false")}:1"
+        v.line(1, 1 => '0').should     == "1 #{l_h("1\x00false")}:1"
+
+        v.line(1, 1 => nil).should    == "1 #{l_h("1\x00null")}:1"
         v.line(1, 1 => 'null').should == "1 #{l_h("1\x00null")}:1"
         v.line(1, 1 => 'NULL').should == "1 #{l_h("1\x00null")}:1"
-        v.line(1, 1 => '\N').should == "1 #{l_h("1\x00null")}:1"
+        v.line(1, 1 => '\N').should   == "1 #{l_h("1\x00null")}:1"
+        v.line(1, 1 => '').should     == "1 #{l_h("1\x00null")}:1"
+        v.line(1, 1 => " \t ").should == "1 #{l_h("1\x00null")}:1"
       end
     end
 
@@ -171,7 +179,7 @@ describe VectorEmbed do
       v.line(3, 1 => '\N').should == v.line(3, 1 => 0)
     end
 
-    it "stores strings as m-category attributes" do
+    it "stores strings as m-category attributes, ignoring whitespace" do
       v = VectorEmbed.new
       v.line(1, 1 => 'sfh').should == "1 #{l_h("1\x00sfh")}:1"
       v.line(1, 1 => 'mfh').should == "1 #{l_h("1\x00mfh")}:1"
@@ -307,8 +315,35 @@ describe VectorEmbed do
     end
   end
 
-  describe 'debug mode' do
-    
+  describe '#stats_report' do
+    it "reports statistics on the embedded features" do
+      v = VectorEmbed.new dict: {}
+      v.line(1, 1 => 1)
+      v.line(1, 1 => 2)
+      v.line(1, 2 => 1)
+      v.line(1, 2 => nil)
+      v.line(1, 3 => '2010-01-01')
+      v.line(1, 3 => '2011-01-01')
+      v.line(1, 4 => true)
+      v.line(1, 5 => true)
+      v.line(1, 5 => false)
+      v.line(1, 5 => nil)
+      v.line(1, 'foo' => 'bar')
+      v.line(1, 'foo' => 'biz')
+      v.line(1, 'foo' => 'baz')
+      v.stats_report.should == [
+        'Feature | Class                       | Cardinality',
+        '---------------------------------------------------',
+        '1       | VectorEmbed::Maker::Number  |           1',
+        '2       | VectorEmbed::Maker::Number  |           1',
+        '3       | VectorEmbed::Maker::Date    |           1',
+        '4       | VectorEmbed::Maker::Boolean |           1',
+        '5       | VectorEmbed::Maker::Boolean |           3',
+        'foo     | VectorEmbed::Maker::Phrase  |           3',
+        '                                                 10',
+        '',
+      ].join("\n")
+    end
   end
 
   private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vector_embed
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-14 00:00:00.000000000 Z
+date: 2013-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3