vector_embed 0.1.0 → 0.1.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YWRiOGI3YTdkMDIwNDljNTA0NDkzOGNiNGVkNThiZjczNjdlNDU3OQ==
+  data.tar.gz: !binary |-
+    ZDIxY2Q1NDFjMjkxNDFkOGJkZTk0NTZiMDc4NTgwNjYwZGE5MDI1MQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YzAwOGQ0NjlmNThiYzNmZmQxNDk0ZTY2ZTIxMTA2M2NjMjZiNGE4MGU0NzE3
+    YTZhNTkzM2ViMWU2M2FhZDE5NTk5YTdkOWU3NGYxYjVjNmJkYmZjNGVhMTU0
+    ODY1OTkzNmFhNDIwODY4MjUwYTJjODU0ZDgzNWYzNmE1ZTljYzg=
+  data.tar.gz: !binary |-
+    ZjM3OGUyNDk1YzIxMjc5OTgyYmI3ZWY3NGEwYjRjYWY1MTk0MjJhYzhjODU0
+    NTg0OWYxNWU4ZTQ3YTQ0ZDFlOWVjYzMyYmZhOTE0MjNlMDRjMjMyMDkwNGMy
+    NDNhMjhkYjUzNGRhZmRhNjliMGFiNzY1NmNiNzk5MDhhOTUyOTU=

data/CHANGELOG

@@ -1,3 +1,10 @@
+0.1.1 / 2013-04-04
+
+* Enhancements
+
+  * If you pass a Hash as :dict, it will start feature labels at 1 rather than murmurhashing; you can then serialize VectorEmbed#dict
+  * Better error message if you run csv2libsvm on a CSV without a "label" column
+
 0.1.0 / 2013-02-20
 
 * Enhancements

data/bin/csv2libsvm

@@ -12,6 +12,8 @@ csv_path = ARGV[0]
 v = VectorEmbed.new
 CSV.foreach(csv_path, headers: :first_row) do |row|
   features = row.to_hash
-  label = features.delete('label')
+  unless label = features.delete('label')
+    raise "No label found - do you have a column named 'label'?"
+  end
   puts v.line(label, features)
 end

data/lib/vector_embed/maker/boolean.rb

@@ -4,7 +4,7 @@ class VectorEmbed
   class Maker
     class Boolean < Maker
       class << self
-        def want?(k, v, parent)
+        def want?(v, parent)
           case v
           when NilClass, TrueClass, FalseClass, TRUE, FALSE, T, F, NULL, SLASH_N
             true
@@ -28,11 +28,11 @@ class VectorEmbed
       def pairs(v)
         case v
         when TrueClass, TRUE, T
-          [ [ Maker.index(k, 'true'), 1 ] ]
+          [ [ parent.index([k, 'true']), 1 ] ]
         when FalseClass, FALSE, F
-          [ [ Maker.index(k, 'false'), 1 ] ]
+          [ [ parent.index([k, 'false']), 1 ] ]
         when NilClass, NULL, SLASH_N, BLANK
-          [ [ Maker.index(k, 'null'), 1 ] ]
+          [ [ parent.index([k, 'null']), 1 ] ]
         else
           raise ArgumentError, "Can't embed #{v.inspect} in boolean feature #{k.inspect}"
         end

data/lib/vector_embed/maker/ngram.rb

@@ -4,7 +4,7 @@ class VectorEmbed
   class Maker
     class Ngram < Maker
       class << self
-        def want?(k, v, parent)
+        def want?(v, parent)
           parent.options[:ngram_len]
         end
       end
@@ -31,7 +31,7 @@ class VectorEmbed
         else
           raise "Word n-gram not supported yet"
         end.map do |ngram|
-          [ [ Maker.index(k, 'ngram', ngram), 1 ] ]
+          [ [ parent.index([k, 'ngram', ngram]), 1 ] ]
         end
       end
     end

data/lib/vector_embed/maker/number.rb

@@ -4,7 +4,7 @@ class VectorEmbed
   class Maker
     class Number < Maker
       class << self
-        def want?(k, v, parent)
+        def want?(v, parent)
           v.is_a?(::Numeric) or v =~ JUST_A_NUMBER
         end
 

data/lib/vector_embed/maker/phrase.rb

@@ -4,14 +4,14 @@ class VectorEmbed
   class Maker
     class Phrase < Maker
       class << self
-        def want?(k, v, parent)
+        def want?(v, parent)
           true
         end
       end
 
       def pairs(v)
         v = parent.preprocess v.to_s
-        [ [ Maker.index(k, v), 1 ] ]
+        [ [ parent.index([k, v]), 1 ] ]
       end
     end
   end

data/lib/vector_embed/maker.rb

@@ -3,23 +3,17 @@ require 'vector_embed/maker/ngram'
 require 'vector_embed/maker/number'
 require 'vector_embed/maker/boolean'
 
-require 'murmurhash3'
-
 class VectorEmbed
   class Maker
     class << self
       def pick(choices, k, first_v, parent)
-        if klass = choices.detect { |klass| klass.want?(k, first_v, parent) }
+        if klass = choices.detect { |klass| klass.want?(first_v, parent) }
           parent.logger.debug { "Interpreting #{k.inspect} as #{klass.name.split('::').last} given first value #{first_v.inspect}" }
           klass.new k, parent
         else
           raise "Can't use #{first_v.class} for #{k.inspect} given #{first_v.inspect} and choices #{choices.inspect}"
         end
       end
-
-      def index(*parts)
-        MurmurHash3::V32.str_hash(parts.join(NULL_BYTE)).to_s[0..6].to_i
-      end
     end
 
     attr_reader :parent
@@ -35,11 +29,11 @@ class VectorEmbed
       when Array
         memo = []
         v.each_with_index do |vv, i|
-          memo << [ Maker.index(k, i), value(vv) ]
+          memo << [ parent.index([k, i]), value(vv) ]
         end
         memo
       else
-        [ [ Maker.index(k), value(v) ] ]
+        [ [ parent.index([k]), value(v) ] ]
       end
     end
   end

data/lib/vector_embed/version.rb

@@ -1,3 +1,3 @@
 class VectorEmbed
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/vector_embed.rb

@@ -1,4 +1,6 @@
 require 'logger'
+require 'digest/md5'
+require 'murmurhash3'
 
 require 'vector_embed/version'
 require 'vector_embed/maker'
@@ -16,15 +18,21 @@ class VectorEmbed
   FALSE = /\Afalse\z/i
   F = /\Af\z/i
   NULL_BYTE = "\x00"
+  LABEL_MAKERS =   [Maker::Boolean, Maker::Number]
+  FEATURE_MAKERS = [Maker::Boolean, Maker::Number, Maker::Ngram, Maker::Phrase]
 
   attr_reader :options
   attr_accessor :logger
+  attr_reader :dict
 
   def initialize(options = {})
+    @options = options.dup
     @mutex = Mutex.new
     @feature_makers = {}
     @logger = options[:logger] || (l = Logger.new($stderr); l.level = Logger::INFO; l)
-    @options = options.dup
+    if dict = @options.delete(:dict)
+      @dict = dict.dup
+    end
   end
 
   def line(label, features = {})
@@ -50,6 +58,18 @@ class VectorEmbed
     StopWord.remove stop_words, v
   end
 
+  def index(parts)
+    k = parts.join NULL_BYTE
+    if dict
+      k = Digest::MD5.digest k
+      dict[k] || @mutex.synchronize do
+        dict[k] ||= dict.length + 1
+      end
+    else
+      MurmurHash3::V32.str_hash(k).to_s[0..6].to_i
+    end
+  end
+
   private
 
   def stop_words
@@ -60,13 +80,13 @@ class VectorEmbed
 
   def label_maker(label)
     @label_maker || @mutex.synchronize do
-      @label_maker ||= Maker.pick([Maker::Boolean, Maker::Number], 'label', label, self)
+      @label_maker ||= Maker.pick(LABEL_MAKERS, 'label', label, self)
     end
   end
 
   def feature_maker(k, v)
     @feature_makers[k] || @mutex.synchronize do
-      @feature_makers[k] ||= Maker.pick([Maker::Boolean, Maker::Number, Maker::Ngram, Maker::Phrase], k, v, self)
+      @feature_makers[k] ||= Maker.pick(FEATURE_MAKERS, k, v, self)
     end
   end
 end

data/spec/vector_embed_spec.rb

@@ -31,6 +31,29 @@ describe VectorEmbed do
     end
   end
 
+  describe 'using a dictionary' do
+    it "starts at feature label 1" do
+      v = VectorEmbed.new dict: {}
+      v.line(1, 'foo' => 5).should == "1 1:5"
+      v.line(1, 'bar' => 3).should == "1 2:3"
+      v.line(1, 'foo' => 3, 'bar' => 5).should == "1 1:3 2:5"
+    end
+
+    it "does not modify the original dict" do
+      orig = {}
+      v = VectorEmbed.new dict: orig
+      v.line(1, 'foo' => 5)
+      orig.should == {}
+    end
+
+    it "provides the latest dict on demand" do
+      require 'digest/md5'
+      v = VectorEmbed.new dict: {}
+      v.line(1, 'foo' => 5)
+      v.dict.should == { Digest::MD5.digest('foo') => 1 }
+    end
+  end
+
   # aka dimension indexes
   describe 'in feature keys' do
     it "stores values as their string equivalents" do

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: vector_embed
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.1.1
 platform: ruby
 authors:
 - Seamus Abshere
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-22 00:00:00.000000000 Z
+date: 2013-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -106,27 +97,26 @@ files:
 - vector_embed.gemspec
 homepage: https://github.com/seamusabshere/vector_embed
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Vector embedding of strings, booleans, numerics, and arrays into LIBSVM /
   LIBLINEAR format.
 test_files: