vector_embed 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/CHANGELOG

@@ -0,0 +1,3 @@
+0.0.1 / 2013-02-20
+
+* First release!

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in vector_embed.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Seamus Abshere
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Seamus Abshere
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,111 @@
+# VectorEmbed
+
+Vector embedding of strings, booleans, numerics, and arrays into [LIBSVM](http://www.csie.ntu.edu.tw/~cjlin/libsvm/) / [LIBLINEAR](http://www.csie.ntu.edu.tw/~cjlin/liblinear/) format.
+
+Inspired by [Sally](http://www.mlsec.org/sally/), except `VectorEmbed` is meant to handle categorical and continuous data at the same time.
+
+## Usage
+
+Create a `VectorEmbed` instance, which auto-detects and then remembers what kind of data goes into each feature:
+
+    >> require 'vector_embed'
+    => true
+    >> v = VectorEmbed.new
+    => #<VectorEmbed:0x007fd605815208 [...]>
+
+Output a line with a label and arbitrary features:
+
+    >> label = 1
+    => 1
+    >> features = { color: 'red', year: 1995, weight: 5.4e9 }
+    => {:color=>"red", :year=>1995, :weight=>5400000000.0}
+    >> v.line(label, features)
+    => "1 1997960:1 5556418:5400000000.0 8227451:1995"
+
+Output another line:
+
+    >> label = 0
+    => 0
+    >> features = { color: 'blue', year: 1821, weight: 3.3 }
+    => {:color=>"blue", :year=>1821, :weight=>3.3}
+    >> v.line(label, features)
+    => "0 1089740:1 5556418:3.3 8227451:1821"
+
+Note that `color: 'red'` and `color: 'blue'` are being translated into categories:
+
+    1997960:1 # murmur3("color\x00red"):1
+    1089740:1 # murmur3("color\x00blue"):1
+
+A similar thing happens with `true`/`false`:
+
+    >> v.line(1, yes: true, no: false)
+    => "1 1559987:1 3324244:1"
+
+i.e.
+
+    1559987:1 # murmur3("yes\x00true"):1
+    3324244:1 # murmur3("no\x00false"):1
+
+## N-grams
+
+Currently uses same parameter names as [Sally](http://www.mlsec.org/sally/manual.html).
+
+### Word ngrams
+
+    >> v = VectorEmbed.new ngram_len: 1, ngram_delim: /\s+/, stop_words: %w{the and or}
+    => #<VectorEmbed:0x007fd6033b77f8 [...]>
+    >> v.line(1, notes: 'the quick brown fox')
+    => "1 1512788:1 3426202:1 5079692:1"
+
+You get the idea: ("the" has been filtered out by stop words)
+
+    1512788:1 # murmur3("notes\x00quick"):1
+    3426202:1 # murmur3("notes\x00brown"):1
+    5079692:1 # murmur3("notes\x00fox"):1
+
+### Byte n-grams
+
+    >> v = VectorEmbed.new ngram_len: 2, ngram_delim: ''
+    => #<VectorEmbed:0x007fd60337ea20 [...]>
+    >> v.line(1, notes: 'foobar')
+    => "1 2148745:1 2878919:1 3600333:1 3621715:1 5885921:1"
+
+So therefore:
+
+    2148745:1 # murmur3("notes\x00fo"):1
+    2878919:1 # murmur3("notes\x00oo"):1
+    3600333:1 # murmur3("notes\x00ob"):1
+    3621715:1 # murmur3("notes\x00ba"):1
+    5885921:1 # murmur3("notes\x00ar"):1
+
+## Debugging
+
+`VectorEmbed` tries to do the right thing, but if it's not, try turning on debugging:
+
+    >> v = VectorEmbed.new
+    => #<VectorEmbed:0x007fd6034020a0 [...]>
+    >> v.logger.level = Logger::DEBUG
+    => 0
+    >> v.line(1, '3' => 7, foo: 'bar', truthy: false, nullity: nil)
+    D, [2013-02-20T16:55:00.139299 #21595] DEBUG -- : Interpreting "3" as Number given first value 7
+    D, [2013-02-20T16:55:00.139561 #21595] DEBUG -- : Interpreting :foo as Phrase given first value "bar"
+    D, [2013-02-20T16:55:00.139671 #21595] DEBUG -- : Interpreting :truthy as Boolean given first value false
+    D, [2013-02-20T16:55:00.139755 #21595] DEBUG -- : Interpreting :nullity as Boolean given first value nil
+    D, [2013-02-20T16:55:00.139872 #21595] DEBUG -- : Interpreting "label" as Number given first value 1
+    => "1 2647413:7 4091306:1 7123386:1 9259635:1"
+
+One thing it doesn't like: (assuming you have already performed the lines above)
+
+    >> v.line(1, '3' => 'bar')
+    ArgumentError: Can't embed "bar" in number feature "3".
+
+It's saying that, given you first passed it `7`, it thought `"3"` was a feature that held numbers.
+
+## Gotchas
+
+* Following Sally, it only uses the first 22 bits of the murmur hash for feature indices... more and LIBSVM seems to choke.
+* Stop words are currently filtered out of feature indices... probably shouldn't be.
+
+## Copyright
+
+Copyright 2013 Seamus Abshere

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'yard'
+YARD::Rake::YardocTask.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+task :default => :spec

data/bin/vector_embed

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'vector_embed'

data/lib/vector_embed.rb

@@ -0,0 +1,66 @@
+require 'logger'
+
+require 'vector_embed/version'
+require 'vector_embed/maker'
+
+require 'vector_embed/stop_word'
+
+class VectorEmbed
+  # http://stackoverflow.com/questions/638565/parsing-scientific-notation-sensibly
+  JUST_A_NUMBER = /\A\s*[+\-]?(?:0|[1-9]\d*)(?:\.\d*)?(?:[eE][+\-]?\d+)?\s*\z/
+  BLANK = /\A\s*\z/
+  NULL_BYTE = "\x00"
+
+  attr_reader :options
+  attr_accessor :logger
+
+  def initialize(options = {})
+    @mutex = Mutex.new
+    @feature_makers = {}
+    @logger = options[:logger] || (l = Logger.new($stderr); l.level = Logger::INFO; l)
+    @options = options.dup
+  end
+
+  def line(label, features = {})
+    feature_pairs = features.inject([]) do |memo, (k, v)|
+      case v
+      when Array
+        v.each_with_index do |vv, i|
+          memo.concat feature_maker([k, i].join(NULL_BYTE), vv).pairs(vv)
+        end
+      else
+        memo.concat feature_maker(k, v).pairs(v)
+      end
+      memo
+    end.compact.sort_by do |k_value, _|
+      k_value
+    end.map do |pair|
+      pair.join ':'
+    end
+    ([label_maker(label).value(label)] + feature_pairs).join ' '
+  end
+
+  def preprocess(v)
+    StopWord.remove stop_words, v
+  end
+
+  private
+
+  def stop_words
+    @stop_words ||= options.fetch(:stop_words, []).map do |raw_stop_word|
+      StopWord.new raw_stop_word
+    end
+  end
+
+  def label_maker(label)
+    @label_maker || @mutex.synchronize do
+      @label_maker ||= Maker.pick([Maker::Boolean, Maker::Number], 'label', label, self)
+    end
+  end
+
+  def feature_maker(k, v)
+    @feature_makers[k] || @mutex.synchronize do
+      @feature_makers[k] ||= Maker.pick([Maker::Boolean, Maker::Number, Maker::Ngram, Maker::Phrase], k, v, self)
+    end
+  end
+end

data/lib/vector_embed/maker.rb

@@ -0,0 +1,46 @@
+require 'vector_embed/maker/phrase'
+require 'vector_embed/maker/ngram'
+require 'vector_embed/maker/number'
+require 'vector_embed/maker/boolean'
+
+require 'murmurhash3'
+
+class VectorEmbed
+  class Maker
+    class << self
+      def pick(choices, k, first_v, parent)
+        if klass = choices.detect { |klass| klass.want?(k, first_v, parent) }
+          parent.logger.debug { "Interpreting #{k.inspect} as #{klass.name.split('::').last} given first value #{first_v.inspect}" }
+          klass.new k, parent
+        else
+          raise "Can't use #{first_v.class} for #{k.inspect} given #{first_v.inspect} and choices #{choices.inspect}"
+        end
+      end
+
+      def index(*parts)
+        MurmurHash3::V32.str_hash(parts.join(NULL_BYTE)).to_s[0..6].to_i
+      end
+    end
+
+    attr_reader :parent
+    attr_reader :k
+
+    def initialize(k, parent)
+      @k = k
+      @parent = parent
+    end
+
+    def pairs(v)
+      case v
+      when Array
+        memo = []
+        v.each_with_index do |vv, i|
+          memo << [ Maker.index(k, i), value(vv) ]
+        end
+        memo
+      else
+        [ [ Maker.index(k), value(v) ] ]
+      end
+    end
+  end
+end

data/lib/vector_embed/maker/boolean.rb

@@ -0,0 +1,42 @@
+require 'vector_embed/maker'
+
+class VectorEmbed
+  class Maker
+    class Boolean < Maker
+      class << self
+        def want?(k, v, parent)
+          case v
+          when NilClass, TrueClass, FalseClass, 'true', 'false', 'null'
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def value(v)
+        case v
+        when TrueClass, 'true', 't', 'yes', 'on'
+          1
+        when FalseClass, 'false', 'f', 'no', 'off'
+          0
+        else
+          raise "Can't embed #{v.inspect} in boolean feature #{k.inspect}"
+        end
+      end
+
+      def pairs(v)
+        case v
+        when TrueClass, 'true', 't', 'yes', 'on'
+          [ [ Maker.index(k, 'true'), 1 ] ]
+        when FalseClass, 'false', 'f', 'no', 'off'
+          [ [ Maker.index(k, 'false'), 1 ] ]
+        when NilClass, 'null', BLANK
+          [ [ Maker.index(k, 'null'), 1 ] ]
+        else
+          raise ArgumentError, "Can't embed #{v.inspect} in boolean feature #{k.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/vector_embed/maker/ngram.rb

@@ -0,0 +1,39 @@
+require 'vector_embed/maker'
+
+class VectorEmbed
+  class Maker
+    class Ngram < Maker
+      class << self
+        def want?(k, v, parent)
+          parent.options[:ngram_len]
+        end
+      end
+
+      attr_reader :len
+      attr_reader :delim
+
+      def initialize(k, parent)
+        super
+        @len = parent.options[:ngram_len].to_i
+        raise ArgumentError, ":ngram_len must be > 0" unless @len > 0
+        @delim = parent.options[:ngram_delim]
+      end
+
+      def pairs(v)
+        raise "Ngram can't handle #{v.inspect}, only a single string for now" unless v.is_a?(String)
+        v = parent.preprocess v.to_s
+        if len == 1
+          # word mode
+          v.split delim
+        elsif delim == ''
+          # byte mode
+          (0..v.length-len).map { |i| v[i,len] }
+        else
+          raise "Word n-gram not supported yet"
+        end.map do |ngram|
+          [ [ Maker.index(k, 'ngram', ngram), 1 ] ]
+        end
+      end
+    end
+  end
+end

data/lib/vector_embed/maker/number.rb

@@ -0,0 +1,37 @@
+require 'vector_embed/maker'
+
+class VectorEmbed
+  class Maker
+    class Number < Maker
+      class << self
+        def want?(k, v, parent)
+          v.is_a?(::Numeric) or v =~ JUST_A_NUMBER
+        end
+
+        def numify(v)
+          num = if v.is_a?(String)
+            if v.include?('.') or v.include?('e')
+              v.to_f
+            else
+              v.to_i
+            end
+          else
+            v
+          end
+          num > 1e10 ? ('%.10e' % num) : num
+        end
+      end
+
+      def value(v)
+        case v
+        when Numeric, JUST_A_NUMBER
+          Number.numify v
+        when NilClass
+          0
+        else
+          raise ArgumentError, "Can't embed #{v.inspect} in number feature #{k.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/vector_embed/maker/phrase.rb

@@ -0,0 +1,18 @@
+require 'vector_embed/maker'
+
+class VectorEmbed
+  class Maker
+    class Phrase < Maker
+      class << self
+        def want?(k, v, parent)
+          true
+        end
+      end
+
+      def pairs(v)
+        v = parent.preprocess v.to_s
+        [ [ Maker.index(k, v), 1 ] ]
+      end
+    end
+  end
+end

data/lib/vector_embed/stop_word.rb

@@ -0,0 +1,24 @@
+require 'vector_embed'
+
+class VectorEmbed
+  class StopWord
+    class << self
+      def remove(stop_words, str)
+        memo = str.dup
+        stop_words.each do |stop_word|
+          stop_word.apply! memo
+        end
+        memo.gsub! /\s+/, ' '
+        memo.strip!
+        memo
+      end
+    end
+
+    def initialize(raw_stop_word)
+      @pattern = /\s*\b#{raw_stop_word}\b\s*/i
+    end
+    def apply!(str)
+      str.gsub! @pattern, ' '
+    end
+  end
+end

data/lib/vector_embed/version.rb

@@ -0,0 +1,3 @@
+class VectorEmbed
+  VERSION = "0.0.1"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require 'pry'
+
+require 'vector_embed'

data/spec/vector_embed_spec.rb

@@ -0,0 +1,219 @@
+require 'spec_helper'
+
+describe VectorEmbed do
+  describe 'in labels' do
+    it "stores true/false as 1/0" do
+      v = VectorEmbed.new
+      v.line(true).should == '1'
+      v.line(false).should == '0'
+      v.line('true').should == '1'
+      v.line('false').should == '0'
+    end
+
+    it "stores numbers as numbers" do
+      v = VectorEmbed.new
+      v.line(5.4).should == '5.4'
+      v.line(-3.9).should == '-3.9'
+    end
+
+    it "doesn't allow strings" do
+      v = VectorEmbed.new
+      lambda { v.line('foo') }.should raise_error(/string.*label/i)
+    end
+
+    it "doesn't allow mixing" do
+      v = VectorEmbed.new
+      v.line(5.4)
+      lambda { v.line(true) }.should raise_error(/Can't embed.*number/)
+      v = VectorEmbed.new
+      v.line(true)
+      lambda { v.line(5.4) }.should raise_error(/Can't embed.*boolean/)
+    end
+  end
+
+  # aka dimension indexes
+  describe 'in feature keys' do
+    it "stores values as their string equivalents" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 9).should == "1 #{l_h('1')}:9"
+      v.line(1, 5.4 => 9).should == "1 #{l_h('5.4')}:9"
+      v.line(1, '5.4' => 9).should == "1 #{l_h('5.4')}:9"
+      v.line(1, '5.4 ' => 9).should == "1 #{l_h('5.4 ')}:9"
+      v.line(1, 'foo' => 9).should == "1 #{l_h('foo')}:9"
+      v.line(1, 'foo bar' => 9).should == "1 #{l_h('foo bar')}:9"
+      v.line(1, true => 9).should == "1 #{l_h('true')}:9"
+      v.line(1, 'true' => 9).should == "1 #{l_h('true')}:9"
+      v.line(1, false => 9).should == "1 #{l_h('false')}:9"
+      v.line(1, 'false' => 9).should == "1 #{l_h('false')}:9"
+    end
+
+    it "treats nil as a blank string" do
+      v = VectorEmbed.new
+      v.line(1, nil => 9).should == "1 #{l_h('')}:9"
+    end
+
+    it "leaves whitespace alone" do
+      v = VectorEmbed.new
+      v.line(1, '' => 9).should == "1 #{l_h('')}:9"
+      v.line(1, ' ' => 9).should == "1 #{l_h(' ')}:9"
+      v.line(1, '  ' => 9).should == "1 #{l_h('  ')}:9"
+      v.line(1, ' foo ' => 9).should == "1 #{l_h(' foo ')}:9"
+      v.line(1, '5.4 ' => 9).should == "1 #{l_h('5.4 ')}:9"
+    end
+
+    it "orders feature names" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 3, 2 => 7).should == "1 #{l_h('2')}:7 #{l_h('1')}:3"
+    end
+
+    it "allows mixed string and number feature values" do
+      v = VectorEmbed.new
+      v.line(1, a: :b).should == "1 #{l_h("a\x00b")}:1"
+      v.line(1, a: 13).should == "1 #{l_h("a\x0013")}:1"
+      v.line(1, 1 => 9).should == "1 #{l_h('1')}:9" # 9 is not hashed, 1 is
+    end
+  end
+
+  describe 'feature values' do
+    it "stores true/false/nil as (1,0,0)/(0,1,0)/(0,0,1)" do
+      v = VectorEmbed.new
+      v.line(1, 1 => true).should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => 'true').should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => false).should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => 'false').should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => nil).should == "1 #{l_h("1\x00null")}:1"
+      v.line(1, 1 => 'null').should == "1 #{l_h("1\x00null")}:1"
+    end
+
+    it "stores numbers as numbers" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 9).should == "1 #{l_h('1')}:9"
+      v.line(1, 1 => '9').should == "1 #{l_h('1')}:9"
+      v.line(1, 1 => 5.4).should == "1 #{l_h('1')}:5.4"
+      v.line(1, 1 => '5.4').should == "1 #{l_h('1')}:5.4"
+      v.line(1, 1 => 9e9).should == "1 #{l_h('1')}:9000000000.0"
+      v.line(1, 1 => '9e9').should == "1 #{l_h('1')}:9000000000.0"
+    end
+
+    it "stores strings as m-category attributes" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 'sfh').should == "1 #{l_h("1\x00sfh")}:1"
+      v.line(1, 1 => 'mfh').should == "1 #{l_h("1\x00mfh")}:1"
+      v.line(1, 1 => 'foo bar').should == "1 #{l_h("1\x00foo bar")}:1"
+      v.line(1, 1 => 'foo  bar ').should == "1 #{l_h("1\x00foo bar")}:1"
+      v.line(1, 1 => ' foo   bar  ').should == "1 #{l_h("1\x00foo bar")}:1"
+    end
+
+    it "in string mode, treats true/false/nil as strings" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 'foo').should == "1 #{l_h("1\x00foo")}:1"
+      v.line(1, 1 => true).should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => false).should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => nil).should == "1 #{l_h("1\x00")}:1"
+    end
+
+    it "in string mode, treats numbers as strings" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 'foo').should == "1 #{l_h("1\x00foo")}:1"
+      v.line(1, 1 => 1).should == "1 #{l_h("1\x001")}:1"
+      v.line(1, 1 => 5.4).should == "1 #{l_h("1\x005.4")}:1"
+      v.line(1, 1 => 9e9).should == "1 #{l_h("1\x00" + 9e9.to_s)}:1"
+    end
+
+    it "flattens and stores arrays" do
+      v = VectorEmbed.new
+      v.line(1, 'foo' => [7,13,19]).should == sortme("1 #{l_h("foo\x001")}:13 #{l_h("foo\x000")}:7 #{l_h("foo\x002")}:19")
+      v.line(1, 'bar' => ['a','b','c']).should == sortme("1 #{l_h("bar\x001\x00b")}:1 #{l_h("bar\x000\x00a")}:1 #{l_h("bar\x002\x00c")}:1")
+    end
+
+    it "in number mode, treats null as 0" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 9).should == "1 #{l_h('1')}:9"
+      v.line(1, 1 => nil).should == "1 #{l_h('1')}:0"
+    end
+
+    it "doesn't allow embedding boolean in number mode or vice-versa" do
+      v = VectorEmbed.new
+      v.line(1, 1 => true)
+      v.line(1, 2 => 5.4) # that's fine, different dimension
+      lambda { v.line(1, 1 => 5.4) }.should raise_error(ArgumentError)
+      v = VectorEmbed.new
+      v.line(1, 1 => 5.4)
+      v.line(1, 2 => true) # that's fine, diff dim
+      lambda { v.line(1, 1 => true) }.should raise_error(ArgumentError)
+    end
+
+    it "doesn't allow embedding string in number mode" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 9)
+      v.line(1, 2 => 'foo') # that's fine, different dimension
+      lambda { v.line(1, 1 => 'foo') }.should raise_error(ArgumentError)
+    end
+
+    it "uses scientific notation for large numbers" do
+      v = VectorEmbed.new
+      v.line(5, 1 => 8.12e13).should == "5 #{l_h('1')}:8.1200000000e+13"
+    end
+
+    it "detects numbers in strings" do
+      v = VectorEmbed.new
+      v.line(5, 1 => '8.12e13').should == "5 #{l_h('1')}:8.1200000000e+13"
+    end
+
+    it "allows 2 byte n-grams" do
+      v = VectorEmbed.new ngram_len: 2, ngram_delim: ''
+      v.line(1, 1 => 'foo').should == sortme("1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1")
+      v.line(1, 1 => 'bar').should == sortme("1 #{l_h("1\x00ngram\x00ar")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'baz').should == sortme("1 #{l_h("1\x00ngram\x00az")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'foobar').should == sortme("1 #{l_h("1\x00ngram\x00ar")}:1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1 #{l_h("1\x00ngram\x00ob")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'foo bar').should == sortme("1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1 #{l_h("1\x00ngram\x00o ")}:1 #{l_h("1\x00ngram\x00 b")}:1 #{l_h("1\x00ngram\x00ba")}:1 #{l_h("1\x00ngram\x00ar")}:1")
+    end
+
+    it "allows word-grams" do
+      v = VectorEmbed.new ngram_len: 1, ngram_delim: /\s+/
+      v.line(1, 1 => 'foo').should == sortme("1 #{l_h("1\x00ngram\x00foo")}:1")
+      v.line(1, 1 => 'foobar').should == sortme("1 #{l_h("1\x00ngram\x00foobar")}:1")
+      v.line(1, 1 => 'foo bar').should == sortme("1 #{l_h("1\x00ngram\x00bar")}:1 #{l_h("1\x00ngram\x00foo")}:1")
+    end
+
+    it "allows 2 byte n-grams with stop words" do
+      v = VectorEmbed.new ngram_len: 2, ngram_delim: '', stop_words: %w{the and or}
+      v.line(1, 1 => 'foo or').should == sortme("1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1")
+      v.line(1, 1 => 'the bar').should == sortme("1 #{l_h("1\x00ngram\x00ar")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'and baz').should == sortme("1 #{l_h("1\x00ngram\x00az")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'foobar or the and').should == sortme("1 #{l_h("1\x00ngram\x00ar")}:1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1 #{l_h("1\x00ngram\x00ob")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+      v.line(1, 1 => 'foo or and the bar').should == sortme("1 #{l_h("1\x00ngram\x00 b")}:1 #{l_h("1\x00ngram\x00ar")}:1 #{l_h("1\x00ngram\x00fo")}:1 #{l_h("1\x00ngram\x00oo")}:1 #{l_h("1\x00ngram\x00o ")}:1 #{l_h("1\x00ngram\x00ba")}:1")
+    end
+
+    it "allows word-grams with stop words" do
+      v = VectorEmbed.new ngram_len: 1, ngram_delim: /\s+/, stop_words: %w{the and or}
+      v.line(1, 1 => 'foo or').should == "1 #{l_h("1\x00ngram\x00foo")}:1"
+      v.line(1, 1 => 'foo the bar').should == "1 #{l_h("1\x00ngram\x00bar")}:1 #{l_h("1\x00ngram\x00foo")}:1"
+      v.line(1, 1 => 'foo bar and').should == "1 #{l_h("1\x00ngram\x00bar")}:1 #{l_h("1\x00ngram\x00foo")}:1"
+    end
+
+    it "doesn't do anything weird when you have multiple features" do
+      v = VectorEmbed.new
+      v.line(1, 1 => 'foo', 2 => 'bar', 'baz' => 'zoo').should == sortme("1 #{l_h("1\x00foo")}:1 #{l_h("2\x00bar")}:1 #{l_h("baz\x00zoo")}:1")
+    end
+
+  end
+
+  private
+
+  def h(v)
+    MurmurHash3::V32.str_hash v
+  end
+
+  # for labels
+  def l_h(v)
+    h(v).to_s[0..6].to_i
+  end
+
+  def sortme(line)
+    parts = line.split(' ')
+    label = parts.shift
+    features = parts.map { |p| p.split(':') }.sort_by { |k, v| k.to_i }.map { |k, v| [k, v].join(':') }
+    ([label] + features).join(' ')
+  end
+end

data/vector_embed.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/vector_embed/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "vector_embed"
+  gem.version       = VectorEmbed::VERSION
+  gem.authors       = ["Seamus Abshere"]
+  gem.email         = ["seamus@abshere.net"]
+  gem.description   = %q{Vector embedding of strings, booleans, numerics, and arrays into LIBSVM / LIBLINEAR format.}
+  gem.summary       = %q{Vector embedding of strings, booleans, numerics, and arrays into LIBSVM / LIBLINEAR format.}
+  gem.homepage      = "https://github.com/seamusabshere/vector_embed"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'murmurhash3'
+  
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'yard'
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: vector_embed
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Seamus Abshere
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: murmurhash3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Vector embedding of strings, booleans, numerics, and arrays into LIBSVM
+  / LIBLINEAR format.
+email:
+- seamus@abshere.net
+executables:
+- vector_embed
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- CHANGELOG
+- Gemfile
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/vector_embed
+- lib/vector_embed.rb
+- lib/vector_embed/maker.rb
+- lib/vector_embed/maker/boolean.rb
+- lib/vector_embed/maker/ngram.rb
+- lib/vector_embed/maker/number.rb
+- lib/vector_embed/maker/phrase.rb
+- lib/vector_embed/stop_word.rb
+- lib/vector_embed/version.rb
+- spec/spec_helper.rb
+- spec/vector_embed_spec.rb
+- vector_embed.gemspec
+homepage: https://github.com/seamusabshere/vector_embed
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Vector embedding of strings, booleans, numerics, and arrays into LIBSVM /
+  LIBLINEAR format.
+test_files:
+- spec/spec_helper.rb
+- spec/vector_embed_spec.rb
+has_rdoc: