vector_embed 0.0.1 → 0.1.0

data/CHANGELOG

@@ -1,3 +1,14 @@
+0.1.0 / 2013-02-20
+
+* Enhancements
+
+  * csv2libsvm binary
+  * "null" and "NULL" are treated as nil (0 in number mode)
+
+* Breaking changes
+  
+  * yes', 'on' / 'no', 'off' no longer treated as true/false
+
 0.0.1 / 2013-02-20
 
 * First release!

data/README.md

@@ -4,6 +4,21 @@ Vector embedding of strings, booleans, numerics, and arrays into [LIBSVM](http:/
 
 Inspired by [Sally](http://www.mlsec.org/sally/), except `VectorEmbed` is meant to handle categorical and continuous data at the same time.
 
+## csv2libsvm
+
+    $ cat houses.csv 
+    label,household_income,zip_code_id,year
+    +1,35893,53703,1904
+    -1,43708,53711,1977
+    +1,103214,53719,NULL
+    -1,49250,53704,1950
+
+    $ csv2libsvm houses.csv 
+    1 1243483:35893 6439848:53703 8227451:1904
+    -1 1243483:43708 6439848:53711 8227451:1977
+    1 1243483:103214 6439848:53719 8227451:0
+    -1 1243483:49250 6439848:53704 8227451:1950
+
 ## Usage
 
 Create a `VectorEmbed` instance, which auto-detects and then remembers what kind of data goes into each feature:

data/bin/csv2libsvm

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+if File.exist?('Gemfile')
+  require 'bundler/setup'
+end
+
+require 'csv'
+require 'vector_embed'
+
+csv_path = ARGV[0]
+
+v = VectorEmbed.new
+CSV.foreach(csv_path, headers: :first_row) do |row|
+  features = row.to_hash
+  label = features.delete('label')
+  puts v.line(label, features)
+end

data/lib/vector_embed/maker/boolean.rb

@@ -6,7 +6,7 @@ class VectorEmbed
       class << self
         def want?(k, v, parent)
           case v
-          when NilClass, TrueClass, FalseClass, 'true', 'false', 'null'
+          when NilClass, TrueClass, FalseClass, TRUE, FALSE, T, F, NULL, SLASH_N
             true
           else
             false
@@ -16,9 +16,9 @@ class VectorEmbed
 
       def value(v)
         case v
-        when TrueClass, 'true', 't', 'yes', 'on'
+        when TrueClass, TRUE, T
           1
-        when FalseClass, 'false', 'f', 'no', 'off'
+        when FalseClass, FALSE, F
           0
         else
           raise "Can't embed #{v.inspect} in boolean feature #{k.inspect}"
@@ -27,11 +27,11 @@ class VectorEmbed
 
       def pairs(v)
         case v
-        when TrueClass, 'true', 't', 'yes', 'on'
+        when TrueClass, TRUE, T
           [ [ Maker.index(k, 'true'), 1 ] ]
-        when FalseClass, 'false', 'f', 'no', 'off'
+        when FalseClass, FALSE, F
           [ [ Maker.index(k, 'false'), 1 ] ]
-        when NilClass, 'null', BLANK
+        when NilClass, NULL, SLASH_N, BLANK
           [ [ Maker.index(k, 'null'), 1 ] ]
         else
           raise ArgumentError, "Can't embed #{v.inspect} in boolean feature #{k.inspect}"

data/lib/vector_embed/maker/number.rb

@@ -26,7 +26,7 @@ class VectorEmbed
         case v
         when Numeric, JUST_A_NUMBER
           Number.numify v
-        when NilClass
+        when NilClass, NULL, SLASH_N
           0
         else
           raise ArgumentError, "Can't embed #{v.inspect} in number feature #{k.inspect}"

data/lib/vector_embed/version.rb

@@ -1,3 +1,3 @@
 class VectorEmbed
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/vector_embed.rb

@@ -9,6 +9,12 @@ class VectorEmbed
   # http://stackoverflow.com/questions/638565/parsing-scientific-notation-sensibly
   JUST_A_NUMBER = /\A\s*[+\-]?(?:0|[1-9]\d*)(?:\.\d*)?(?:[eE][+\-]?\d+)?\s*\z/
   BLANK = /\A\s*\z/
+  NULL = /\Anull\z/i
+  SLASH_N = '\N'
+  TRUE = /\Atrue\z/i
+  T = /\At\z/i
+  FALSE = /\Afalse\z/i
+  F = /\Af\z/i
   NULL_BYTE = "\x00"
 
   attr_reader :options

data/spec/vector_embed_spec.rb

@@ -79,10 +79,18 @@ describe VectorEmbed do
       v = VectorEmbed.new
       v.line(1, 1 => true).should == "1 #{l_h("1\x00true")}:1"
       v.line(1, 1 => 'true').should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => 'TRUE').should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => 't').should == "1 #{l_h("1\x00true")}:1"
+      v.line(1, 1 => 'T').should == "1 #{l_h("1\x00true")}:1"
       v.line(1, 1 => false).should == "1 #{l_h("1\x00false")}:1"
       v.line(1, 1 => 'false').should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => 'FALSE').should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => 'f').should == "1 #{l_h("1\x00false")}:1"
+      v.line(1, 1 => 'F').should == "1 #{l_h("1\x00false")}:1"
       v.line(1, 1 => nil).should == "1 #{l_h("1\x00null")}:1"
       v.line(1, 1 => 'null').should == "1 #{l_h("1\x00null")}:1"
+      v.line(1, 1 => 'NULL').should == "1 #{l_h("1\x00null")}:1"
+      v.line(1, 1 => '\N').should == "1 #{l_h("1\x00null")}:1"
     end
 
     it "stores numbers as numbers" do
@@ -130,6 +138,9 @@ describe VectorEmbed do
       v = VectorEmbed.new
       v.line(1, 1 => 9).should == "1 #{l_h('1')}:9"
       v.line(1, 1 => nil).should == "1 #{l_h('1')}:0"
+      v.line(1, 1 => 'null').should == "1 #{l_h('1')}:0"
+      v.line(1, 1 => 'NULL').should == "1 #{l_h('1')}:0"
+      v.line(1, 1 => '\N').should == "1 #{l_h('1')}:0"
     end
 
     it "doesn't allow embedding boolean in number mode or vice-versa" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vector_embed
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-20 00:00:00.000000000 Z
+date: 2013-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: murmurhash3
@@ -80,7 +80,7 @@ description: Vector embedding of strings, booleans, numerics, and arrays into LI
 email:
 - seamus@abshere.net
 executables:
-- vector_embed
+- csv2libsvm
 extensions: []
 extra_rdoc_files: []
 files:
@@ -92,7 +92,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/vector_embed
+- bin/csv2libsvm
 - lib/vector_embed.rb
 - lib/vector_embed/maker.rb
 - lib/vector_embed/maker/boolean.rb

data/bin/vector_embed

@@ -1,3 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'vector_embed'