iterationlabs-rarff 0.2.5

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.2@rarff --create

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+gem 'shoulda'
+gem 'jeweler'

data/Gemfile.lock

@@ -0,0 +1,17 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2.2)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler
+  shoulda

data/History.txt

@@ -0,0 +1,18 @@
+== Changes
+
+=== 0.2.3
+
+* Added set_string_attributes_to_nominal for easy conversion of string attributes to nominal ones
+
+=== 0.2.2 (unofficial)
+
+* Handles boolean inputs, which are modelled as nominals
+* Handles spaces in nominals, which are replaced by underscores. Probably should be quoting these, but is good enough for me right now
+
+=== 0.2.1 (unofficial)
+
+* Handles missing data in output, encoded internally as nil values
+
+== 0.2.0 ?
+
+* Sparse ARFF files (thanks to Tom Adams)

data/Manifest.txt

@@ -0,0 +1,5 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/rarff.rb

data/README.txt

@@ -0,0 +1,90 @@
+= rarff
+
+http://adenserparlance.blogspot.com/2007/01/rarff-simple-arff-library-in-ruby.html
+
+== DESCRIPTION:
+
+Rarff is a Ruby library for dealing with Attribute-Relation File Format (ARFF) files. ARFF files are used to specify data sets for data mining and machine learning.
+
+
+== FEATURES/PROBLEMS:
+
+=== FEATURES
+* Missing values - '?' are handled in creation of ARFF files
+
+=== PROBLEMS
+* Spaces or quotes in nominal types
+* Commas in quoted attributes or in nominal types
+* Add error checking/validation
+* Creation of sparse ARFF files
+* Dates - do some work to create, translate, and interpret date format strings.
+
+== SYNOPSIS:
+
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 NUMERIC
+@ATTRIBUTE Attr3 STRING
+@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+@DATA
+1.4, 'foo bar', 5, baz, "1900-08-08 12:12:12"
+20.9, ruby, 46, rocks, "2005-10-23 12:12:12"
+0, ruby, 46, rocks, "2001-02-19 12:12:12"
+68.1, stuff, 728, 'is cool', "1974-02-10 12:12:12"
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [1.4, 'foo bar', 5, 'baz', "1900-08-08 12:12:12"],
+      [20.9, 'ruby', 46, 'rocks', "2005-10-23 12:12:12"],
+      [0, 'ruby', 46, 'rocks', "2001-02-19 12:12:12"],
+      [68.1, 'stuff', 728, 'is cool', "1974-02-10 12:12:12"]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[1].name = 'subject'
+    rel.attributes[4].name = 'birthday'
+    rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "Arff creation test failed.")
+
+== REQUIREMENTS:
+
+== INSTALL:
+
+* sudo gem install wwood-rarff
+
+== LICENSE:
+
+Copyright (c) 2008 Andy Payne
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+        * Redistributions of source code must retain the above copyright notice,
+        this list of conditions and the following disclaimer.
+        * Redistributions in binary form must reproduce the above copyright notice,
+        this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+        * Neither the name of the COPYRIGHT OWNER nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "iterationlabs-rarff"
+    gem.summary = %Q{Rarff is a Ruby library for dealing with Attribute-Relation File Format (ARFF) files}
+    gem.description = %Q{Rarff is a Ruby library for dealing with Attribute-Relation File Format (ARFF) files. ARFF files are used to specify 
+data sets for data mining and machine learning.}
+    gem.email = "donttrustben near gmail.com"
+    gem.homepage = "http://github.com/iterationlabs/rarff"
+    gem.authors = ["Ben J Woodcroft","Andy Payne", "Andrew Cantino"]
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :default => :test
+
+# task :test => :check_dependencies
+
+# require 'rdoc/task'
+# Rake::RDocTask.new do |rdoc|
+#   version = File.exist?('VERSION') ? File.read('VERSION') : ""
+# 
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title = "blah #{version}"
+#   rdoc.rdoc_files.include('README*')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end

data/VERSION

@@ -0,0 +1 @@
+0.2.5

data/lib/rarff.rb

@@ -0,0 +1,338 @@
+# = rarff
+
+# This is the top-level include file for rarff. See the README file for
+# details.
+
+################################################################################
+
+# Custom scan that returns a boolean indicating whether the regex matched.
+# TODO: Is there a way to avoid doing this?
+class String
+  def my_scan(re)
+    hit = false
+    scan(re) { |arr|
+      yield arr if block_given?
+      hit = true
+    }
+    hit
+  end
+end
+
+################################################################################
+
+module Enumerable
+  # This map_with_index hack allows access to the index of each item as the map
+  # iterates.
+  # TODO: Is there a better way?
+  def map_with_index
+    # Ugly, but I need the yield to be the last statement in the map.
+    i = -1 
+    return map { |item|
+      i += 1
+      yield item, i
+    }
+  end
+end
+
+################################################################################
+
+module Rarff
+
+  COMMENT_MARKER = '%'
+  RELATION_MARKER = '@RELATION'
+  ATTRIBUTE_MARKER = '@ATTRIBUTE'
+  DATA_MARKER = '@DATA'
+
+  SPARSE_ARFF_BEGIN = '{'
+  ESC_SPARSE_ARFF_BEGIN = '\\' + SPARSE_ARFF_BEGIN
+  SPARSE_ARFF_END = '}'
+  ESC_SPARSE_ARFF_END = '\\' + SPARSE_ARFF_END
+
+  ATTRIBUTE_NUMERIC = 'NUMERIC'
+  ATTRIBUTE_REAL = 'REAL'
+  ATTRIBUTE_INTEGER = 'INTEGER'
+  ATTRIBUTE_STRING = 'STRING'
+  ATTRIBUTE_DATE = 'DATE'
+  # Model Boolean as a Nominal Attribute.
+  # Use {false, true} not {true, false} because then in visualisations in Weka
+  # true is to the right, which makes more intuitive sense
+  ATTRIBUTE_BOOLEAN = '{false, true}'
+
+  MISSING = '?'
+
+  ################################################################################
+
+  class Attribute
+    attr_accessor :name
+    attr_reader :type
+
+    def initialize(name='', type='')
+      @name = name
+    
+      @type_is_nominal = false
+      @type = type
+
+      check_nominal()
+    end
+
+
+    def type=(type)
+      @type = type
+      check_nominal()
+    end
+
+
+    # Convert string representation of nominal type to array, if necessary
+    # TODO: This might falsely trigger on wacky date formats.
+    def check_nominal
+      if @type =~ /^\s*\{.*(\,.*)+\}\s*$/
+        @type_is_nominal = true
+        # Example format: "{nom1,nom2, nom3, nom4,nom5 } "
+        # Split on '{'  ','  or  '}'
+        #        @type = @type.gsub(/^\s*\{\s*/, '').gsub(/\s*\}\s*$/, '').split(/\s*\,\s*/)
+        @type = @type.split(/\s*\,\s*/)
+      end
+    end
+
+
+    def add_nominal_value(str)
+      if @type_is_nominal == false
+        @type = Array.new
+      end
+
+      @type << str
+    end
+
+
+    def to_arff
+      if @type_is_nominal == true
+        ATTRIBUTE_MARKER + " #{@name} #{@type.join(',').gsub(' ','_')}"
+      else
+        ATTRIBUTE_MARKER + " #{@name} #{@type}"
+      end
+    end
+
+
+    def to_s
+      to_arff
+    end
+
+  end
+
+
+
+  class Relation
+    attr_accessor :name, :attributes
+    attr_reader :instances
+
+
+    def initialize(name='')
+      @name = name
+      @attributes = Array.new
+      @instances = Array.new
+    end
+
+
+    def parse(str)
+      in_data_section = false
+
+      # TODO: Doesn't handle commas in quoted attributes.
+      str.split("\n").each { |line|
+        next if line =~ /^\s*$/
+        next if line =~ /^\s*#{COMMENT_MARKER}/
+        next if line.my_scan(/^\s*#{RELATION_MARKER}\s*(.*)\s*$/i) { |name| @name = name }
+        next if line.my_scan(/^\s*#{ATTRIBUTE_MARKER}\s*([^\s]*)\s+(.*)\s*$/i) { |name, type|
+          @attributes.push(Attribute.new(name, type))
+        }
+        next if line.my_scan(/^\s*#{DATA_MARKER}/i) { in_data_section = true }
+        next if in_data_section == false  ## Below is data section handling
+        #      next if line.gsub(/^\s*(.*)\s*$/, "\\1").my_scan(/^\s*#{SPARSE_ARFF_BEGIN}(.*)#{SPARSE_ARFF_END}\s*$/) { |data|
+        next if line.gsub(/^\s*(.*)\s*$/, "\\1").my_scan(/^#{ESC_SPARSE_ARFF_BEGIN}(.*)#{ESC_SPARSE_ARFF_END}$/) { |data|
+          # Sparse ARFF
+          # TODO: Factor duplication with non-sparse data below
+          @instances << expand_sparse(data.first)
+          create_attributes(true)
+        }
+        next if line.my_scan(/^\s*(.*)\s*$/) { |data|
+          @instances << data.first.split(/,\s*/).map { |field|
+            # Remove outer single quotes on strings, if any ('foo bar' --> foo bar)
+            field.gsub(/^\s*\'(.*)\'\s*$/, "\\1")
+          }
+          create_attributes(true)
+        }
+      }
+    end
+
+
+    # Assign instances to the internal array
+    # parse: choose to parse strings into numerics
+    def instances=(instances, parse=false)
+      @instances = instances
+      create_attributes(parse)
+    end
+
+
+    
+    def create_attributes(attr_parse=false)
+      raise Exception, "Not enough data to create ARFF attributes" if @instances.nil? or 
+        @instances.empty? or 
+        @instances[0].empty?
+      
+      # Keep track of whether an attribute has been defined or not.
+      # The only reason an attribute would not be defined in the first
+      # row is if it has nil's in it. The geek inside screams for a binary
+      # encoding like chmod but eh.
+      attributes_defined = {}
+      @instances.each_with_index { |row, i|
+        row.each_with_index { |col, j|
+          next if attributes_defined[j] or col.nil?
+          
+          attributes_defined[j] = true #whatever happens, we are going to define it
+          if attr_parse
+            if col =~ /^\-?\d+\.?\d*$/
+              @attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_NUMERIC)
+            end
+            next #parse next column - this one is finished
+          end
+          
+          # No parsing - just take it how it is
+          if col.kind_of?(Numeric)
+            @attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_NUMERIC)
+          elsif col.kind_of?(String)
+            @attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_STRING)
+          elsif col == false or col == true #exactly equal to a boolean
+            @attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_BOOLEAN)
+          else
+            raise Exception, "Could not parse attribute to ARFF data type: #{col.inspect}"
+          end
+        }
+      }
+      
+      # Make sure all attributes have a definition, because otherwise
+      # needless errors are thrown
+      @instances[0].each_index do |i|
+        @attributes[i] ||= Attribute.new("Attr#{i}", ATTRIBUTE_NUMERIC)
+      end
+    end
+
+    # Make all String type attributes into nominal attributes, because
+    # they are more useful in WEKA because more techniques handle them than
+    # strings.
+    #
+    # column_indices is an optional argumetn specifying the columns that
+    # are to be set to nominal (0 based indexes). if nil (the default), then
+    # all columns are included
+    def set_string_attributes_to_nominal(column_indices = nil)
+      nominals = {}
+      # Frustratingly, we have to traverse this 2D array with the
+      # wrong dimension first. Oh well.
+      @instances.each_with_index do |row, row_index|
+        row.each_with_index do |string, col_index|
+          next unless @attributes[col_index].type == ATTRIBUTE_STRING
+          next unless column_indices.nil? or column_indices.include?(col_index)
+
+          nominals[col_index] ||= {}
+          nominals[col_index][string] ||= true
+        end
+      end
+
+      nominals.each do |index, strings|
+        @attributes[index].type = "{#{strings.keys.join(',')}}"
+      end
+    end
+
+    def expand_sparse(str)
+      arr = Array.new(@attributes.size, 0)
+      str.gsub(/^\s*\{(.*)\}\s*$/, "\\1").split(/\s*\,\s*/).map { |pr|
+        pra = pr.split(/\s/)
+        arr[pra[0].to_i] = pra[1]
+      }
+      arr
+    end
+
+
+    def to_arff(sparse=false)
+      RELATION_MARKER + " #{@name}\n" +
+        @attributes.join("\n") +
+        "\n" +
+        DATA_MARKER + "\n" +
+        
+        @instances.map { |inst|
+        mapped = inst.map_with_index { |col, i|
+          # First pass - quote strings with spaces, and dates
+          # TODO: Doesn't handle cases in which strings already contain
+          # quotes or are already quoted.
+          unless col.nil?
+            if @attributes[i].type =~ /^#{ATTRIBUTE_STRING}$/i
+              if col =~ /[,\s]+/
+                col = "'" + col + "'"
+              end
+            elsif @attributes[i].type =~ /^#{ATTRIBUTE_DATE}/i  ## Hack comparison. Ugh.
+              col = '"' + col + '"'
+            end
+          end
+          
+          # Do the final output
+          if sparse
+            if col.nil? or
+                (@attributes[i].type =~ /^#{ATTRIBUTE_NUMERIC}$/i and col == 0)
+              nil
+            else
+              "#{i} #{col}"
+            end
+          else
+            if col.nil?
+              MISSING
+            else
+              col
+            end
+          end
+        }
+        
+        if sparse
+          mapped.reject{|col| col.nil?}.join(', ')
+        else
+          mapped.join(", ")
+        end
+      }.join("\n")
+    end
+
+
+    def to_s
+      to_arff
+    end
+
+  end
+
+
+end    # module Rarff
+
+################################################################################
+
+if $0 == __FILE__ then
+
+
+  if ARGV[0]
+    in_file = ARGV[0]
+    contents = ''
+
+    contents = File.open(in_file).read
+
+    rel = Rarff::Relation.new
+    rel.parse(contents)
+
+  else
+    exit
+  end
+
+  puts '='*80
+  puts '='*80
+  puts "ARFF:"
+  puts rel
+
+
+end
+
+################################################################################
+
+

data/test/test_arff.arff

@@ -0,0 +1,27 @@
+% 1. Title: Iris Plants Database
+% 
+% 2. Sources:
+%      (a) Creator: R.A. Fisher
+%      (b) Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov)
+%      (c) Date: July, 1988
+% 
+@RELATION iris
+
+@ATTRIBUTE sepallength  NUMERIC
+@ATTRIBUTE sepalwidth   NUMERIC
+@ATTRIBUTE petallength  NUMERIC
+@ATTRIBUTE petalwidth   NUMERIC
+@ATTRIBUTE class        {Iris-setosa,Iris-versicolor,Iris-virginica}
+
+@DATA
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3.0,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5.0,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5.0,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+

data/test/test_rarff.rb

@@ -0,0 +1,340 @@
+# See the README file for more information.
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+require 'test/unit'
+require 'rarff'
+
+class TestArffLib < Test::Unit::TestCase
+
+  # Test creation of an arff file string.
+  def test_arff_creation
+
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 NUMERIC
+@ATTRIBUTE Attr3 STRING
+@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+@DATA
+1.4, 'foo bar', 5, baz, "1900-08-08 12:12:12"
+20.9, ruby, 46, 'roc,ks', "2005-10-23 12:12:12"
+0, ruby, 46, rocks, "2001-02-19 12:12:12"
+68.1, stuff, 728, 'is cool', "1974-02-10 12:12:12"
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [1.4, 'foo bar', 5, 'baz', "1900-08-08 12:12:12"],
+      [20.9, 'ruby', 46, 'roc,ks', "2005-10-23 12:12:12"],
+      [0, 'ruby', 46, 'rocks', "2001-02-19 12:12:12"],
+      [68.1, 'stuff', 728, 'is cool', "1974-02-10 12:12:12"]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[1].name = 'subject'
+    rel.attributes[4].name = 'birthday'
+    rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "Arff creation test failed.")
+  end
+ 
+  #  # Test creation of a sparse arff file string.
+  #  def test_sparse_arff_creation
+  #
+  #    arff_file_str = <<-END_OF_ARFF_FILE
+  #@RELATION MyCoolRelation
+  #@ATTRIBUTE Attr0 NUMERIC
+  #@ATTRIBUTE subject STRING
+  #@ATTRIBUTE Attr2 NUMERIC
+  #@ATTRIBUTE Attr3 STRING
+  #@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+  #@DATA
+  #{0 1.4, 1 'foo bar', 3 baz, 4 "1900-08-08 12:12:12"}
+  #{0 20.9, 1 ruby, 2 46, 3 rocks, 4 "2005-10-23 12:12:12"}
+  #{1 ruby, 2 46, 3 rocks, 4 "2001-02-19 12:12:12"}
+  #{0 68.1, 1 stuff, 3 'is cool', 4 "1974-02-10 12:12:12"}
+  #    END_OF_ARFF_FILE
+  #
+  #    arff_file_str.gsub!(/\n$/, '')
+  #
+  #    instances = [ [1.4, 'foo bar', 0, 'baz', "1900-08-08 12:12:12"],
+  #      [20.9, 'ruby', 46, 'rocks', "2005-10-23 12:12:12"],
+  #      [0.0, 'ruby', 46, 'rocks', "2001-02-19 12:12:12"],
+  #      [68.1, 'stuff', 0, 'is cool', "1974-02-10 12:12:12"]]
+  #
+  #    rel = Rarff::Relation.new('MyCoolRelation')
+  #    rel.instances = instances
+  #    rel.attributes[1].name = 'subject'
+  #    rel.attributes[4].name = 'birthday'
+  #    rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+  #
+  #    #               puts "rel.to_arff(true):\n(\n#{rel.to_arff(true)}\n)\n"
+  #    assert_equal( arff_file_str, rel.to_arff(true), "test_sparse_arff_creation.")
+  #  end
+  #
+  #
+  #  # Test parsing of an arff file.
+  #  def test_arff_parse
+  #    in_file = './test_arff.arff'
+  #    rel = Rarff::Relation.new
+  #    rel.parse(File.open(File.join(File.dirname(__FILE__),in_file)).read)
+  #
+  #    assert_equal(rel.instances[2][1], 3.2)
+  #    assert_equal(rel.instances[7][4], 'Iris-setosa')
+  #  end
+  #
+  #
+  #  # Test parsing of sparse ARFF format
+  #  def test_sparse_arff_parse
+  #    in_file = './test_sparse_arff.arff'
+  #    rel = Rarff::Relation.new
+  #    rel.parse(File.open(File.join(File.dirname(__FILE__),in_file)).read)
+  #
+  #    assert_equal(13, rel.instances[0].size)
+  #    assert_equal(0, rel.instances[0][1])
+  #    assert_equal(7, rel.instances[0][3])
+  #    assert_equal(2.4, rel.instances[1][1])
+  #    assert_equal(0, rel.instances[1][2])
+  #    assert_equal(19, rel.instances[1][12])
+  #    assert_equal(6, rel.instances[2][6])
+  #    assert_equal(0, rel.instances[3][12])
+  #    #               puts "\n\nARFF: (\n#{rel.to_arff}\n)"
+  #  end
+  def test_output_missing
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 NUMERIC
+@ATTRIBUTE Attr3 STRING
+@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+@DATA
+?, 'foo bar', 5, baz, ?
+20.9, ruby, 46, ?, "2005-10-23 12:12:12"
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [nil, 'foo bar', 5, 'baz', nil],
+      [20.9, 'ruby', 46, nil, "2005-10-23 12:12:12"]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[1].name = 'subject'
+    rel.attributes[4].name = 'birthday'
+    rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "missing data output failure")
+  end
+  
+  def test_output_missing_undefined_first_row
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 NUMERIC
+@ATTRIBUTE Attr3 NUMERIC
+@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+@DATA
+?, ?, ?, ?, ?
+20.9, ruby, 46, ?, "2005-10-23 12:12:12"
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [nil, nil, nil, nil, nil],
+      [20.9, 'ruby', 46, nil, "2005-10-23 12:12:12"]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[1].name = 'subject'
+    rel.attributes[4].name = 'birthday'
+    rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "missing data from first line output failure")
+  end
+  
+  def test_boolean
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 {false,true}
+@DATA
+true
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [true]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "missing data from first line output failure")   
+  end
+  
+  def test_boolean_multipl
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 {false,true}
+@ATTRIBUTE Attr1 {false,true}
+@ATTRIBUTE Attr2 {false,true}
+@DATA
+true, false, true
+true, true, true
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [true,false,true],[true,true,true]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "missing data from first line output failure")   
+  end
+
+  def test_strings_as_nominal
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 {two,one}
+@ATTRIBUTE Attr1 {three,four}
+@DATA
+one, three
+two, four
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ ['one','three'],['two','four']]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.set_string_attributes_to_nominal
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "test_strings_as_nominal")
+  end
+    
+  def test_set_strings_nominal2
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE Attr1 {three,four}
+@DATA
+1, three
+2, four
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [1,'three'],[2,'four']]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.set_string_attributes_to_nominal
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "test_strings_as_nominal")
+  end
+
+  def test_strings_nominal_with_arguments1
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE Attr1 STRING
+@DATA
+1, three
+2, four
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [1,'three'],[2,'four']]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.set_string_attributes_to_nominal([0])
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "test_strings_as_nominal")
+  end
+
+  def test_strings_nominal_with_arguments2
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE Attr1 {three,four}
+@DATA
+1, three
+2, four
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [1,'three'],[2,'four']]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.set_string_attributes_to_nominal([0,1])
+
+    #               puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+    assert_equal(arff_file_str, rel.to_arff, "test_strings_as_nominal")
+  end
+  
+  def test_boolean_2
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 {false,true}
+@DATA
+?, ?, ?
+20.9, ruby, true
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [ [nil, nil, nil],
+      [20.9, 'ruby', true]]
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[1].name = 'subject'
+
+    assert_equal(arff_file_str, rel.to_arff, "missing data output failure")   
+  end
+
+  def test_commas_in_attribute_name
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE subject {ruby_yeh,ruby}
+@ATTRIBUTE Attr1 {duh}
+@DATA
+ruby__yeh, duh
+ruby, duh
+    END_OF_ARFF_FILE
+
+    arff_file_str.gsub!(/\n$/, '')
+
+    instances = [
+      ['ruby, yeh','duh'],
+      ['ruby','duh']
+    ]
+
+
+    rel = Rarff::Relation.new('MyCoolRelation')
+    rel.instances = instances
+    rel.attributes[0].name = 'subject'
+    rel.set_string_attributes_to_nominal
+
+    assert_equal(arff_file_str, rel.to_arff, "comma in string attribute failure")
+  end
+end
+
+
+

data/test/test_sparse_arff.arff

@@ -0,0 +1,24 @@
+% Sample sparse ARFF file
+@RELATION sparseness
+
+@ATTRIBUTE attr1  NUMERIC
+@ATTRIBUTE attr2   NUMERIC
+@ATTRIBUTE attr3  NUMERIC
+@ATTRIBUTE attr4   NUMERIC
+@ATTRIBUTE attr5   NUMERIC
+@ATTRIBUTE attr6   NUMERIC
+@ATTRIBUTE attr7   NUMERIC
+@ATTRIBUTE attr8   NUMERIC
+@ATTRIBUTE attr9   NUMERIC
+@ATTRIBUTE attr10   NUMERIC
+@ATTRIBUTE attr11   NUMERIC
+@ATTRIBUTE attr12   NUMERIC
+@ATTRIBUTE attr13   NUMERIC
+
+@DATA
+{3 7, 10 34}
+{1 2.4, 4 62, 12 19}
+{0 0, 1 1, 2 2, 3 3, 4 4, 5 5, 6 6, 7 7, 8 8, 9 9, 10 10, 11 11, 12 12}
+{9 42}
+{2 54.3, 3 92, 11 10.2}
+

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: iterationlabs-rarff
+version: !ruby/object:Gem::Version
+  version: 0.2.5
+  prerelease: 
+platform: ruby
+authors:
+- Ben J Woodcroft
+- Andy Payne
+- Andrew Cantino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-23 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &70346083110640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70346083110640
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &70346083109860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70346083109860
+- !ruby/object:Gem::Dependency
+  name: thoughtbot-shoulda
+  requirement: &70346083108740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70346083108740
+description: ! "Rarff is a Ruby library for dealing with Attribute-Relation File Format
+  (ARFF) files. ARFF files are used to specify \ndata sets for data mining and machine
+  learning."
+email: donttrustben near gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.txt
+files:
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- VERSION
+- lib/rarff.rb
+- test/test_arff.arff
+- test/test_rarff.rb
+- test/test_sparse_arff.arff
+homepage: http://github.com/iterationlabs/rarff
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Rarff is a Ruby library for dealing with Attribute-Relation File Format (ARFF)
+  files
+test_files: []