rarff 0.1.0

data/README

@@ -0,0 +1,68 @@
+= rarff
+
+Rarff - Ruby ARFF Library
+
+Rarff is a Ruby library for dealing with Attribute-Relation File Format (ARFF) files. ARFF files are used to specify data sets for data mining and machine learning.
+
+
+== License
+
+Copyright (c) 2006, Andy Payne
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+	* Redistributions of source code must retain the above copyright notice,
+        this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above copyright notice,
+        this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+	* Neither the name of the COPYRIGHT OWNER nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+== Todo
+
+* Spaces or quotes in nominal types
+* Commas in quoted attributes or in nominal types
+* Add error checking/validation
+* Creation of sparse ARFF files
+* Missing values - '?'
+* Dates - do some work to create, translate, and interpret date format strings.
+
+
+== Weka
+
+Weka is "a collection of machine learning algorithms for data mining tasks."
+(http://www.cs.waikato.ac.nz/ml/weka/)  Weka accompanies the following book:
+
+Ian H. Witten and Eibe Frank (2005) "Data Mining: Practical machine learning
+tools and techniques", 2nd Edition, Morgan Kaufmann, San Francisco, 2005.
+
+
+== ARFF Information
+
+ARFF files are similar to CSV files, but are strongly-typed, have a pre-defined
+set of data types, and include a sparse representation.
+
+Links to documentation:
+
+* http://www.cs.waikato.ac.nz/~ml/weka/arff.html
+* http://weka.sourceforge.net/wekadoc/index.php/en:ARFF_%283.4.6%29
+
+
+
+

data/lib/rarff.rb

@@ -0,0 +1,262 @@
+# = rarff
+
+# This is the top-level include file for rarff. See the README file for
+# details.
+
+################################################################################
+
+# Custom scan that returns a boolean indicating whether the regex matched.
+# TODO: Is there a way to avoid doing this?
+class String
+    def my_scan(re)
+        hit = false
+        scan(re) { |arr|
+            yield arr if block_given?
+            hit = true
+        }
+        hit
+    end
+end
+
+################################################################################
+
+module Enumerable
+# This map_with_index hack allows access to the index of each item as the map
+# iterates.
+# TODO: Is there a better way?
+	def map_with_index
+        # Ugly, but I need the yield to be the last statement in the map.
+		i = -1 
+		return map { |item|
+			i += 1
+			yield item, i
+		}
+	end
+end
+
+################################################################################
+
+module Rarff
+
+COMMENT_MARKER = '%'
+RELATION_MARKER = '@RELATION'
+ATTRIBUTE_MARKER = '@ATTRIBUTE'
+DATA_MARKER = '@DATA'
+
+SPARSE_ARFF_BEGIN = '{'
+ESC_SPARSE_ARFF_BEGIN = '\\' + SPARSE_ARFF_BEGIN
+SPARSE_ARFF_END = '}'
+ESC_SPARSE_ARFF_END = '\\' + SPARSE_ARFF_END
+
+ATTRIBUTE_NUMERIC = 'NUMERIC'
+ATTRIBUTE_REAL = 'REAL'
+ATTRIBUTE_INTEGER = 'INTEGER'
+ATTRIBUTE_STRING = 'STRING'
+ATTRIBUTE_DATE = 'DATE'
+
+
+################################################################################
+
+class Attribute
+	attr_accessor :name, :type
+
+	def initialize(name='', type='')
+		@name = name
+
+		@type_is_nominal = false
+		@type = type
+
+		check_nominal()
+	end
+
+
+	def type=(type)
+		@type = type
+		check_nominal()
+	end
+
+
+    # Convert string representation of nominal type to array, if necessary
+    # TODO: This might falsely trigger on wacky date formats.
+	def check_nominal
+		if @type =~ /^\s*\{.*(\,.*)+\}\s*$/
+			@type_is_nominal = true
+			# Example format: "{nom1,nom2, nom3, nom4,nom5 } "
+			# Split on '{'  ','  or  '}'
+			@type = @type.gsub(/^\s*\{\s*/, '').gsub(/\s*\}\s*$/, '').split(/\s*\,\s*/)
+		end
+	end
+
+
+	def add_nominal_value(str)
+		if @type_is_nominal == false
+			@type = Array.new
+		end
+
+		@type << str
+	end
+
+
+	def to_arff
+		if @type_is_nominal == true
+			ATTRIBUTE_MARKER + " #{@name} #{@type.join(',')}"
+		else
+			ATTRIBUTE_MARKER + " #{@name} #{@type}"
+		end
+	end
+
+
+	def to_s
+		to_arff
+	end
+
+end
+
+
+
+class Relation
+	attr_accessor :name, :attributes, :instances
+
+
+	def initialize(name='')
+		@name = name
+		@attributes = Array.new
+		@instances = Array.new
+	end
+
+
+	def parse(str)
+		in_data_section = false
+
+		# TODO: Doesn't handle commas in quoted attributes.
+		str.split("\n").each { |line|
+			next if line =~ /^\s*$/
+			next if line =~ /^\s*#{COMMENT_MARKER}/
+			next if line.my_scan(/^\s*#{RELATION_MARKER}\s*(.*)\s*$/i) { |name| @name = name }
+			next if line.my_scan(/^\s*#{ATTRIBUTE_MARKER}\s*([^\s]*)\s+(.*)\s*$/i) { |name, type|
+				@attributes.push(Attribute.new(name, type))
+			}
+			next if line.my_scan(/^\s*#{DATA_MARKER}/i) { in_data_section = true }
+			next if in_data_section == false	## Below is data section handling
+#			next if line.gsub(/^\s*(.*)\s*$/, "\\1").my_scan(/^\s*#{SPARSE_ARFF_BEGIN}(.*)#{SPARSE_ARFF_END}\s*$/) { |data|
+			next if line.gsub(/^\s*(.*)\s*$/, "\\1").my_scan(/^#{ESC_SPARSE_ARFF_BEGIN}(.*)#{ESC_SPARSE_ARFF_END}$/) { |data|
+				# Sparse ARFF
+				# TODO: Factor duplication with non-sparse data below
+				@instances << expand_sparse(data.first)
+				create_attributes()
+			}
+			next if line.my_scan(/^\s*(.*)\s*$/) { |data|
+				@instances << data.first.split(/,\s*/).map { |field|
+					# Remove outer single quotes on strings, if any ('foo bar' --> foo bar)
+					field.gsub(/^\s*\'(.*)\'\s*$/, "\\1")
+				}
+				create_attributes()
+			}
+		}
+	end
+
+
+	def instances=(instances)
+		@instances = instances
+		create_attributes()
+	end
+
+
+	def create_attributes
+		attr_pass = true
+
+		@instances.each_index { |i|
+			@instances[i].each_index { |j|
+				if @instances[i][j].class != String
+					if attr_pass == true
+						@attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_NUMERIC)
+					end
+				elsif @instances[i][j] =~ /^\-?\d+\.?\d*$/
+					# TODO: Should I have a separate to_i conversion, or is to_f sufficient?
+					@instances[i][j] = @instances[i][j].to_f
+					if attr_pass == true
+						@attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_NUMERIC)
+					end
+				else
+					if attr_pass == true
+						@attributes[j] = Attribute.new("Attr#{j}", ATTRIBUTE_STRING)
+					end
+				end
+			}
+
+			attr_pass = false
+		}
+	end
+
+
+	def expand_sparse(str)
+		arr = Array.new(@attributes.size, 0)
+		str.gsub(/^\s*\{(.*)\}\s*$/, "\\1").split(/\s*\,\s*/).map { |pr|
+			pra = pr.split(/\s/)
+			arr[pra[0].to_i] = pra[1]
+		}
+		arr
+	end
+
+
+	def to_arff
+		RELATION_MARKER + " #{@name}\n" +
+		@attributes.map{ |attr| attr.to_arff }.join("\n") +
+		"\n" +
+		DATA_MARKER + "\n" +
+		@instances.map { |inst|
+			inst.map_with_index { |col, i|
+				# Quote strings with spaces.
+				# TODO: Doesn't handle cases in which strings already contain
+				# quotes or are already quoted.
+				if @attributes[i].type =~ /^#{ATTRIBUTE_STRING}$/i
+					if col =~ /\s+/
+						col = "'" + col + "'"
+					end
+				elsif @attributes[i].type =~ /^#{ATTRIBUTE_DATE}/i  ## Hack comparison. Ugh.
+					col = '"' + col + '"'
+				end
+				col
+			}.join(', ')
+		}.join("\n")
+	end
+
+
+	def to_s
+		to_arff
+	end
+
+end
+
+
+end		# module Rarff
+
+################################################################################
+
+if $0 == __FILE__ then
+
+
+if ARGV[0]
+	in_file = ARGV[0]
+	contents = ''
+
+	contents = File.open(in_file).read
+
+	rel = Rarff::Relation.new
+	rel.parse(contents)
+
+else
+	exit
+end
+
+puts '='*80
+puts '='*80
+puts "ARFF:"
+puts rel
+
+
+end
+
+################################################################################
+
+

data/tests/test_arff.arff

@@ -0,0 +1,27 @@
+% 1. Title: Iris Plants Database
+% 
+% 2. Sources:
+%      (a) Creator: R.A. Fisher
+%      (b) Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov)
+%      (c) Date: July, 1988
+% 
+@RELATION iris
+
+@ATTRIBUTE sepallength  NUMERIC
+@ATTRIBUTE sepalwidth   NUMERIC
+@ATTRIBUTE petallength  NUMERIC
+@ATTRIBUTE petalwidth   NUMERIC
+@ATTRIBUTE class        {Iris-setosa,Iris-versicolor,Iris-virginica}
+
+@DATA
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3.0,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5.0,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5.0,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+

data/tests/test_sparse_arff.arff

@@ -0,0 +1,24 @@
+% Sample sparse ARFF file
+@RELATION sparseness
+
+@ATTRIBUTE attr1  NUMERIC
+@ATTRIBUTE attr2   NUMERIC
+@ATTRIBUTE attr3  NUMERIC
+@ATTRIBUTE attr4   NUMERIC
+@ATTRIBUTE attr5   NUMERIC
+@ATTRIBUTE attr6   NUMERIC
+@ATTRIBUTE attr7   NUMERIC
+@ATTRIBUTE attr8   NUMERIC
+@ATTRIBUTE attr9   NUMERIC
+@ATTRIBUTE attr10   NUMERIC
+@ATTRIBUTE attr11   NUMERIC
+@ATTRIBUTE attr12   NUMERIC
+@ATTRIBUTE attr13   NUMERIC
+
+@DATA
+{3 7, 10 34}
+{1 2.4, 4 62, 12 19}
+{0 0, 1 1, 2 2, 3 3, 4 4, 5 5, 6 6, 7 7, 8 8, 9 9, 10 10, 11 11, 12 12}
+{9 42}
+{2 54.3, 3 92, 11 10.2}
+

data/tests/ts_rarff.rb

@@ -0,0 +1,74 @@
+# See the README file for more information.
+
+require 'test/unit'
+require 'rarff'
+
+class TestArffLib < Test::Unit::TestCase
+
+	# Test creation of an arff file string.
+    def test_arff_creation
+
+    arff_file_str = <<-END_OF_ARFF_FILE
+@RELATION MyCoolRelation
+@ATTRIBUTE Attr0 NUMERIC
+@ATTRIBUTE subject STRING
+@ATTRIBUTE Attr2 NUMERIC
+@ATTRIBUTE Attr3 STRING
+@ATTRIBUTE birthday DATE "yyyy-MM-dd HH:mm:ss"
+@DATA
+1.4, 'foo bar', 5, baz, "1900-08-08 12:12:12"
+20.9, ruby, 46, rocks, "2005-10-23 12:12:12"
+20.9, ruby, 46, rocks, "2001-02-19 12:12:12"
+68.1, stuff, 728, 'is cool', "1974-02-10 12:12:12"
+END_OF_ARFF_FILE
+
+        arff_file_str.gsub!(/\n$/, '')
+
+        instances = [ [1.4, 'foo bar', 5, 'baz', "1900-08-08 12:12:12"],
+	        		[20.9, 'ruby', 46, 'rocks', "2005-10-23 12:12:12"],
+		        	[20.9, 'ruby', 46, 'rocks', "2001-02-19 12:12:12"],
+        			[68.1, 'stuff', 728, 'is cool', "1974-02-10 12:12:12"]]
+
+        rel = Rarff::Relation.new('MyCoolRelation')
+        rel.instances = instances
+        rel.attributes[1].name = 'subject'
+        rel.attributes[4].name = 'birthday'
+        rel.attributes[4].type = 'DATE "yyyy-MM-dd HH:mm:ss"'
+
+#		puts "rel.to_arff:\n(\n#{rel.to_arff}\n)\n"
+        assert_equal(rel.to_arff, arff_file_str, "Arff creation test failed.")
+    end
+
+
+	# Test parsing of an arff file.
+	def test_arff_parse
+		in_file = './test_arff.arff'
+		rel = Rarff::Relation.new
+		rel.parse(File.open(in_file).read)
+
+		assert_equal(rel.instances[2][1], 3.2)
+		assert_equal(rel.instances[7][4], 'Iris-setosa')
+	end
+
+
+	# Test parsing of sparse ARFF format
+    def test_sparse_arff_parse
+		in_file = './test_sparse_arff.arff'
+		rel = Rarff::Relation.new
+		rel.parse(File.open(in_file).read)
+
+		assert_equal(rel.instances[0].size, 13)
+		assert_equal(rel.instances[0][1], 0)
+		assert_equal(rel.instances[0][3], 7)
+		assert_equal(rel.instances[1][1], 2.4)
+		assert_equal(rel.instances[1][2], 0)
+		assert_equal(rel.instances[1][12], 19)
+		assert_equal(rel.instances[2][6], 6)
+		assert_equal(rel.instances[3][12], 0)
+#		puts "\n\nARFF: (\n#{rel.to_arff}\n)"
+	end
+end
+
+
+
+

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: rarff
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-09-18 00:00:00 -05:00
+summary: Library for handling Weka ARFF files
+require_paths: 
+  - lib
+email: apayne@gmail.com
+homepage: TODO
+rubyforge_project: 
+description: 
+autorequire: rarff
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+  - Andy Payne
+files: 
+  - lib/rarff.rb
+  - tests/test_arff.arff
+  - tests/test_sparse_arff.arff
+  - tests/ts_rarff.rb
+  - README
+test_files: 
+  - tests/ts_rarff.rb
+rdoc_options: []
+extra_rdoc_files: 
+  - README
+executables: []
+extensions: []
+requirements: []
+dependencies: []