ruby_ngrams 0.0.1 → 0.0.2

data/bin/ruby_ngrams

@@ -0,0 +1,58 @@
+#!/usr/bin/ruby -w
+
+require 'ruby_cli'
+require 'ruby_ngrams'
+
+class App
+	include RubyCLI	
+
+	def define_command_options() @options = {:regex => //, :n => 2}	end
+
+	# Define an OptionParser to parse the command line
+	def parse_options?
+		#configure an OptionParser
+		@opt_parser = OptionParser.new do |opts|		
+			opts.banner = "Usage: #{__FILE__} [OPTIONS]... [FILE]..."
+			opts.separator ""
+			opts.separator "Specific options:"
+			opts.on('-h', '--help', 'displays help information') do
+				@default_options[:help] = true
+				output_help
+			end
+			opts.on('-V','--verbose','Run verbosely') do 
+				@default_options[:verbose] = true
+			end
+			opts.on('-n', '--n NUM', Integer, 'set length n for n-grams') do |n|
+				@options[:n] = n
+			end
+			opts.on('-r', '--regex REGEX', Regexp, 'set regex to split string into tokens') do |r|
+				@options[:regex] = r
+			end
+		end
+		@opt_parser.parse!(@default_argv) rescue return false
+		true
+	end
+
+	def command
+		# If arguments were provided, then they have to be names of files.
+		# These files will be handled using Ruby's ARGF builtin variable.
+		# If arguments are not filenames, then this application will produce a 
+		# a runtime error informing the user that the given file could not be opened.
+	
+		# ARGF is a stream designed for use in scripts that process files given as 
+		#	command-line arguments or passed in via STDIN.
+		# The arguments passed to your script are stored in the ARGV Array, 
+		#	one argument per element. ARGF assumes that any arguments that aren’t 
+		# filenames have been removed from ARGV.
+		text = ARGF.read
+		text.ngrams(@options).each { |ngram| puts ngram.inspect }
+	end
+
+end
+
+
+if __FILE__ == $0
+	app = App.new(ARGV)
+	app.run
+end
+

data/lib/ruby_ngrams.rb

@@ -0,0 +1,47 @@
+# This is an extension of Ruby's core String class.
+# It add methods to extract a set of n-grams from a string.
+# Typically, the most used set of n-grams are unigrams, bigrams, and trigrams; 
+# sets of n-grams of length 1, 2, and 3, respectively.
+class String
+
+	# An n-gram is a sequence of units of text of length n, where those units are 
+	# typically single characters or words delimited by space characters.
+	# However, a token could also be a fixed length character sequence, strings 
+	# with embedded spaces, etc. depending on the intended application.
+	# Typically, n-grams are formed of contiguous tokens.
+	#
+	# This function splits the string into a set of n-grams. 
+	# The default regex used tokenizes the string into characters.
+	#
+	# Regex Examples:
+	#		// 			=> splits into characters
+	#		/\s+/ 	=> splits into words delimited by one or more space characters
+	#		/\n+/ => splits into lines delimted by one or more newline characters
+	#
+	# TODO: Determine efficiency of this function on long strings.
+	# TODO: Determine how well this works on strings in binary format.
+	def ngrams(options = {:regex=>//, :n=>2})
+		ngrams = []
+		tokens = self.split(options[:regex])
+		max_pos = tokens.length - options[:n]
+		for i in 0..max_pos
+			ngrams.push(tokens[i..i+(options[:n]-1)])
+		end
+		ngrams
+	end
+	
+	# This function splits the string into unigrams,
+	# tokenizes into chars by default
+	def unigrams(regex = //) ngrams({:regex => regex, :n => 1}); end
+	
+	# This function splits the string into bigrams
+	# tokenizes into chars by default
+	def bigrams(regex = //) ngrams({:regex => regex, :n => 2}); end
+	
+	# This function splits the string into trigrams
+	# tokenizes into chars by default
+	def trigrams(regex = //) ngrams({:regex => regex, :n => 3}); end
+
+end #class String
+
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_ngrams
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Martin Velez
@@ -15,11 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-19 00:00:00 -07:00
+date: 2011-11-11 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
-description: A simple Ruby library to parse a string into n-grams
+description: A simple extension of the Ruby core string class to parse a string into n-grams
 email: mvelez999@gmail.com
 executables: 
 - ruby_ngrams