ruby_tokenizer 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9159d68232eb0ceada325a52566e93ecfb4fc3ff
-  data.tar.gz: 3367ad1e921f3fdf894a0acfec6f7d8b995aac7c
+  metadata.gz: 536056f6442aa02f631f5636c83136a749a6780c
+  data.tar.gz: d1af87a98c8cf3d3792f08fe79386b22abd2ef77
 SHA512:
-  metadata.gz: 171dfde354ee7916ad10ca8161a1169d4fb1e95eea854e44ad80c46660dc246c78f382bbb018250209dbe674ad66dafffba8821de6a8b7647e4a23be40a6821c
-  data.tar.gz: c5bf1cdb2c95e6400443a6bd2d4af5b20787d23650f643ee2ff80968ed137c740eeb36db08ab5f02de9f1343432c2b9f7bf83929071d140c588a694ba8146af2
+  metadata.gz: d49e6a5b9d7cc069e144242389c0523cfe843a06a3873d04fb6e420107e9a528f170478305c4ab79c6fd7626bc821d54c927a6a496e82ceb1a674bd98aac6ed8
+  data.tar.gz: a4559c6d39ecf48e43b08af90d1d4f746ab1b88b15586e9ebb9dc49be5dc5cdf32732d61ee8ec695212a34df05b73b34d8c00f006ce0cef5db3f83bc878a683e

data/bin/tokenizer

@@ -1,19 +1,11 @@
 #!/usr/bin/env ruby
 
 require 'ruby_tokenizer'
+require 'input_stream_module/input_stream.rb'
 require 'pry'
 
-if ARGV.empty?
-	puts "--- Please input your text below ----"
-	text = STDIN.gets
-elsif ARGV.detect { |input| File.file?(input) == false }
-	puts "File not found: Please try again."
-	exit 2
-else
-	text = ARGF.read
-end
-
-input = RubyTokenizer::Tokenizer.new(text)
-top_ten = input.rank
+input = InputStream.analyze(ARGV)
+output = RubyTokenizer::Tokenizer.new(input)
+top_ten = output.rank
 
 Pry::ColorPrinter.pp(top_ten)

data/lib/input_stream_module/input_stream.rb

@@ -0,0 +1,28 @@
+module InputStream
+
+  def self.analyze(input)
+    if input.empty?
+      self.prompt("--- Please input your text below ----")
+      self.get_input
+    else
+      self.read_file(input)
+    end
+  end
+
+  def self.prompt(string)
+    puts string
+  end
+
+  def self.get_input
+    STDIN.gets
+  end
+
+  def self.read_file(input)
+    if input.detect { |path| File.file?(path) == false }
+      raise LoadError, "File not found: Please try again."
+      exit 2
+    else
+      ARGF.read
+    end
+  end
+end

data/lib/patterns_module/patterns.rb

@@ -1,11 +1,11 @@
 module Patterns
-  PUNCTUATION = [':', ',', '—', '!', '?', ';', '.', '"']
+  PUNCTUATION = [':', ',', '—', '!', '?', ';', '"']
   PARENS = ['{', '[', '}', ']', '(', ')']
   MISC = ['<', '«', '„', '>', '»', '“', '*']
   # CONJUNCTIONS = ['for ', 'and ', 'nor ', 'but ', 'or ', 'yet ', 'so ', 'as ']
   # DETERMINERS = ['the ', 'this ', 'that ']
 
-  BASIC = PUNCTUATION + MISC
+  BASIC = PUNCTUATION + PARENS + MISC
 
   def self.basic
     Regexp.union(BASIC)

data/lib/ruby_tokenizer.rb

@@ -1,6 +1,5 @@
 require "ruby_tokenizer/version"
 require "patterns_module/patterns"
-require 'pry'
 
 module RubyTokenizer
 
@@ -20,7 +19,11 @@ module RubyTokenizer
     protected
     
     def filter
-      text.downcase.gsub(Patterns.basic, '')
+      text.downcase.gsub(Patterns.basic, ' ')
+    end
+
+    def tokenize
+      self.filter.scan(/[-\w'’.@]+/).map { |token| token.gsub(/[._-]$/, '') }
     end
 
     def frequency
@@ -30,9 +33,5 @@ module RubyTokenizer
 
       return count
     end
-
-    def tokenize
-      self.filter.scan(/[-\w'’]+/)
-    end
   end
 end

data/lib/ruby_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module RubyTokenizer
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - irinarenteria
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-26 00:00:00.000000000 Z
+date: 2016-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -92,6 +92,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/tokenizer
+- lib/input_stream_module/input_stream.rb
 - lib/patterns_module/patterns.rb
 - lib/ruby_tokenizer.rb
 - lib/ruby_tokenizer/version.rb