ruby_tokenizer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 76efe7d10a7f4d99a30a918c36017811c685b7a0
+  data.tar.gz: 72d35250146292cc25d5a06de9f112bb05233d6a
+SHA512:
+  metadata.gz: 16301007ff881f9928a4b54a541ccd3419c49487160cfbe5c2d9310b10eab0f226c1146d70e8dd2fb6267b263470e1d34b0bf2523d853efa4293cb19885200d3
+  data.tar.gz: f2a63effc18259d44bf690977aa718767e924121d0345bba05e5a29ca2ce765d66c6afeb98f964ad366257d09f49c75c63fe75d3986c34bb8e4f919439f20ff1

data/bin/tokenizer

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'ruby_tokenizer'
+require 'pry'
+
+if ARGV.empty?
+	puts "--- Please input your text below ----"
+	text = STDIN.gets
+elsif ARGV.detect { |input| File.file?(input) == false }
+	puts "File not found: Please try again."
+	exit 2
+else
+	text = ARGF.read
+end
+
+input = RubyTokenizer::Tokenizer.new(text)
+top_ten = input.rank
+
+Pry::ColorPrinter.pp(top_ten)

data/lib/patterns_module/patterns.rb

@@ -0,0 +1,13 @@
+module Patterns
+  PUNCTUATION = [':', ',', '—', '!', '?', ';', '.', '"']
+  PARENS = ['{', '[', '}', ']', '(', ')']
+  MISC = ['<', '«', '„', '>', '»', '“', '*']
+  # CONJUNCTIONS = ['for ', 'and ', 'nor ', 'but ', 'or ', 'yet ', 'so ', 'as ']
+  # DETERMINERS = ['the ', 'this ', 'that ']
+
+  BASIC = PUNCTUATION + MISC
+
+  def self.basic
+    Regexp.union(BASIC)
+  end
+end

data/lib/ruby_tokenizer/version.rb

@@ -0,0 +1,3 @@
+module RubyTokenizer
+  VERSION = "0.1.0"
+end

data/lib/ruby_tokenizer.rb

@@ -0,0 +1,38 @@
+require "ruby_tokenizer/version"
+require "patterns_module/patterns"
+require 'pry'
+
+module RubyTokenizer
+
+  class Tokenizer
+    include Patterns
+    attr_reader :text
+
+    def initialize(text)
+      @text = text
+    end
+
+    def rank
+      ranked = Hash[self.frequency.sort_by { |_word, count| count }.reverse]
+      ranked.first(10)
+    end
+
+    protected
+    
+    def filter
+      text.downcase.gsub(Patterns.basic, '')
+    end
+
+    def frequency
+      count = Hash.new(0)
+      parsed = self.tokenize
+      parsed.each { |word| count[word] += 1 }
+
+      return count
+    end
+
+    def tokenize
+      self.filter.scan(/[-\w'’]+/)
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: ruby_tokenizer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- irinarenteria
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- renteria.irina@gmail.com
+executables:
+- tokenizer
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/tokenizer
+- lib/patterns_module/patterns.rb
+- lib/ruby_tokenizer.rb
+- lib/ruby_tokenizer/version.rb
+homepage: https://github.com/irinarenteria/ruby_tokenizer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Simple NPL tokenizer in Ruby
+test_files: []