textoken 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cf70d771ada2f8b5edaec1b2f6b4c97a4efa563a
+  data.tar.gz: eead0c387504164a302e687fa5858b1bf270d972
+SHA512:
+  metadata.gz: 13f3ff9bd02f7b57954e791ea0c068c3c1b529d7056a7067fa66100c0992e1299cfa0e897a7b27522fa557dfbb6290b2191bf8654a366df38ceb209a30ae9c7f
+  data.tar.gz: d691c3914c7cb78dd396d80d9b8524e2d5e1b3e2786a8518da1558c13916937b59692f29b7723e963aa935c5e42e764c608e9c80d5d7e1b705a1d331e3dfd2c7

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+require "rake"
+
+RSpec::Core::RakeTask.new
+
+task default: :spec
+task test: :spec

data/lib/textoken/base.rb

@@ -0,0 +1,39 @@
+module Textoken
+  # Inits options, findings and responds to tokens
+  # Does not raise error when text or options are nil
+  # Splits the text and makes it ready for other operations
+  class Base
+    attr_reader :text, :dont_split, :findings, :options
+
+    def initialize(text, opt = nil)
+      @text     = initial_split(text)
+      @options  = Options.new(opt)
+    end
+
+    # we do take intersection array of results
+    # returning from multiple options
+    def tokens
+      options.collection.each do |option|
+        if @findings.nil?
+          @findings = option.tokenize(self)
+        else
+          @findings &= option.tokenize(self)
+        end
+      end
+
+      Tokenizer.new(self).tokens
+    end
+
+    def words
+      # tokenize options but do not make the last split
+      @dont_split = true
+      tokens
+    end
+
+    private
+
+    def initial_split(text)
+      text ? text.split(' ') : []
+    end
+  end
+end

data/lib/textoken/errors.rb

@@ -0,0 +1,7 @@
+module Textoken
+  class ExpressionError < Exception
+  end
+
+  class TypeError < Exception
+  end
+end

data/lib/textoken/factories/option_factory.rb

@@ -0,0 +1,23 @@
+module Textoken
+  # This factory created option objects throug user input option
+  # User input option like 'more_than: 3' (more_than) gets camelized
+  # and objects get initialized with error handling
+  module OptionFactory
+    def self.build(key, value)
+      option_klass(key).new(value)
+    end
+
+    private
+
+    def self.camelize(key)
+      key.to_s.split('_').map(&:capitalize).join
+    end
+
+    def self.option_klass(key)
+      # add a module to make other classes unaccessible
+      Textoken.const_get(camelize(key).to_sym)
+    rescue NameError
+      Textoken.expression_err("#{key}: is not a valid option.")
+    end
+  end
+end

data/lib/textoken/findings.rb

@@ -0,0 +1,34 @@
+module Textoken
+  # This class duty is to collect splitted words with array index no
+  # And also handles making uniq of multiple words in the same index
+  # due to Regexps that reveal same/closer results
+  class Findings
+    def initialize
+      @collection = []
+    end
+
+    # Here we will push items to collection array with index number
+    # index number will help us to sort and make array unique
+    def push(index, word)
+      type_check(index, word)
+      @collection << [index, word]
+    end
+
+    # collection will return a sorted and unique array of tokens
+    def collection
+      @collection.uniq { |w| w[0].to_s + w[1] }.sort_by(&:first)
+    end
+
+    # result will return a one dimensional array of words
+    def result
+      collection.map(&:last)
+    end
+
+    private
+
+    def type_check(i, word)
+      return if word.is_a?(String) && (i.is_a?(Fixnum) || i.is_a?(Float))
+      Textoken.type_err("#{word} and #{i} has to be a String and Integer")
+    end
+  end
+end

data/lib/textoken/options/exclude.rb

@@ -0,0 +1,28 @@
+module Textoken
+  # This option object excludes words in text via matching regexp
+  # Non-excluded words pushed to findings array
+  class Exclude
+    attr_reader :regexps, :findings
+
+    def priority
+      1
+    end
+
+    def initialize(values)
+      @regexps  = Searcher.new(values).regexps
+      @findings = Findings.new
+    end
+
+    # base.text is raw tokens splitted with ' '
+    # values are Regexps array to search
+    # base.findings, Findings object for pushing matching tokens
+    def tokenize(base)
+      regexps.each do |r|
+        base.text.each_with_index do |t, i|
+          findings.push(i, t) unless t.match(r)
+        end
+      end
+      findings.result
+    end
+  end
+end

data/lib/textoken/options/less_than.rb

@@ -0,0 +1,32 @@
+module Textoken
+  # This option object picks words in text with less than length
+  # of the option value
+  class LessThan
+    attr_reader :number, :findings
+
+    def priority
+      2
+    end
+
+    def initialize(value)
+      check_value(value)
+      @number = value
+      @findings = Findings.new
+    end
+
+    def tokenize(base)
+      base.text.each_with_index do |w, i|
+        findings.push(i, w) if w.length < number
+      end
+      findings.result
+    end
+
+    private
+
+    def check_value(value)
+      return if value.class == Fixnum && value > 1
+      Textoken.type_err "value #{value} is not permitted for
+        less_than option it has to be 2 at least."
+    end
+  end
+end

data/lib/textoken/options/more_than.rb

@@ -0,0 +1,32 @@
+module Textoken
+  # This option object picks words in text with more than length
+  # of the option value
+  class MoreThan
+    attr_reader :number, :findings
+
+    def priority
+      2
+    end
+
+    def initialize(value)
+      check_value(value)
+      @number   = value
+      @findings = Findings.new
+    end
+
+    def tokenize(base)
+      base.text.each_with_index do |w, i|
+        findings.push(i, w) if w.length > number
+      end
+      findings.result
+    end
+
+    private
+
+    def check_value(value)
+      return if value.class == Fixnum && value >= 0
+      Textoken.type_err "value #{value} is not permitted for
+        more_than option it has to be 0 at least."
+    end
+  end
+end

data/lib/textoken/options/only.rb

@@ -0,0 +1,28 @@
+module Textoken
+  # This option object selects words in text via matching regexp
+  # regexp should be defined in option_values.yml
+  class Only
+    attr_reader :regexps, :findings
+
+    def priority
+      1
+    end
+
+    def initialize(values)
+      @regexps = Searcher.new(values).regexps
+      @findings = Findings.new
+    end
+
+    # base.text is raw tokens splitted with ' '
+    # values are Regexps array to search
+    # base.findings, Findings object for pushing matching tokens
+    def tokenize(base)
+      regexps.each do |r|
+        base.text.each_with_index do |t, i|
+          findings.push(i, t) if t.match(r)
+        end
+      end
+      findings.result
+    end
+  end
+end

data/lib/textoken/options.rb

@@ -0,0 +1,36 @@
+module Textoken
+  # Creates collection array, checks basic hash format.
+  # Responds to collection as an array of options sorted by
+  # priority. Does not raise an exception when options are nil
+  class Options
+    attr_reader :options
+
+    def initialize(opt)
+      @options = opt
+      @collection = []
+      check_options_format
+    end
+
+    def collection
+      options.each { |k, v| init_option(k, v) } if options
+      sort_collection
+      @collection
+    end
+
+    private
+
+    def check_options_format
+      return if options.nil? || options.is_a?(Hash)
+      Textoken.expression_err("#{options} is not a valid format, you can use;
+        Textoken('Alfa beta.', exclude: 'dates, phones', more_than: 3)")
+    end
+
+    def init_option(key, value)
+      @collection << Textoken::OptionFactory.build(key, value)
+    end
+
+    def sort_collection
+      @collection.sort! { |a, b| a.priority <=> b.priority }
+    end
+  end
+end

data/lib/textoken/regexps/option_values.yml

@@ -0,0 +1,22 @@
+# This file stands for options you can pass as an argument like,
+# only: 'dates, numbers, etc..' or exclude: 'dates'
+
+# You can add many regexps under one type, all of them will be searchable
+
+punctuations:
+  # punctuations
+  - \W+
+
+numerics:
+  - (?:\d*\.)?\d+
+  - (?:\d*\,)?\d+
+
+phones:
+  # covers p:444-555-1234 f:246.555.8888 m:1235554567
+  - \b\d{3}[-.]?\d{3}[-.]?\d{4}\b
+
+dates:
+  # mm/dd/yyyy format
+  - ^(0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])[- /.](19|20)\d\d$
+  # dd-mm-yyyy format
+  - ^(0[1-9]|[12][0-9]|3[01])[- /.](0[1-9]|1[012])[- /.](19|20)\d\d$

data/lib/textoken/scanner.rb

@@ -0,0 +1,33 @@
+module Textoken
+  # Scanner finds tokens in a word with regexp
+  # If word does not match regexp returns an empty Array
+  class Scanner
+    attr_reader :word, :regexp
+
+    def initialize(word, regexp)
+      @word       = word
+      @regexp     = regexp
+      check_types
+    end
+
+    def result
+      scan = word.scan(regexp)
+      scan.length > 0 ? partition(scan, word) : nil
+    end
+
+    private
+
+    def partition(scan, word)
+      scan.each do |p|
+        word = word.gsub(p, ' ' + p + ' ')
+      end
+      word.split(' ')
+    end
+
+    def check_types
+      return if word.is_a?(String) && regexp.is_a?(Regexp)
+      Textoken.type_err("#{regexp} should be Regexp and #{word}
+        has to be a string.")
+    end
+  end
+end

data/lib/textoken/searcher.rb

@@ -0,0 +1,45 @@
+module Textoken
+  # This class reads YAML file under searchables/values,
+  # returns array of regexps representing values passed as an argument
+  # todo: add regexp support
+  class Searcher
+    attr_reader :values, :yaml
+
+    def initialize(values)
+      @values   = check_and_init_values(values)
+      @yaml     = load_file
+      @regexps  = []
+    end
+
+    def regexps
+      match_keys
+      @regexps.map.each { |r| Regexp.new(r) }
+    end
+
+    private
+
+    def check_and_init_values(values)
+      values.split(',').map(&:strip)
+    rescue
+      Textoken.expression_err("#{values} are not supported. Correct format,
+        has to be 'numbers' or 'numbers, dates, phones'")
+    end
+
+    def load_file
+      YAML.load_file("#{GEM_ROOT}/lib/textoken/regexps/option_values.yml")
+    end
+
+    # here we do check for option values user supplied
+    # option values has to be declared at option_values.yml
+    def match_keys
+      values.each do |v|
+        Textoken.expression_err("#{v}: is not permitted.") unless yaml.key?(v)
+        add_regexps(yaml[v])
+      end
+    end
+
+    def add_regexps(arr)
+      @regexps += arr
+    end
+  end
+end

data/lib/textoken/tokenizer.rb

@@ -0,0 +1,33 @@
+module Textoken
+  # This is the last step in the process
+  # Takes findings objects and makes finishing punctuation split
+  # Does not split if base.dont_split is true
+  class Tokenizer
+    attr_reader :base, :result, :col
+
+    def initialize(base)
+      @base    = base
+      @result  = []
+      @col     = base.options.collection.length > 0 ? base.findings : base.text
+    end
+
+    def tokens
+      return col if base.dont_split
+      split_punctuations
+      @result
+    end
+
+    private
+
+    def split_punctuations
+      col.each do |w|
+        @result += Scanner.new(w, default_regexp).result || [w]
+      end
+    end
+
+    # will be used for finding punctuations
+    def default_regexp
+      Regexp.new('\W+')
+    end
+  end
+end

data/lib/textoken/version.rb

@@ -0,0 +1,3 @@
+module Textoken
+  VERSION = "1.0.0"
+end

data/lib/textoken.rb

@@ -0,0 +1,42 @@
+require 'yaml'
+
+require 'textoken/version'
+require 'textoken/base'
+require 'textoken/errors'
+require 'textoken/searcher'
+require 'textoken/options'
+require 'textoken/findings'
+require 'textoken/tokenizer'
+require 'textoken/scanner'
+
+require 'textoken/options/less_than'
+require 'textoken/options/more_than'
+require 'textoken/options/only'
+require 'textoken/options/exclude'
+
+require 'textoken/factories/option_factory'
+
+# Textoken is a library for customizable tokenization of texts.
+# Customizable tokenization can be used in many areas including NLP purposes
+module Textoken
+  GEM_ROOT = File.expand_path('../..', __FILE__)
+
+  class << self
+    # Expression error raised in wrong user input of options & values
+    def expression_err(msg)
+      fail ExpressionError, msg
+    end
+
+    # Type error raised when user input format is right but the option value
+    # is not suitable for option
+    def type_err(msg)
+      fail TypeError, msg
+    end
+  end
+end
+
+# A shortcut to initialize gem
+# Textoken('I had rather be first in a village than second at Rome.')
+def Textoken(text, options = nil)
+  Textoken::Base.new(text, options)
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: textoken
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Mehmet Cetin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "Textoken is a Ruby library for text tokenization. \n  This gem extracts
+  words from text with many customizations. \n  It can be used in many fields like
+  crawling and Natural Language Processing."
+email:
+- mcetin.cm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Rakefile
+- lib/textoken.rb
+- lib/textoken/base.rb
+- lib/textoken/errors.rb
+- lib/textoken/factories/option_factory.rb
+- lib/textoken/findings.rb
+- lib/textoken/options.rb
+- lib/textoken/options/exclude.rb
+- lib/textoken/options/less_than.rb
+- lib/textoken/options/more_than.rb
+- lib/textoken/options/only.rb
+- lib/textoken/regexps/option_values.yml
+- lib/textoken/scanner.rb
+- lib/textoken/searcher.rb
+- lib/textoken/tokenizer.rb
+- lib/textoken/version.rb
+homepage: https://github.com/manorie/textoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Simple and customizable text tokenization gem.
+test_files: []