pragmatic_tokenizer 3.0.4 → 3.2.0

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "3.0.4".freeze
+  VERSION = "3.2.0".freeze
 end

data/pragmatic_tokenizer.gemspec

@@ -1,5 +1,4 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pragmatic_tokenizer/version'
 
@@ -9,9 +8,9 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Kevin S. Dias"]
   spec.email         = ["diasks2@gmail.com"]
 
-  spec.summary       = %q{A multilingual tokenizer}
-  spec.description   = %q{A multilingual tokenizer to split a string into tokens.}
-  spec.homepage      = "https://github.com/diasks2/pragmatic_tokenizer"
+  spec.summary       = 'A multilingual tokenizer'
+  spec.description   = 'A multilingual tokenizer to split a string into tokens.'
+  spec.homepage      = 'https://github.com/diasks2/pragmatic_tokenizer'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
@@ -20,7 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "unicode"
   spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "stackprof"
   spec.add_development_dependency "rubocop"

data/spec/languages/english_spec.rb

@@ -88,6 +88,12 @@ describe PragmaticTokenizer do
           expect(pt.tokenize(text)).to eq(["#ab-cd"])
         end
 
+        it 'tokenizes a string #015' do
+          text = "In 2004, he co-founded Palantir Technologies, which offers platforms for finance companies and intelligence, defense, and law enforcement communities to integrate, visualize, and analyze the world's information."
+          pt = PragmaticTokenizer::Tokenizer.new
+          expect(pt.tokenize(text)).to eq(["in", "2004", ",", "he", "co-founded", "palantir", "technologies", ",", "which", "offers", "platforms", "for", "finance", "companies", "and", "intelligence", ",", "defense", ",", "and", "law", "enforcement", "communities", "to", "integrate", ",", "visualize", ",", "and", "analyze", "the", "world's", "information", "."])
+        end
+
         it 'handles numbers with symbols 2' do
           text = "Pittsburgh Steelers won 18:16 against Cincinnati Bengals!"
           pt = PragmaticTokenizer::Tokenizer.new
@@ -543,6 +549,13 @@ describe PragmaticTokenizer do
           )
           expect(pt.tokenize(text)).to eq(["some", "main", "categories", "of", "the", "mathematics", "test", "have", "sub", "examples", "that", "most", "14", "year", "olds", "can't", "answer", ",", "therefor", "the", "implementation", "instruction", "made", "in", "the", "1990", "years", "needs", "to", "be", "revised", "."])
         end
+        it 'tokenizes something with a slash' do
+          text = "EO/AA/M/F/Veterans/Disability/Sexual Orientation/Gender Identity"
+          pt = PragmaticTokenizer::Tokenizer.new(
+              long_word_split: 1
+          )
+          expect(pt.tokenize(text)).to eq(["eo", "aa", "m", "f", "veterans", "disability", "sexual", "orientation", "gender", "identity"])
+        end
       end
 
       context 'option (clean)' do

data/spec/languages/french_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe PragmaticTokenizer do
   context 'Language: French (fr)' do
     it 'tokenizes a string #001' do
-      text = "L'art de l'univers, c'est un art"
+      text = "D'art de l'univers, c'est un art"
       pt = PragmaticTokenizer::Tokenizer.new(
           language: 'fr'
       )
-      expect(pt.tokenize(text)).to eq(["l'", "art", "de", "l'", "univers", ",", "c'est", "un", "art"])
+      expect(pt.tokenize(text)).to eq(["d'", "art", "de", "l'", "univers", ",", "c'" ,"est", "un", "art"])
     end
   end
 end

data/spec/performance_spec.rb

@@ -1,4 +1,3 @@
-# -*- encoding : utf-8 -*-
 require 'benchmark'
 require 'spec_helper'
 require 'stackprof'

data/spec/spec_helper.rb

@@ -1,2 +1,2 @@
-$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
 require 'pragmatic_tokenizer'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.2.0
 platform: ruby
 authors:
 - Kevin S. Dias
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-02 00:00:00.000000000 Z
+date: 2020-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode
@@ -42,16 +42,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -111,7 +111,6 @@ files:
 - README.md
 - Rakefile
 - lib/pragmatic_tokenizer.rb
-- lib/pragmatic_tokenizer/full_stop_separator.rb
 - lib/pragmatic_tokenizer/languages.rb
 - lib/pragmatic_tokenizer/languages/arabic.rb
 - lib/pragmatic_tokenizer/languages/bulgarian.rb
@@ -140,6 +139,7 @@ files:
 - lib/pragmatic_tokenizer/languages/turkish.rb
 - lib/pragmatic_tokenizer/post_processor.rb
 - lib/pragmatic_tokenizer/pre_processor.rb
+- lib/pragmatic_tokenizer/regex.rb
 - lib/pragmatic_tokenizer/tokenizer.rb
 - lib/pragmatic_tokenizer/version.rb
 - pragmatic_tokenizer.gemspec
@@ -153,7 +153,7 @@ files:
 homepage: https://github.com/diasks2/pragmatic_tokenizer
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -168,9 +168,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
 specification_version: 4
 summary: A multilingual tokenizer
 test_files:

data/lib/pragmatic_tokenizer/full_stop_separator.rb

@@ -1,62 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module PragmaticTokenizer
-  # This class separates true full stops while ignoring
-  # periods that are part of an abbreviation
-  class FullStopSeparator
-
-    REGEXP_ENDS_WITH_DOT   = /\A(.*\w)\.\z/
-    REGEXP_ONLY_LETTERS    = /\A[a-z]\z/i
-    REGEXP_ABBREVIATION    = /[a-z](?:\.[a-z])+\z/i
-    DOT                    = '.'.freeze
-
-    def initialize(tokens:, abbreviations:, downcase:)
-      @tokens        = tokens
-      @abbreviations = abbreviations
-      @downcase      = downcase
-    end
-
-    def separate
-      create_cleaned_tokens
-      replace_last_token unless @cleaned_tokens.empty?
-      @cleaned_tokens
-    end
-
-    private
-
-      def create_cleaned_tokens
-        @cleaned_tokens = []
-        @tokens.each_with_index do |token, position|
-          if @tokens[position + 1] && token =~ REGEXP_ENDS_WITH_DOT
-            match = Regexp.last_match(1)
-            if abbreviation?(match)
-              @cleaned_tokens += [match, DOT]
-              next
-            end
-          end
-          @cleaned_tokens << token
-        end
-      end
-
-      def abbreviation?(token)
-        !defined_abbreviation?(token) && token !~ REGEXP_ONLY_LETTERS && token !~ REGEXP_ABBREVIATION
-      end
-
-      def defined_abbreviation?(token)
-        @abbreviations.include?(inverse_case(token))
-      end
-
-      def inverse_case(token)
-        @downcase ? token : Unicode.downcase(token)
-      end
-
-      def replace_last_token
-        last_token = @cleaned_tokens[-1]
-        return if defined_abbreviation?(last_token.chomp(DOT)) || last_token !~ REGEXP_ENDS_WITH_DOT
-        @cleaned_tokens[-1] = Regexp.last_match(1)
-        @cleaned_tokens << DOT
-      end
-
-  end
-
-end