pragmatic_tokenizer 1.3.1 → 1.4.0

data/spec/languages/french_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe PragmaticTokenizer do
+  context 'Language: French (fr)' do
+    it 'tokenizes a string #001' do
+      text = "L'art de l'univers, c'est un art"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'fr'
+      )
+      expect(pt.tokenize).to eq(["l'", "art", "de", "l'", "univers", ",", "c'est", "un", "art"])
+    end
+  end
+end

data/spec/performance_spec.rb

@@ -0,0 +1,62 @@
+# -*- encoding : utf-8 -*-
+require 'benchmark'
+require 'spec_helper'
+require 'stackprof'
+
+describe PragmaticTokenizer do
+
+  # Speed benchmarks tests
+
+  # it 'is fast?' do
+  #   string = "Hello World. My name is Jonas. What is your name? My name is Jonas. There it is! I found it. My name is Jonas E. Smith. Please turn to p. 55. Were Jane and co. at the party? They closed the deal with Pitt, Briggs & Co. at noon. Let's ask Jane and co. They should know. They closed the deal with Pitt, Briggs & Co. It closed yesterday. I can't see Mt. Fuji from here. St. Michael's Church is on 5th st. near the light. That is JFK Jr.'s book. I visited the U.S.A. last year. I live in the E.U. How about you? I live in the U.S. How about you? I work for the U.S. Government in Virginia. I have lived in the U.S. for 20 years. She has $100.00 in her bag. She has $100.00. It is in her bag. He teaches science (He previously worked for 5 years as an engineer.) at the local University. Her email is Jane.Doe@example.com. I sent her an email. The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out. She turned to him, 'This is great.' she said. She turned to him, \"This is great.\" she said. She turned to him, \"This is great.\" She held the book out to show him. Hello!! Long time no see. Hello?? Who is there? Hello!? Is that you? Hello?! Is that you? 1.) The first item 2.) The second item 1.) The first item. 2.) The second item. 1) The first item 2) The second item 1) The first item. 2) The second item. 1. The first item 2. The second item 1. The first item. 2. The second item. • 9. The first item • 10. The second item ⁃9. The first item ⁃10. The second item a. The first item b. The second item c. The third list item This is a sentence\ncut off in the middle because pdf. It was a cold \nnight in the city. features\ncontact manager\nevents, activities\n You can find it at N°. 1026.253.553. That is where the treasure is. She works at Yahoo! in the accounting department. We make a good team, you and I. Did you see Albert I. Jones yesterday? Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .” \"Bohr [...] used the analogy of parallel stairways [...]\" (Smith 55). If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence. I never meant that.... She left the store. I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it. One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . ."
+  #   benchmark do
+  #     10.times do
+  #       data = StackProf.run(mode: :cpu, interval: 1000) do
+  #         PragmaticTokenizer::Tokenizer.new(string * 100,
+  #           language: 'en',
+  #           clean: true,
+  #           remove_numbers: true,
+  #           minimum_length: 3,
+  #           expand_contractions: true,
+  #           remove_stop_words: true
+  #         ).tokenize
+  #       end
+  #       puts StackProf::Report.new(data).print_text
+  #     end
+  #   end
+  # end
+
+  # 26.8
+  # 8.2
+  # 9.6
+  # it 'is fast? (long strings)' do
+  #   string = "Hello World. My name is Jonas. What is your name? My name is Jonas IV Smith. There it is! I found it. My name is Jonas E. Smith. Please turn to p. 55. Were Jane and co. at the party? They closed the deal with Pitt, Briggs & Co. at noon. Let's ask Jane and co. They should know. They closed the deal with Pitt, Briggs & Co. It closed yesterday. I can't see Mt. Fuji from here. St. Michael's Church is on 5th st. near the light. That is JFK Jr.'s book. I visited the U.S.A. last year. I live in the E.U. How about you? I live in the U.S. How about you? I work for the U.S. Government in Virginia. I have lived in the U.S. for 20 years. She has $100.00 in her bag. She has $100.00. It is in her bag. He teaches science (He previously worked for 5 years as an engineer.) at the local University. Her email is Jane.Doe@example.com. I sent her an email. The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out. She turned to him, 'This is great.' she said. She turned to him, \"This is great.\" she said. She turned to him, \"This is great.\" She held the book out to show him. Hello!! Long time no see. Hello?? Who is there? Hello!? Is that you? Hello?! Is that you? 1.) The first item 2.) The second item 1.) The first item. 2.) The second item. 1) The first item 2) The second item 1) The first item. 2) The second item. 1. The first item 2. The second item 1. The first item. 2. The second item. • 9. The first item • 10. The second item ⁃9. The first item ⁃10. The second item a. The first item b. The second item c. The third list item This is a sentence\ncut off in the middle because pdf. It was a cold \nnight in the city. features\ncontact manager\nevents, activities\n You can find it at N°. 1026.253.553. That is where the treasure is. She works at Yahoo! in the accounting department. We make a good team, you and I. Did you see Albert I. Jones yesterday? Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .” \"Bohr [...] used the analogy of parallel stairways [...]\" (Smith 55). If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence. I never meant that.... She left the store. I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it. One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . ." * 1000
+  #   puts "LENGTH: #{string.length}"
+  #   benchmark do
+  #     PragmaticTokenizer::Tokenizer.new(string,
+  #       language: 'en',
+  #       clean: true,
+  #       remove_numbers: true,
+  #       minimum_length: 3,
+  #       expand_contractions: true,
+  #       remove_stop_words: true,
+  #       remove_roman_numerals: true,
+  #       punctuation: 'none'
+  #     ).tokenize
+  #   end
+  # end
+
+  # it 'is the baseline' do
+  #   string = "Hello World. My name is Jonas. What is your name? My name is Jonas IV Smith. There it is! I found it. My name is Jonas E. Smith. Please turn to p. 55. Were Jane and co. at the party? They closed the deal with Pitt, Briggs & Co. at noon. Let's ask Jane and co. They should know. They closed the deal with Pitt, Briggs & Co. It closed yesterday. I can't see Mt. Fuji from here. St. Michael's Church is on 5th st. near the light. That is JFK Jr.'s book. I visited the U.S.A. last year. I live in the E.U. How about you? I live in the U.S. How about you? I work for the U.S. Government in Virginia. I have lived in the U.S. for 20 years. She has $100.00 in her bag. She has $100.00. It is in her bag. He teaches science (He previously worked for 5 years as an engineer.) at the local University. Her email is Jane.Doe@example.com. I sent her an email. The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out. She turned to him, 'This is great.' she said. She turned to him, \"This is great.\" she said. She turned to him, \"This is great.\" She held the book out to show him. Hello!! Long time no see. Hello?? Who is there? Hello!? Is that you? Hello?! Is that you? 1.) The first item 2.) The second item 1.) The first item. 2.) The second item. 1) The first item 2) The second item 1) The first item. 2) The second item. 1. The first item 2. The second item 1. The first item. 2. The second item. • 9. The first item • 10. The second item ⁃9. The first item ⁃10. The second item a. The first item b. The second item c. The third list item This is a sentence\ncut off in the middle because pdf. It was a cold \nnight in the city. features\ncontact manager\nevents, activities\n You can find it at N°. 1026.253.553. That is where the treasure is. She works at Yahoo! in the accounting department. We make a good team, you and I. Did you see Albert I. Jones yesterday? Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .” \"Bohr [...] used the analogy of parallel stairways [...]\" (Smith 55). If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence. I never meant that.... She left the store. I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it. One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . ." * 1000
+  #   puts "LENGTH: #{string.length}"
+  #   benchmark do
+  #     string.split(' ')
+  #   end
+  # end
+end
+
+def benchmark(&block)
+  block.call
+  time = Benchmark.realtime { block.call }
+  puts "RUNTIME: #{time}"
+end

data/spec/pragmatic_tokenizer_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe PragmaticTokenizer do
+  it 'has a version number' do
+    expect(PragmaticTokenizer::VERSION).not_to be nil
+  end
+
+  describe '#initialize' do
+    it 'raises an error if the text argument is nil' do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new(nil, language: 'en').tokenize).to raise_error }
+    end
+
+    it 'raises an error if the text argument is empty' do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new('', language: 'en').tokenize).to raise_error }
+    end
+
+    it 'raises an error if minimum_length is not an Integer' do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new("heelo", minimum_length: "strawberry").tokenize).to raise_error }
+    end
+
+    it 'raises an error if long_word_split is not an Integer' do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new("heeloo", long_word_split: "yes!").tokenize).to raise_error }
+    end
+
+    it 'raises an error if the text is not a String' do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new(5).tokenize).to raise_error }
+    end
+
+    it "raises an error if the punctuation argument is not nil, 'all', 'semi', or 'none'" do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new('', language: 'en', punctuation: 'world').tokenize).to raise_error }
+    end
+
+    it "raises an error if the numbers argument is not nil, 'all', 'semi', or 'none'" do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new('', language: 'en', numbers: 'world').tokenize).to raise_error }
+    end
+
+    it "raises an error if the mentions argument is not nil, 'all', 'semi', or 'none'" do
+      lambda { expect(PragmaticTokenizer::Tokenizer.new('', language: 'en', mentions: 'world').tokenize).to raise_error }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'pragmatic_tokenizer'

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
 date: 2016-01-22 00:00:00.000000000 Z
 dependencies:
@@ -94,8 +94,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
 - lib/pragmatic_tokenizer.rb
 - lib/pragmatic_tokenizer/ending_punctuation_separator.rb
 - lib/pragmatic_tokenizer/full_stop_separator.rb
@@ -130,6 +128,13 @@ files:
 - lib/pragmatic_tokenizer/tokenizer.rb
 - lib/pragmatic_tokenizer/version.rb
 - pragmatic_tokenizer.gemspec
+- spec/languages/bulgarian_spec.rb
+- spec/languages/deutsch_spec.rb
+- spec/languages/english_spec.rb
+- spec/languages/french_spec.rb
+- spec/performance_spec.rb
+- spec/pragmatic_tokenizer_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/diasks2/pragmatic_tokenizer
 licenses: []
 metadata: {}
@@ -153,4 +158,11 @@ rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: A multilingual tokenizer
-test_files: []
+test_files:
+- spec/languages/bulgarian_spec.rb
+- spec/languages/deutsch_spec.rb
+- spec/languages/english_spec.rb
+- spec/languages/french_spec.rb
+- spec/performance_spec.rb
+- spec/pragmatic_tokenizer_spec.rb
+- spec/spec_helper.rb

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "pragmatic_tokenizer"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start

data/bin/setup

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-IFS=$'\n\t'
-
-bundle install
-
-# Do any other automated setup that you need to do here