pragmatic_tokenizer 1.5.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41ba3758c8ff32d83451b66e0b28ca8f33248843
-  data.tar.gz: 1f96d2fbf5a7a66a3031631b58a9b91b77991fc5
+  metadata.gz: b2372718412cd437eac22feb65b9bdae0798c26c
+  data.tar.gz: a9f6d1d84b00494232fa7b84a82de5c4cd8c7700
 SHA512:
-  metadata.gz: 7e62958fbf69b55d62c00e391a9c6ed8da4c55c0336e0160c172363198d3a1a711dbdc6310a94e54a9a65316b9c34913a886b92ee7eb6afe78530556a185663b
-  data.tar.gz: 9fbb6d481494ef235fd1d74b9c34833bb96887c29d530c28add6ee3e5109379b051f2a7520208f9385d686e0ec2ba8c655c849dfb43a6d7b850c3ec0092b526f
+  metadata.gz: 2f7c7dad88d79b99de60a5073d4af18cbae5d6395b141425d76b3edc278e2260a7623abc1a3aa9f861593d6b935a06254d2248fc1408e223d01a0f56fd12c39d
+  data.tar.gz: 5973501489bd774e81eb68b9fc0b8e1b061a3f13d360f0cbc888975c44c43169f720efd9f76ce5f20f3e853248cdd4a013502e8076193f30169660bc3595a446

data/.rubocop.yml

@@ -49,9 +49,9 @@ Style/MultilineMethodCallIndentation:
   EnforcedStyle: indented
   IndentationWidth: 4
 
-# unsure how we'd want it
-#Style/MultilineOperationIndentation:
-#  IndentationWidth: 4
+Style/MultilineOperationIndentation:
+  EnforcedStyle: indented
+  IndentationWidth: 4
 
 # these are not continued (e.g. 'private'), so we keep them at 2 chars
 Style/AccessModifierIndentation:

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2016-01-23 03:18:41 +0100 using RuboCop version 0.36.0.
+# on 2016-01-24 21:09:34 +0100 using RuboCop version 0.36.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -8,30 +8,30 @@
 
 # Offense count: 11
 Metrics/AbcSize:
-  Max: 137
+  Max: 118
 
 # Offense count: 2
 # Configuration parameters: CountComments.
 Metrics/ClassLength:
-  Max: 214
+  Max: 218
 
-# Offense count: 9
+# Offense count: 7
 Metrics/CyclomaticComplexity:
-  Max: 41
+  Max: 40
 
-# Offense count: 8
+# Offense count: 7
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
   Max: 57
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: CountComments.
 Metrics/ModuleLength:
-  Max: 134
+  Max: 140
 
 # Offense count: 6
 Metrics/PerceivedComplexity:
-  Max: 43
+  Max: 41
 
 # Offense count: 4
 # Cop supports --auto-correct.
@@ -43,10 +43,12 @@ Style/CommentIndentation:
 Style/Documentation:
   Enabled: false
 
-# Offense count: 2
-Style/MultilineBlockChain:
-  Exclude:
-    - 'lib/pragmatic_tokenizer/post_processor.rb'
+# Offense count: 17
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
+# SupportedStyles: aligned, indented
+Style/MultilineOperationIndentation:
+  Enabled: false
 
 # Offense count: 1
 # Configuration parameters: SuspiciousParamNames.
@@ -55,12 +57,11 @@ Style/OptionHash:
   Exclude:
     - 'lib/pragmatic_tokenizer/tokenizer.rb'
 
-# Offense count: 4
+# Offense count: 3
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, AllowInnerSlashes.
 # SupportedStyles: slashes, percent_r, mixed
 Style/RegexpLiteral:
   Exclude:
     - 'lib/pragmatic_tokenizer/post_processor.rb'
-    - 'lib/pragmatic_tokenizer/pre_processor.rb'
     - 'lib/pragmatic_tokenizer/tokenizer.rb'

data/lib/pragmatic_tokenizer/full_stop_separator.rb

@@ -4,10 +4,11 @@ module PragmaticTokenizer
   # This class separates true full stops while ignoring
   # periods that are part of an abbreviation
   class FullStopSeparator
-    attr_reader :tokens, :abbreviations
-    def initialize(tokens:, abbreviations:)
+    attr_reader :tokens, :abbreviations, :downcase
+    def initialize(tokens:, abbreviations:, downcase:)
       @tokens = tokens
       @abbreviations = abbreviations
+      @downcase = downcase
     end
 
     def separate
@@ -19,7 +20,12 @@ module PragmaticTokenizer
       tokens.each_with_index do |_t, i|
         if tokens[i + 1] && tokens[i] =~ /\A(.+)\.\z/
           w = Regexp.last_match(1)
-          unless abbr[Unicode.downcase(w)] || w =~ /\A[a-z]\z/i ||
+          if downcase
+            abbreviation = abbr[w]
+          else
+            abbreviation = abbr[Unicode.downcase(w)]
+          end
+          unless abbreviation || w =~ /\A[a-z]\z/i ||
                  w =~ /[a-z](?:\.[a-z])+\z/i
             cleaned_tokens << w
             cleaned_tokens << '.'
@@ -28,7 +34,12 @@ module PragmaticTokenizer
         end
         cleaned_tokens << tokens[i]
       end
-      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbreviations.include?(Unicode.downcase(cleaned_tokens[-1]).chomp("."))
+      if downcase
+        abbreviation = abbreviations.include?(cleaned_tokens[-1].chomp(".")) unless cleaned_tokens[-1].nil?
+      else
+        abbreviation = abbreviations.include?(Unicode.downcase(cleaned_tokens[-1]).chomp(".")) unless cleaned_tokens[-1].nil?
+      end
+      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbreviation
         cleaned_tokens[-1] = Regexp.last_match(1)
         cleaned_tokens.push '.'
       end

data/lib/pragmatic_tokenizer/post_processor.rb

@@ -1,66 +1,115 @@
 module PragmaticTokenizer
   class PostProcessor
 
-    attr_reader :text, :abbreviations
-    def initialize(text:, abbreviations:)
-      @text = text
+    REGEX_SYMBOL         = /[♳ ♴ ♵ ♶ ♷ ♸ ♹ ♺ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☠ ☢ ☣ ☤ ☥ ☦ ☧ ☀ ☁ ☂ ☃ ☄ ☮ ♔ ♕ ♖ ♗ ♘ ♙ ♚ ⚘ ⚭]/.freeze
+    REGEXP_COMMAS        = /^(,|‚)+/.freeze
+    REGEXP_SINGLE_QUOTES = /(.+)(’|'|‘|`)$/.freeze
+    REGEXP_SLASH         = /^(?!(https?:|www\.))(.*)\/(.*)/.freeze
+    REGEXP_QUESTION_MARK = /^(?!(https?:|www\.))(.*)(\?)(.*)/.freeze
+    REGEXP_PLUS_SIGN     = /(.+)\+(.+)/.freeze
+    REGEXP_COLON         = /^(\:)(\S{2,})/.freeze
+    REGEXP_EMOJI         = /(\u{2744}[\u{FE0E}|\u{FE0F}])/.freeze
+
+    REGEX_UNIFIED1       = Regexp.union(REGEXP_SLASH,
+                                        REGEXP_QUESTION_MARK,
+                                        REGEXP_PLUS_SIGN,
+                                        REGEXP_COLON,
+                                        REGEXP_EMOJI,
+                                        PragmaticTokenizer::Languages::Common::PREFIX_EMOJI_REGEX,
+                                        PragmaticTokenizer::Languages::Common::POSTFIX_EMOJI_REGEX
+    ).freeze
+
+    REGEX_UNIFIED2       = Regexp.union(REGEXP_SINGLE_QUOTES,
+                                        REGEXP_COMMAS
+    ).freeze
+
+    attr_reader :text, :abbreviations, :downcase
+
+    def initialize(text:, abbreviations:, downcase:)
+      @text          = text
       @abbreviations = abbreviations
+      @downcase      = downcase
     end
 
     def post_process
-      tokens = text.split
-          .flat_map { |t| (t[0] == '‚' || t[0] == ',') && t.length > 1 ? t.split(/(,|‚)/).flatten : t }
-          .flat_map { |t| (t[-1] == '’' || t[-1] == "'" || t[-1] == '‘' || t[-1] == '`') && t.length > 1 ? t.split(/(’|'|‘|`)/).flatten : t }
-          .map { |t| convert_sym_to_punct(t) }
-      full_stop_separated_tokens = FullStopSeparator.new(tokens: tokens, abbreviations: abbreviations).separate
-      EndingPunctuationSeparator.new(tokens: EndingPunctuationSeparator.new(tokens: full_stop_separated_tokens).separate.flat_map { |t| t.include?("/") && t !~ /(http|https|www)(\.|:)/ ? t.gsub!(/\//, '\1 \2').split(' ').flatten : t }
-        .flat_map { |t| t.include?("?") && t !~ /(http|https|www)(\.|:)/ && t.length > 1 ? t.gsub(/\?/, '\1 \2').split(' ').flatten : t }
-        .flat_map { |t| t.include?("+") ? t.gsub!(/\+/, '\1 \2').split(' ').flatten : t }
-        .flat_map { |t| t =~ /\A\.[^\.]/ && t.length > 1 ? t.gsub(/\./, '\1 ').split(' ').flatten : t }
-        .flat_map { |t| t =~ /\A\:\S{2,}/ ? t.gsub(/\:/, ': ').split(' ').flatten : t }
-        .flat_map do |t|
-          (
-          if t.include?(".") &&
-              t !~ /(http|https|www)(\.|:)/ &&
-              t !~ /\.(com|net|org|edu|gov|mil|int)/ &&
-              t !~ /\.[a-z]{2}/ &&
-              t.length > 2 &&
-              t !~ /\A[a-zA-Z]{1}\./ &&
-              t.count(".") == 1 &&
-              t !~ /\d+/ &&
-              !abbreviations.include?(Unicode.downcase(t.split(".")[0].nil? ? '' : t.split(".")[0])) &&
-              t !~ /\S+(＠|@)\S+/
-            t.gsub(/\./, '\1 . \2').split(' ').flatten
-          else
-            t
-          end)
-        end
-        .flat_map do |t|
-          (
-          if t.include?(".") &&
-              t !~ /(http|https|www)(\.|:)/ &&
-              t.length > 1 &&
-              t !~ /(\s+|\A)[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}(:[0-9]{1,5})?(\/.*)?/ix &&
-              t !~ /\S+(＠|@)\S+/ &&
-              abbreviations.include?(Unicode.downcase(t.split(".")[0].nil? ? '' : t.split(".")[0]))
-            t.gsub(/\./, '\1. \2').split(' ').flatten
-          else
-            t
-          end)
-        end
-        .flat_map { |t| t =~ /\u{2744}\u{FE0F}/ ? t.gsub(/\u{2744}\u{FE0F}/, " \u{2744}\u{FE0F} ").split(' ').flatten : t }
-        .flat_map { |t| t =~ /\u{2744}\u{FE0E}/ ? t.gsub(/\u{2744}\u{FE0E}/, " \u{2744}\u{FE0E} ").split(' ').flatten : t }
-        .flat_map { |t| t =~ /(\A|\S)\u{2744}[^\u{FE0E}|\u{FE0F}]/ ? t.gsub(/\u{2744}/, " \u{2744} ").split(' ').flatten : t }
-        .flat_map { |t| t =~ PragmaticTokenizer::Languages::Common::PREFIX_EMOJI_REGEX ? t.gsub(PragmaticTokenizer::Languages::Common::PREFIX_EMOJI_REGEX, '\1 \2').split(' ').flatten : t }
-        .flat_map { |t| t =~ PragmaticTokenizer::Languages::Common::POSTFIX_EMOJI_REGEX ? t.gsub(PragmaticTokenizer::Languages::Common::POSTFIX_EMOJI_REGEX, '\1 \2').split(' ').flatten : t }
-                                    ).separate
+      EndingPunctuationSeparator.new(tokens: method_name3).separate
     end
 
     private
 
+      def method_name3
+        separated = EndingPunctuationSeparator.new(tokens: full_stop_separated_tokens).separate
+        procs     = [unified1, split_unknown_period1, split_unknown_period2, split_emoji]
+        procs.reduce(separated) { |a, e| a.flat_map(&e) }
+      end
+
+      def unified1
+        proc { |token| token.split(REGEX_UNIFIED1) }
+      end
+
+      def full_stop_separated_tokens
+        FullStopSeparator.new(tokens: split_and_convert_commas_and_quotes, abbreviations: abbreviations, downcase: downcase).separate
+      end
+
+      def split_and_convert_commas_and_quotes
+        text
+            .split
+            .flat_map { |token| token.split(REGEX_UNIFIED2) }
+            .flat_map { |token| convert_sym_to_punct(token) }
+      end
+
+      def split_emoji
+        proc { |token| (token =~ /(\A|\S)\u{2744}[^\u{FE0E}|\u{FE0F}]/) ? token.split(/(\u{2744})/) : token }
+      end
+
+      def split_unknown_period1
+        proc { |token| unknown_period1?(token) ? token.split(/(.*\.)/) : token }
+      end
+
+      def split_unknown_period2
+        proc { |token| unknown_period2?(token) ? token.split(/(\.)/) : token }
+      end
+
+      def unknown_period1?(token)
+        token.include?(".") &&
+            token !~ /(http|https|www)(\.|:)/ &&
+            token.length > 1 &&
+            token !~ /(\s+|\A)[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}(:[0-9]{1,5})?(\/.*)?/ix &&
+            token !~ /\S+(＠|@)\S+/ &&
+            abbreviations.include?(extract_abbreviation(token))
+      end
+
+      def unknown_period2?(token)
+        token.include?(".") &&
+            token !~ /(http|https|www)(\.|:)/ &&
+            token !~ /\.(com|net|org|edu|gov|mil|int)/ &&
+            token !~ /\.[a-zA-Z]{2}(\s|\z)/ &&
+            token.length > 2 &&
+            token !~ /\A[a-zA-Z]{1}\./ &&
+            token.count(".") == 1 &&
+            token !~ /\d+/ &&
+            !abbreviations.include?(extract_abbreviation(token)) &&
+            token !~ /\S+(＠|@)\S+/
+      end
+
+      def extract_abbreviation(token)
+        if downcase
+          token.split(/(\.)/)[0]
+        else
+          Unicode.downcase(token.split(/(\.)/)[0])
+        end
+      end
+
       def convert_sym_to_punct(token)
-        symbol_matches = /[♳ ♴ ♵ ♶ ♷ ♸ ♹ ♺ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☠ ☢ ☣ ☤ ☥ ☦ ☧ ☀ ☁ ☂ ☃ ☄ ☮ ♔ ♕ ♖ ♗ ♘ ♙ ♚ ⚘ ⚭]/.match(token)
-        symbol_matches.nil? ? token : token.gsub!(symbol_matches[0], PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP.key(symbol_matches[0]))
+        symbol_matches = REGEX_SYMBOL.match(token)
+        if symbol_matches.nil?
+          token
+        else
+          pattern     = symbol_matches[0]
+          replacement = PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP.key(pattern)
+          token.gsub!(pattern, replacement)
+        end
       end
+
   end
 end

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -127,8 +127,11 @@ module PragmaticTokenizer
       end
 
       def post_process(text)
-        @tokens = PostProcessor.new(text: text, abbreviations: abbreviations).post_process
-        downcase! if downcase
+        if downcase
+          @tokens = PostProcessor.new(text: Unicode.downcase(text), abbreviations: abbreviations, downcase: downcase).post_process
+        else
+          @tokens = PostProcessor.new(text: text, abbreviations: abbreviations, downcase: downcase).post_process
+        end
         expand_contractions!(contractions) if expand_contractions
         clean! if clean
         classic_filter! if classic_filter
@@ -146,15 +149,11 @@ module PragmaticTokenizer
         @tokens.reject(&:empty?)
       end
 
-      def downcase!
-        @tokens.map! { |t| Unicode.downcase(t) }
-      end
-
       def expand_contractions!(contractions)
         @tokens = if downcase
                     @tokens.flat_map do |t|
-                      if contractions.key?(Unicode.downcase(t.gsub(/[‘’‚‛‹›＇´`]/, "'")))
-                        contractions[Unicode.downcase(t.gsub(/[‘’‚‛‹›＇´`]/, "'"))]
+                      if contractions.key?(t.gsub(/[‘’‚‛‹›＇´`]/, "'"))
+                        contractions[t.gsub(/[‘’‚‛‹›＇´`]/, "'")]
                             .split(' ')
                             .flatten
                       else
@@ -212,7 +211,11 @@ module PragmaticTokenizer
         when 'semi'
           @tokens.delete_if { |t| t =~ /\A\d+\z/ }
         when 'none'
-          @tokens.delete_if { |t| t =~ /\D*\d+\d*/ || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?(Unicode.downcase(t)) || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?("#{Unicode.downcase(t)}.") }
+          if downcase
+            @tokens.delete_if { |t| t =~ /\D*\d+\d*/ || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?(t) || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?("#{t}.") }
+          else
+            @tokens.delete_if { |t| t =~ /\D*\d+\d*/ || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?(Unicode.downcase(t)) || PragmaticTokenizer::Languages::Common::ROMAN_NUMERALS.include?("#{Unicode.downcase(t)}.") }
+          end
         when 'only'
           @tokens.delete_if { |t| t =~ /\A\D+\z/ }
         end

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "1.5.0".freeze
+  VERSION = "1.5.1".freeze
 end

data/spec/performance_spec.rb

@@ -8,21 +8,18 @@ describe PragmaticTokenizer do
 
   # it 'is fast?' do
   #   string = "Hello World. My name is Jonas. What is your name? My name is Jonas. There it is! I found it. My name is Jonas E. Smith. Please turn to p. 55. Were Jane and co. at the party? They closed the deal with Pitt, Briggs & Co. at noon. Let's ask Jane and co. They should know. They closed the deal with Pitt, Briggs & Co. It closed yesterday. I can't see Mt. Fuji from here. St. Michael's Church is on 5th st. near the light. That is JFK Jr.'s book. I visited the U.S.A. last year. I live in the E.U. How about you? I live in the U.S. How about you? I work for the U.S. Government in Virginia. I have lived in the U.S. for 20 years. She has $100.00 in her bag. She has $100.00. It is in her bag. He teaches science (He previously worked for 5 years as an engineer.) at the local University. Her email is Jane.Doe@example.com. I sent her an email. The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out. She turned to him, 'This is great.' she said. She turned to him, \"This is great.\" she said. She turned to him, \"This is great.\" She held the book out to show him. Hello!! Long time no see. Hello?? Who is there? Hello!? Is that you? Hello?! Is that you? 1.) The first item 2.) The second item 1.) The first item. 2.) The second item. 1) The first item 2) The second item 1) The first item. 2) The second item. 1. The first item 2. The second item 1. The first item. 2. The second item. • 9. The first item • 10. The second item ⁃9. The first item ⁃10. The second item a. The first item b. The second item c. The third list item This is a sentence\ncut off in the middle because pdf. It was a cold \nnight in the city. features\ncontact manager\nevents, activities\n You can find it at N°. 1026.253.553. That is where the treasure is. She works at Yahoo! in the accounting department. We make a good team, you and I. Did you see Albert I. Jones yesterday? Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .” \"Bohr [...] used the analogy of parallel stairways [...]\" (Smith 55). If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence. I never meant that.... She left the store. I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it. One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . ."
-  #   benchmark do
-  #     10.times do
-  #       data = StackProf.run(mode: :cpu, interval: 1000) do
-  #         PragmaticTokenizer::Tokenizer.new(string * 100,
-  #           language: 'en',
-  #           clean: true,
-  #           remove_numbers: true,
-  #           minimum_length: 3,
-  #           expand_contractions: true,
-  #           remove_stop_words: true
-  #         ).tokenize
-  #       end
-  #       puts StackProf::Report.new(data).print_text
-  #     end
+  #   data = StackProf.run(mode: :cpu, interval: 1000) do
+  #     PragmaticTokenizer::Tokenizer.new(string * 100,
+  #       language: 'en',
+  #       clean: true,
+  #       minimum_length: 3,
+  #       expand_contractions: true,
+  #       remove_stop_words: true,
+  #       numbers: :none,
+  #       punctuation: :none
+  #     ).tokenize
   #   end
+  #   puts StackProf::Report.new(data).print_text
   # end
 
   # 26.8
@@ -30,6 +27,7 @@ describe PragmaticTokenizer do
   # 9.6
   # 23.25
   # 24.2
+  # 23.2
   # it 'is fast? (long strings)' do
   #   string = "Hello World. My name is Jonas. What is your name? My name is Jonas IV Smith. There it is! I found it. My name is Jonas E. Smith. Please turn to p. 55. Were Jane and co. at the party? They closed the deal with Pitt, Briggs & Co. at noon. Let's ask Jane and co. They should know. They closed the deal with Pitt, Briggs & Co. It closed yesterday. I can't see Mt. Fuji from here. St. Michael's Church is on 5th st. near the light. That is JFK Jr.'s book. I visited the U.S.A. last year. I live in the E.U. How about you? I live in the U.S. How about you? I work for the U.S. Government in Virginia. I have lived in the U.S. for 20 years. She has $100.00 in her bag. She has $100.00. It is in her bag. He teaches science (He previously worked for 5 years as an engineer.) at the local University. Her email is Jane.Doe@example.com. I sent her an email. The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out. She turned to him, 'This is great.' she said. She turned to him, \"This is great.\" she said. She turned to him, \"This is great.\" She held the book out to show him. Hello!! Long time no see. Hello?? Who is there? Hello!? Is that you? Hello?! Is that you? 1.) The first item 2.) The second item 1.) The first item. 2.) The second item. 1) The first item 2) The second item 1) The first item. 2) The second item. 1. The first item 2. The second item 1. The first item. 2. The second item. • 9. The first item • 10. The second item ⁃9. The first item ⁃10. The second item a. The first item b. The second item c. The third list item This is a sentence\ncut off in the middle because pdf. It was a cold \nnight in the city. features\ncontact manager\nevents, activities\n You can find it at N°. 1026.253.553. That is where the treasure is. She works at Yahoo! in the accounting department. We make a good team, you and I. Did you see Albert I. Jones yesterday? Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .” \"Bohr [...] used the analogy of parallel stairways [...]\" (Smith 55). If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence. I never meant that.... She left the store. I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it. One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . ." * 1000
   #   puts "LENGTH: #{string.length}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-24 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode