pragmatic_tokenizer 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46da26b8afc38bfc2699a2875d9786be47702160
-  data.tar.gz: 9073a0505244bb97d7fa51c48e9fe4ab30983c90
+  metadata.gz: 41ba3758c8ff32d83451b66e0b28ca8f33248843
+  data.tar.gz: 1f96d2fbf5a7a66a3031631b58a9b91b77991fc5
 SHA512:
-  metadata.gz: 8397bca6ada5fae51d1894b26154b5f9fed73e375b9e78b2803da892c20e5f32efb2722f0fe767cbc171586a5a6979a720041de5f0fd5c50824a0498c55b8394
-  data.tar.gz: f58cae264490ce16bef8e5512760e992dd38a11ced8fa872ab5faff2e92f61e6414f7e9ffd732f1e2910de560b2e9a2e98e1d59aa7039426ee0fbf3d8daaaa1f
+  metadata.gz: 7e62958fbf69b55d62c00e391a9c6ed8da4c55c0336e0160c172363198d3a1a711dbdc6310a94e54a9a65316b9c34913a886b92ee7eb6afe78530556a185663b
+  data.tar.gz: 9fbb6d481494ef235fd1d74b9c34833bb96887c29d530c28add6ee3e5109379b051f2a7520208f9385d686e0ec2ba8c655c849dfb43a6d7b850c3ec0092b526f

data/.rubocop.yml

@@ -0,0 +1,184 @@
+#inherit_from: .rubocop_todo.yml
+
+# check https://gist.github.com/jhass/a5ae80d87f18e53e7b56/84972ca6c0c5a59768ae07ee0a639d03cc72d375
+
+Metrics/LineLength:
+  Max: 99
+  Enabled: false  # as it complains about long comments too
+
+Style/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Style/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/ExtraSpacing:
+  AllowForAlignment: true
+
+Style/IndentationConsistency:
+  EnforcedStyle: rails
+
+Style/AndOr:
+  Enabled: false  # complains about "redirect_to ... and return" in Rails controllers
+
+# Fail is an alias of raise. Avoid aliases, it's more cognitive load for no gain.
+# The argument that fail should be used to abort the program is wrong too,
+# there's Kernel#abort for that.
+Style/SignalException:
+  EnforcedStyle: only_raise
+
+Style/StringLiterals:
+  Enabled: false
+
+# allow Rubymine "Continuation indent" of 4 characters:
+Style/CaseIndentation:
+  IndentationWidth: 4
+Style/FirstParameterIndentation:
+  IndentationWidth: 4
+Style/IndentArray:
+  IndentationWidth: 4
+Style/IndentAssignment:
+  IndentationWidth: 4
+Style/IndentHash:
+  IndentationWidth: 4
+
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+  IndentationWidth: 4
+
+# unsure how we'd want it
+#Style/MultilineOperationIndentation:
+#  IndentationWidth: 4
+
+# these are not continued (e.g. 'private'), so we keep them at 2 chars
+Style/AccessModifierIndentation:
+  IndentationWidth: 2
+
+Style/SpaceAroundOperators:
+  AllowForAlignment: true
+
+# Most readable form.
+Style/AlignHash:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+
+# Mixing the styles looks just silly.
+Style/HashSyntax:
+  EnforcedStyle: ruby19_no_mixed_keys
+
+# consider this to keep method parameters short
+# No space makes the method definition shorter and differentiates
+# from a regular assignment.
+Style/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: no_space
+
+Rails:
+  Enabled: false
+
+AllCops:
+  Include:
+    - '**/Rakefile'
+  Exclude:
+    - 'bin/**/*'
+  TargetRubyVersion:
+    2.2
+
+# These are all the cops that are disabled in the default configuration.
+
+Style/AutoResourceCleanup:
+  Description: 'Suggests the usage of an auto resource cleanup version of a method (if available).'
+  Enabled: true
+
+Style/CollectionMethods:
+  Description: 'Preferred collection methods.'
+  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#map-find-select-reduce-size'
+  Enabled: true
+
+#Style/Encoding:
+#  Description: 'Use UTF-8 as the source file encoding.'
+#  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#utf-8'
+#  Enabled: true
+
+#Style/InlineComment:
+#  Description: 'Avoid inline comments.'
+#  Enabled: true
+
+Style/FirstArrayElementLineBreak:
+  Description: >-
+                 Checks for a line break before the first element in a
+                 multi-line array.
+  Enabled: true
+
+Style/FirstHashElementLineBreak:
+  Description: >-
+                 Checks for a line break before the first element in a
+                 multi-line hash.
+  Enabled: true
+
+Style/FirstMethodArgumentLineBreak:
+  Description: >-
+                 Checks for a line break before the first argument in a
+                 multi-line method call.
+  Enabled: true
+
+Style/FirstMethodParameterLineBreak:
+  Description: >-
+                 Checks for a line break before the first parameter in a
+                 multi-line method parameter definition.
+  Enabled: true
+
+Style/MethodCalledOnDoEndBlock:
+  Description: 'Avoid chaining a method call on a do...end block.'
+  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#single-line-blocks'
+  Enabled: true
+
+Style/MissingElse:
+  Description: >-
+                Require if/case expressions to have an else branches.
+                If enabled, it is recommended that
+                Style/UnlessElse and Style/EmptyElse be enabled.
+                This will conflict with Style/EmptyElse if
+                Style/EmptyElse is configured to style "both"
+  Enabled: false
+  EnforcedStyle: both
+  SupportedStyles:
+    # if - warn when an if expression is missing an else branch
+    # case - warn when a case expression is missing an else branch
+    # both - warn when an if or case expression is missing an else branch
+    - if
+    - case
+    - both
+
+#Style/MultilineAssignmentLayout:
+#  Description: 'Check for a newline after the assignment operator in multi-line assignments.'
+#  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#indent-conditional-assignment'
+#  Enabled: true
+
+Style/MutableConstant:
+  Description: 'Do not assign mutable objects to constants.'
+  Enabled: true
+
+Style/OptionHash:
+  Description: "Don't use option hashes when you can use keyword arguments."
+  Enabled: true
+
+Style/Send:
+  Description: 'Prefer `Object#__send__` or `Object#public_send` to `send`, as `send` may overlap with existing methods.'
+  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#prefer-public-send'
+  Enabled: true
+
+Style/StringMethods:
+  Description: 'Checks if configured preferred methods are used over non-preferred.'
+  Enabled: true
+
+#Style/SymbolArray:
+#  Description: 'Use %i or %I for arrays of symbols.'
+#  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-i'
+#  Enabled: true
+
+Lint/LiteralInInterpolation:
+  Description: 'Avoid interpolating literals in strings'
+  AutoCorrect: true

data/.rubocop_todo.yml

@@ -0,0 +1,66 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2016-01-23 03:18:41 +0100 using RuboCop version 0.36.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 11
+Metrics/AbcSize:
+  Max: 137
+
+# Offense count: 2
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 214
+
+# Offense count: 9
+Metrics/CyclomaticComplexity:
+  Max: 41
+
+# Offense count: 8
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 57
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ModuleLength:
+  Max: 134
+
+# Offense count: 6
+Metrics/PerceivedComplexity:
+  Max: 43
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/CommentIndentation:
+  Exclude:
+    - 'lib/pragmatic_tokenizer/tokenizer.rb'
+
+# Offense count: 31
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 2
+Style/MultilineBlockChain:
+  Exclude:
+    - 'lib/pragmatic_tokenizer/post_processor.rb'
+
+# Offense count: 1
+# Configuration parameters: SuspiciousParamNames.
+# SuspiciousParamNames: options, opts, args, params, parameters
+Style/OptionHash:
+  Exclude:
+    - 'lib/pragmatic_tokenizer/tokenizer.rb'
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles, AllowInnerSlashes.
+# SupportedStyles: slashes, percent_r, mixed
+Style/RegexpLiteral:
+  Exclude:
+    - 'lib/pragmatic_tokenizer/post_processor.rb'
+    - 'lib/pragmatic_tokenizer/pre_processor.rb'
+    - 'lib/pragmatic_tokenizer/tokenizer.rb'

data/README.md

@@ -371,15 +371,8 @@ Contractions: No
 ## Resources
 
 * [The Art of Tokenization](https://www.ibm.com/developerworks/community/blogs/nlp/entry/tokenization?lang=en)
-
 * [Handbook Of Natural Language Processing Second Edition](https://karczmarczuk.users.greyc.fr/TEACH/TAL/Doc/Handbook%20Of%20Natural%20Language%20Processing,%20Second%20Edition%20Chapman%20&%20Hall%20Crc%20Machine%20Learning%20&%20Pattern%20Recognition%202010.pdf)
 
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ## Contributing
 
 1. Fork it ( https://github.com/diasks2/pragmatic_tokenizer/fork )

data/Rakefile

@@ -2,4 +2,4 @@ require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
-task :default => :spec
+task default: :spec

data/lib/pragmatic_tokenizer/ending_punctuation_separator.rb

@@ -15,7 +15,7 @@ module PragmaticTokenizer
         if split_punctuation[0].nil?
           cleaned_tokens << a
         else
-          cleaned_tokens << a.tr(split_punctuation[0],'')
+          cleaned_tokens << a.tr(split_punctuation[0], '')
           if split_punctuation[0].length.eql?(1)
             cleaned_tokens << split_punctuation[0]
           else
@@ -28,4 +28,4 @@ module PragmaticTokenizer
       cleaned_tokens
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/full_stop_separator.rb

@@ -18,18 +18,18 @@ module PragmaticTokenizer
       cleaned_tokens = []
       tokens.each_with_index do |_t, i|
         if tokens[i + 1] && tokens[i] =~ /\A(.+)\.\z/
-          w = $1
-          unless abbr[Unicode::downcase(w)] || w =~ /\A[a-z]\z/i ||
-            w =~ /[a-z](?:\.[a-z])+\z/i
-            cleaned_tokens <<  w
+          w = Regexp.last_match(1)
+          unless abbr[Unicode.downcase(w)] || w =~ /\A[a-z]\z/i ||
+                 w =~ /[a-z](?:\.[a-z])+\z/i
+            cleaned_tokens << w
             cleaned_tokens << '.'
             next
           end
         end
         cleaned_tokens << tokens[i]
       end
-      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbreviations.include?(Unicode::downcase(cleaned_tokens[-1]).chomp("."))
-        cleaned_tokens[-1] = $1
+      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/ && !abbreviations.include?(Unicode.downcase(cleaned_tokens[-1]).chomp("."))
+        cleaned_tokens[-1] = Regexp.last_match(1)
         cleaned_tokens.push '.'
       end
       cleaned_tokens

data/lib/pragmatic_tokenizer/languages/arabic.rb

@@ -7,4 +7,4 @@ module PragmaticTokenizer
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/bulgarian.rb

@@ -7,4 +7,4 @@ module PragmaticTokenizer
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/catalan.rb

@@ -7,4 +7,4 @@ module PragmaticTokenizer
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/common.rb

@@ -4,7 +4,7 @@ module PragmaticTokenizer
       PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»', '/', '›', '‹', '^', '”'].freeze
       PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚", "”" => "⚘", "‘" => "⚭" }.freeze
       SEMI_PUNCTUATION = ['。', '．', '.'].freeze
-      ROMAN_NUMERALS = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix', 'xx', 'xxi', 'xxii', 'xxiii', 'xxiv', 'xxv', 'xxvi', 'xxvii', 'xxviii', 'xxix', 'xxx', 'xxxi', 'xxxii', 'xxxiii', 'xxxiv', 'xxxv', 'xxxvi', 'xxxvii', 'xxxviii', 'xxxix', 'xl', 'xli', 'xlii', 'xliii', 'xliv', 'xlv', 'xlvi', 'xlvii', 'xlviii', 'xlix', 'l', 'li', 'lii', 'liii', 'liv', 'lv', 'lvi', 'lvii', 'lviii', 'lix', 'lx', 'lxi', 'lxii', 'lxiii', 'lxiv', 'lxv', 'lxvi', 'lxvii', 'lxviii', 'lxix', 'lxx', 'lxxi', 'lxxii', 'lxxiii', 'lxxiv', 'lxxv', 'lxxvi', 'lxxvii', 'lxxviii', 'lxxix', 'lxxx', 'lxxxi', 'lxxxii', 'lxxxiii', 'lxxxiv', 'lxxxv', 'lxxxvi', 'lxxxvii', 'lxxxviii', 'lxxxix', 'xc', 'xci', 'xcii', 'xciii', 'xciv', 'xcv', 'xcvi', 'xcvii', 'xcviii', 'xcix'].freeze
+      ROMAN_NUMERALS = %w(i ii iii iv v vi vii viii ix x xi xii xiii xiv xv xvi xvii xviii xix xx xxi xxii xxiii xxiv xxv xxvi xxvii xxviii xxix xxx xxxi xxxii xxxiii xxxiv xxxv xxxvi xxxvii xxxviii xxxix xl xli xlii xliii xliv xlv xlvi xlvii xlviii xlix l li lii liii liv lv lvi lvii lviii lix lx lxi lxii lxiii lxiv lxv lxvi lxvii lxviii lxix lxx lxxi lxxii lxxiii lxxiv lxxv lxxvi lxxvii lxxviii lxxix lxxx lxxxi lxxxii lxxxiii lxxxiv lxxxv lxxxvi lxxxvii lxxxviii lxxxix xc xci xcii xciii xciv xcv xcvi xcvii xcviii xcix).freeze
       SPECIAL_CHARACTERS = ['®', '©', '™'].freeze
       ABBREVIATIONS = [].freeze
       STOP_WORDS = [].freeze
@@ -17,11 +17,11 @@ module PragmaticTokenizer
       class SingleQuotes
         def handle_single_quotes(text)
           # Convert left quotes to special character except for 'Twas or 'twas
-          text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
-          text.gsub!(/(\W|^)‘(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' } || text
+          text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { Regexp.last_match(1) ? Regexp.last_match(1) + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/(\W|^)‘(?=.*\w)(?!twas)(?!Twas)/o) { Regexp.last_match(1) ? Regexp.last_match(1) + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' } || text
           text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
           # Separate right single quotes
-          text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { Regexp.last_match(1) + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
         end
       end
     end

data/lib/pragmatic_tokenizer/languages/czech.rb

@@ -7,4 +7,4 @@ module PragmaticTokenizer
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/danish.rb

@@ -3,8 +3,8 @@ module PragmaticTokenizer
     module Danish
       include Languages::Common
       ABBREVIATIONS = [].freeze
-      STOP_WORDS = ["De", "I", "af", "aldrig", "alle", "altid", "bagved", "de", "der", "du", "efter", "eller", "en", "endnu", "et", "fjernt", "for", "foran", "fra", "få", "gennem", "god", "han", "her", "hos", "hovfor", "hun", "hurtig", "hvad", "hvem", "hvonår", "hvor", "hvordan", "hvorhen", "imod", "ja", "jeg", "langsom", "lidt", "mange", "med", "meget", "mellem", "mere", "mindre", "måske", "nede", "nej", "nok", "nu", "når", "og", "oppe", "på", "rask", "sammen", "temmelig", "til", "uden", "udenfor", "under", "ved", "vi"].freeze
+      STOP_WORDS = %w(De I af aldrig alle altid bagved de der du efter eller en endnu et fjernt for foran fra få gennem god han her hos hovfor hun hurtig hvad hvem hvonår hvor hvordan hvorhen imod ja jeg langsom lidt mange med meget mellem mere mindre måske nede nej nok nu når og oppe på rask sammen temmelig til uden udenfor under ved vi).freeze
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/deutsch.rb

@@ -2,27 +2,98 @@ module PragmaticTokenizer
   module Languages
     module Deutsch
       include Languages::Common
-      ABBREVIATIONS = ['a', 'a.d', 'a.k.a', 'a.s.a.p', 'abg', 'alt', 'apr', 'art', 'aug', 'b',
-        'b.a', 'b.s', 'best', 'bgm', 'bldg', 'btw', 'buchst', 'bzgl', 'bzw', 'c',
-        'ca', 'co', 'd', 'd.d', 'd.h', 'd.r', 'dergl', 'dez', 'dgl', 'dr', 'dr ',
-        'dt', 'dzt', 'e', 'e.l', 'e.u', 'e.v', 'ehem', 'eig', 'etc', 'etc.p.p',
-        'eu', 'europ', 'ev', 'ev ', 'evtl', 'f', 'f.d', 'feat', 'feb', 'ff',
-        'fr', 'frz', 'ft', 'g', 'gg', 'ggf', 'ggü', 'griech', 'h', 'h.c', 'h.p',
-        'hon', 'hosp', 'hr', 'i', 'i.a', 'i.d', 'i.d.r', 'i.f', 'i.p', 'i.z',
-        'ii', 'iii', 'inkl', 'int', 'iv', 'ix', 'j', 'jan', 'jul', 'jun', 'k',
-        'k.a', 'k.i.z', 'k.o', 'k.u.k', 'kath ', 'l', 'l.a', 'lfd', 'lt', 'ltd',
-        'm', 'm.e', 'm.w', 'mag', 'max', 'me', 'med', 'mind', 'mio', 'mme', 'mr',
-        'mrd', 'mrs', 'ms', 'mwst', 'mär', 'n', 'nov', 'nr', 'o', 'o.k', 'o.ä',
-        'oct', 'okt', 'omg', 'oö', 'p', 'p.a', 'p.m', 'p.s', 'p.t', 'pol', 'pp',
-        'prof', 'präs', 'q', 'r', 'r.i.p', 'r.r', 'ranz', 'rd', 'rep', 'rt',
-        'russ', 's', 's.g', 'sen', 'sep', 'sog', 'st', 'std', 'str', 't', 'türk',
-        'u', 'u.a', 'u.a  ', 'u.a.m', 'u.a.v', 'u.k', 'u.s', 'u.s.w', 'u.u',
-        'u.v.a', 'u.v.m', 'u.ä', 'ungar', 'usf', 'usw', 'uvm', 'v', 'v.a', 'v.d',
-        'v.m', 'vgl', 'vi', 'vii', 'viii', 'vs', 'w', 'wg', 'wr', 'x', 'xi',
-        'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'y',
-        'z', 'z.b', 'z.t', 'z.z', 'z.zt', 'zb', 'zt', 'zw', 'zzt', 'ä', 'ö',
-        'öffentl', 'öst', 'österr', 'ü'].freeze
-      STOP_WORDS = ["a", "ab", "aber", "ach", "acht", "achte", "achten", "achter", "achtes", "ag", "alle", "allein", "allem", "allen", "aller", "allerdings", "alles", "allgemeinen", "als", "also", "am", "an", "andere", "anderen", "andern", "anders", "au", "auch", "auf", "aus", "ausser", "ausserdem", "außer", "außerdem", "b", "bald", "bei", "beide", "beiden", "beim", "beispiel", "bekannt", "bereits", "besonders", "besser", "besten", "bin", "bis", "bisher", "bist", "c", "d", "d.h", "da", "dabei", "dadurch", "dafür", "dagegen", "daher", "dahin", "dahinter", "damals", "damit", "danach", "daneben", "dank", "dann", "daran", "darauf", "daraus", "darf", "darfst", "darin", "darum", "darunter", "darüber", "das", "dasein", "daselbst", "dass", "dasselbe", "davon", "davor", "dazu", "dazwischen", "daß", "dein", "deine", "deinem", "deiner", "dem", "dementsprechend", "demgegenüber", "demgemäss", "demgemäß", "demselben", "demzufolge", "den", "denen", "denn", "denselben", "der", "deren", "derjenige", "derjenigen", "dermassen", "dermaßen", "derselbe", "derselben", "des", "deshalb", "desselben", "dessen", "deswegen", "dich", "die", "diejenige", "diejenigen", "dies", "diese", "dieselbe", "dieselben", "diesem", "diesen", "dieser", "dieses", "dir", "doch", "dort", "drei", "drin", "dritte", "dritten", "dritter", "drittes", "du", "durch", "durchaus", "durfte", "durften", "dürfen", "dürft", "e", "eben", "ebenso", "ehrlich", "ei", "ei,", "eigen", "eigene", "eigenen", "eigener", "eigenes", "ein", "einander", "eine", "einem", "einen", "einer", "eines", "einige", "einigen", "einiger", "einiges", "einmal", "eins", "elf", "en", "ende", "endlich", "entweder", "er", "erst", "erste", "ersten", "erster", "erstes", "es", "etwa", "etwas", "euch", "euer", "eure", "f", "früher", "fünf", "fünfte", "fünften", "fünfter", "fünftes", "für", "g", "gab", "ganz", "ganze", "ganzen", "ganzer", "ganzes", "gar", "gedurft", "gegen", "gegenüber", "gehabt", "gehen", "geht", "gekannt", "gekonnt", "gemacht", "gemocht", "gemusst", "genug", "gerade", "gern", "gesagt", "geschweige", "gewesen", "gewollt", "geworden", "gibt", "ging", "gleich", "gott", "gross", "grosse", "grossen", "grosser", "grosses", "groß", "große", "großen", "großer", "großes", "gut", "gute", "guter", "gutes", "h", "habe", "haben", "habt", "hast", "hat", "hatte", "hatten", "hattest", "hattet", "heisst", "her", "heute", "hier", "hin", "hinter", "hoch", "hätte", "hätten", "i", "ich", "ihm", "ihn", "ihnen", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "im", "immer", "in", "indem", "infolgedessen", "ins", "irgend", "ist", "j", "ja", "jahr", "jahre", "jahren", "je", "jede", "jedem", "jeden", "jeder", "jedermann", "jedermanns", "jedes", "jedoch", "jemand", "jemandem", "jemanden", "jene", "jenem", "jenen", "jener", "jenes", "jetzt", "k", "kam", "kann", "kannst", "kaum", "kein", "keine", "keinem", "keinen", "keiner", "kleine", "kleinen", "kleiner", "kleines", "km", "kommen", "kommt", "konnte", "konnten", "kurz", "können", "könnt", "könnte", "l", "lang", "lange", "leicht", "leide", "lieber", "los", "m", "machen", "macht", "machte", "mag", "magst", "mahn", "man", "manche", "manchem", "manchen", "mancher", "manches", "mann", "mehr", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mensch", "menschen", "mich", "mir", "mit", "mittel", "mochte", "mochten", "morgen", "muss", "musst", "musste", "mussten", "muß", "mußt", "möchte", "mögen", "möglich", "mögt", "müssen", "müsst", "müßt", "n", "na", "nach", "nachdem", "nahm", "natürlich", "neben", "nein", "neue", "neuen", "neun", "neunte", "neunten", "neunter", "neuntes", "nicht", "nichts", "nie", "niemand", "niemandem", "niemanden", "noch", "nun", "nur", "o", "ob", "oben", "oder", "offen", "oft", "ohne", "p", "q", "r", "recht", "rechte", "rechten", "rechter", "rechtes", "richtig", "rund", "s", "sa", "sache", "sagt", "sagte", "sah", "satt", "schlecht", "schon", "sechs", "sechste", "sechsten", "sechster", "sechstes", "sehr", "sei", "seid", "seien", "sein", "seine", "seinem", "seinen", "seiner", "seines", "seit", "seitdem", "selbst", "sich", "sie", "sieben", "siebente", "siebenten", "siebenter", "siebentes", "sind", "so", "solang", "solche", "solchem", "solchen", "solcher", "solches", "soll", "sollen", "sollst", "sollt", "sollte", "sollten", "sondern", "sonst", "soweit", "sowie", "später", "statt", "t", "tag", "tage", "tagen", "tat", "teil", "tel", "tritt", "trotzdem", "tun", "u", "uhr", "um", "und", "und?", "uns", "unser", "unsere", "unserer", "unter", "v", "vergangenen", "viel", "viele", "vielem", "vielen", "vielleicht", "vier", "vierte", "vierten", "vierter", "viertes", "vom", "von", "vor", "w", "wahr?", "wann", "war", "waren", "wart", "warum", "was", "wegen", "weil", "weit", "weiter", "weitere", "weiteren", "weiteres", "welche", "welchem", "welchen", "welcher", "welches", "wem", "wen", "wenig", "wenige", "weniger", "weniges", "wenigstens", "wenn", "wer", "werde", "werden", "werdet", "weshalb", "wessen", "wie", "wieder", "wieso", "will", "willst", "wir", "wird", "wirklich", "wirst", "wo", "woher", "wohin", "wohl", "wollen", "wollt", "wollte", "wollten", "worden", "wurde", "wurden", "während", "währenddem", "währenddessen", "wäre", "würde", "würden", "x", "y", "z", "z.b", "zehn", "zehnte", "zehnten", "zehnter", "zehntes", "zeit", "zu", "zuerst", "zugleich", "zum", "zunächst", "zur", "zurück", "zusammen", "zwanzig", "zwar", "zwei", "zweite", "zweiten", "zweiter", "zweites", "zwischen", "zwölf", "über", "überhaupt", "übrigens"].freeze
+      ABBREVIATIONS = [
+          'a', 'a.d', 'a.k.a', 'a.s.a.p', 'abg', 'alt', 'apr', 'art', 'aug', 'b',
+          'b.a', 'b.s', 'best', 'bgm', 'bldg', 'btw', 'buchst', 'bzgl', 'bzw', 'c',
+          'ca', 'co', 'd', 'd.d', 'd.h', 'd.r', 'dergl', 'dez', 'dgl', 'dr', 'dr ',
+          'dt', 'dzt', 'e', 'e.l', 'e.u', 'e.v', 'ehem', 'eig', 'etc', 'etc.p.p',
+          'eu', 'europ', 'ev', 'ev ', 'evtl', 'f', 'f.d', 'feat', 'feb', 'ff',
+          'fr', 'frz', 'ft', 'g', 'gg', 'ggf', 'ggü', 'griech', 'h', 'h.c', 'h.p',
+          'hon', 'hosp', 'hr', 'i', 'i.a', 'i.d', 'i.d.r', 'i.f', 'i.p', 'i.z',
+          'ii', 'iii', 'inkl', 'int', 'iv', 'ix', 'j', 'jan', 'jul', 'jun', 'k',
+          'k.a', 'k.i.z', 'k.o', 'k.u.k', 'kath ', 'l', 'l.a', 'lfd', 'lt', 'ltd',
+          'm', 'm.e', 'm.w', 'mag', 'max', 'me', 'med', 'mind', 'mio', 'mme', 'mr',
+          'mrd', 'mrs', 'ms', 'mwst', 'mär', 'n', 'nov', 'nr', 'o', 'o.k', 'o.ä',
+          'oct', 'okt', 'omg', 'oö', 'p', 'p.a', 'p.m', 'p.s', 'p.t', 'pol', 'pp',
+          'prof', 'präs', 'q', 'r', 'r.i.p', 'r.r', 'ranz', 'rd', 'rep', 'rt',
+          'russ', 's', 's.g', 'sen', 'sep', 'sog', 'st', 'std', 'str', 't', 'türk',
+          'u', 'u.a', 'u.a  ', 'u.a.m', 'u.a.v', 'u.k', 'u.s', 'u.s.w', 'u.u',
+          'u.v.a', 'u.v.m', 'u.ä', 'ungar', 'usf', 'usw', 'uvm', 'v', 'v.a', 'v.d',
+          'v.m', 'vgl', 'vi', 'vii', 'viii', 'vs', 'w', 'wg', 'wr', 'x', 'xi',
+          'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'y',
+          'z', 'z.b', 'z.t', 'z.z', 'z.zt', 'zb', 'zt', 'zw', 'zzt', 'ä', 'ö',
+          'öffentl', 'öst', 'österr', 'ü'].freeze
+      STOP_WORDS = [
+          "a", "ab", "aber", "ach", "acht", "achte", "achten", "achter", "achtes",
+          "ag", "alle", "allein", "allem", "allen", "aller", "allerdings", "alles",
+          "allgemeinen", "als", "also", "am", "an", "andere", "anderen", "andern",
+          "anders", "au", "auch", "auf", "aus", "ausser", "ausserdem", "außer",
+          "außerdem", "b", "bald", "bei", "beide", "beiden", "beim", "beispiel",
+          "bekannt", "bereits", "besonders", "besser", "besten", "bin", "bis",
+          "bisher", "bist", "c", "d", "d.h", "da", "dabei", "dadurch", "dafür",
+          "dagegen", "daher", "dahin", "dahinter", "damals", "damit", "danach",
+          "daneben", "dank", "dann", "daran", "darauf", "daraus", "darf", "darfst",
+          "darin", "darum", "darunter", "darüber", "das", "dasein", "daselbst", "dass",
+          "dasselbe", "davon", "davor", "dazu", "dazwischen", "daß", "dein", "deine",
+          "deinem", "deiner", "dem", "dementsprechend", "demgegenüber", "demgemäss",
+          "demgemäß", "demselben", "demzufolge", "den", "denen", "denn", "denselben",
+          "der", "deren", "derjenige", "derjenigen", "dermassen", "dermaßen", "derselbe",
+          "derselben", "des", "deshalb", "desselben", "dessen", "deswegen", "dich", "die",
+          "diejenige", "diejenigen", "dies", "diese", "dieselbe", "dieselben", "diesem",
+          "diesen", "dieser", "dieses", "dir", "doch", "dort", "drei", "drin", "dritte",
+          "dritten", "dritter", "drittes", "du", "durch", "durchaus", "durfte", "durften",
+          "dürfen", "dürft", "e", "eben", "ebenso", "ehrlich", "ei", "ei,", "eigen",
+          "eigene", "eigenen", "eigener", "eigenes", "ein", "einander", "eine", "einem",
+          "einen", "einer", "eines", "einige", "einigen", "einiger", "einiges", "einmal",
+          "eins", "elf", "en", "ende", "endlich", "entweder", "er", "erst", "erste",
+          "ersten", "erster", "erstes", "es", "etwa", "etwas", "euch", "euer", "eure", "f",
+          "früher", "fünf", "fünfte", "fünften", "fünfter", "fünftes", "für", "g", "gab",
+          "ganz", "ganze", "ganzen", "ganzer", "ganzes", "gar", "gedurft", "gegen",
+          "gegenüber", "gehabt", "gehen", "geht", "gekannt", "gekonnt", "gemacht",
+          "gemocht", "gemusst", "genug", "gerade", "gern", "gesagt", "geschweige",
+          "gewesen", "gewollt", "geworden", "gibt", "ging", "gleich", "gott", "gross",
+          "grosse", "grossen", "grosser", "grosses", "groß", "große", "großen", "großer",
+          "großes", "gut", "gute", "guter", "gutes", "h", "habe", "haben", "habt", "hast",
+          "hat", "hatte", "hatten", "hattest", "hattet", "heisst", "her", "heute", "hier",
+          "hin", "hinter", "hoch", "hätte", "hätten", "i", "ich", "ihm", "ihn", "ihnen",
+          "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "im", "immer", "in", "indem",
+          "infolgedessen", "ins", "irgend", "ist", "j", "ja", "jahr", "jahre", "jahren",
+          "je", "jede", "jedem", "jeden", "jeder", "jedermann", "jedermanns", "jedes",
+          "jedoch", "jemand", "jemandem", "jemanden", "jene", "jenem", "jenen", "jener",
+          "jenes", "jetzt", "k", "kam", "kann", "kannst", "kaum", "kein", "keine", "keinem",
+          "keinen", "keiner", "kleine", "kleinen", "kleiner", "kleines", "km", "kommen",
+          "kommt", "konnte", "konnten", "kurz", "können", "könnt", "könnte", "l", "lang",
+          "lange", "leicht", "leide", "lieber", "los", "m", "machen", "macht", "machte",
+          "mag", "magst", "mahn", "man", "manche", "manchem", "manchen", "mancher", "manches",
+          "mann", "mehr", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mensch",
+          "menschen", "mich", "mir", "mit", "mittel", "mochte", "mochten", "morgen", "muss",
+          "musst", "musste", "mussten", "muß", "mußt", "möchte", "mögen", "möglich", "mögt",
+          "müssen", "müsst", "müßt", "n", "na", "nach", "nachdem", "nahm", "natürlich",
+          "neben", "nein", "neue", "neuen", "neun", "neunte", "neunten", "neunter", "neuntes",
+          "nicht", "nichts", "nie", "niemand", "niemandem", "niemanden", "noch", "nun", "nur",
+          "o", "ob", "oben", "oder", "offen", "oft", "ohne", "p", "q", "r", "recht", "rechte",
+          "rechten", "rechter", "rechtes", "richtig", "rund", "s", "sa", "sache", "sagt",
+          "sagte", "sah", "satt", "schlecht", "schon", "sechs", "sechste", "sechsten",
+          "sechster", "sechstes", "sehr", "sei", "seid", "seien", "sein", "seine", "seinem",
+          "seinen", "seiner", "seines", "seit", "seitdem", "selbst", "sich", "sie", "sieben",
+          "siebente", "siebenten", "siebenter", "siebentes", "sind", "so", "solang", "solche",
+          "solchem", "solchen", "solcher", "solches", "soll", "sollen", "sollst", "sollt",
+          "sollte", "sollten", "sondern", "sonst", "soweit", "sowie", "später", "statt", "t",
+          "tag", "tage", "tagen", "tat", "teil", "tel", "tritt", "trotzdem", "tun", "u",
+          "uhr", "um", "und", "und?", "uns", "unser", "unsere", "unserer", "unter", "v",
+          "vergangenen", "viel", "viele", "vielem", "vielen", "vielleicht", "vier", "vierte",
+          "vierten", "vierter", "viertes", "vom", "von", "vor", "w", "wahr?", "wann", "war",
+          "waren", "wart", "warum", "was", "wegen", "weil", "weit", "weiter", "weitere",
+          "weiteren", "weiteres", "welche", "welchem", "welchen", "welcher", "welches", "wem",
+          "wen", "wenig", "wenige", "weniger", "weniges", "wenigstens", "wenn", "wer", "werde",
+          "werden", "werdet", "weshalb", "wessen", "wie", "wieder", "wieso", "will", "willst",
+          "wir", "wird", "wirklich", "wirst", "wo", "woher", "wohin", "wohl", "wollen", "wollt",
+          "wollte", "wollten", "worden", "wurde", "wurden", "während", "währenddem",
+          "währenddessen", "wäre", "würde", "würden", "x", "y", "z", "z.b", "zehn", "zehnte",
+          "zehnten", "zehnter", "zehntes", "zeit", "zu", "zuerst", "zugleich", "zum",
+          "zunächst", "zur", "zurück", "zusammen", "zwanzig", "zwar", "zwei", "zweite",
+          "zweiten", "zweiter", "zweites", "zwischen", "zwölf", "über", "überhaupt",
+          "übrigens"].freeze
       CONTRACTIONS = {
           "auf's"             => "auf das",
           "can't"             => "cannot",
@@ -63,7 +134,7 @@ module PragmaticTokenizer
           "wird's"            => "wird es",
           "wär's"             => "wäre es",
           "ö's"               => "österreichs"
-      }
+      }.freeze
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/languages/dutch.rb

@@ -3,8 +3,8 @@ module PragmaticTokenizer
     module Dutch
       include Languages::Common
       ABBREVIATIONS = [].freeze
-      STOP_WORDS = ["aan", "af", "al", "als", "bij", "dan", "dat", "die", "dit", "een", "en", "er", "had", "heb", "hem", "het", "hij", "hoe", "hun", "ik", "in", "is", "je", "kan", "me", "men", "met", "mij", "nog", "nu", "of", "ons", "ook", "te", "tot", "uit", "van", "was", "wat", "we", "wel", "wij", "zal", "ze", "zei", "zij", "zo", "zou", "aan", "aangaande", "aangezien", "achter", "achterna", "afgelopen", "aldaar", "aldus", "alhoewel", "alias", "alle", "allebei", "alleen", "alsnog", "altijd", "altoos", "ander", "andere", "anders", "anderszins", "behalve", "behoudens", "beide", "beiden", "ben", "beneden", "bent", "bepaald", "betreffende", "binnen", "binnenin", "boven", "bovenal", "bovendien", "bovengenoemd", "bovenstaand", "bovenvermeld", "buiten", "daar", "daarheen", "daarin", "daarna", "daarnet", "daarom", "daarop", "daarvanlangs", "de", "dikwijls", "door", "doorgaand", "dus", "echter", "eer", "eerdat", "eerder", "eerlang", "eerst", "elk", "elke", "enig", "enigszins", "enkel", "erdoor", "even", "eveneens", "evenwel", "gauw", "gedurende", "geen", "gehad", "gekund", "geleden", "gelijk", "gemoeten", "gemogen", "geweest", "gewoon", "gewoonweg", "haar", "hadden", "hare", "hebben", "hebt", "heeft", "hen", "hierbeneden", "hierboven", "hoewel", "hunne", "ikzelf", "inmiddels", "inzake", "jezelf", "jij", "jijzelf", "jou", "jouw", "jouwe", "juist", "jullie", "klaar", "kon", "konden", "krachtens", "kunnen", "kunt", "later", "liever", "maar", "mag", "meer", "mezelf", "mijn", "mijnent", "mijner", "mijzelf", "misschien", "mocht", "mochten", "moest", "moesten", "moet", "moeten", "mogen", "na", "naar", "nadat", "net", "niet", "noch", "nogal", "ofschoon", "om", "omdat", "omhoog", "omlaag", "omstreeks", "omtrent", "omver", "onder", "ondertussen", "ongeveer", "onszelf", "onze", "op", "opnieuw", "opzij", "over", "overeind", "overigens", "pas", "precies", "reeds", "rond", "rondom", "sedert", "sinds", "sindsdien", "slechts", "sommige", "spoedig", "steeds", "tamelijk", "tenzij", "terwijl", "thans", "tijdens", "toch", "toen", "toenmaals", "toenmalig", "totdat", "tussen", "uitgezonderd", "vaakwat", "vandaan", "vanuit", "vanwege", "veeleer", "verder", "vervolgens", "vol", "volgens", "voor", "vooraf", "vooral", "vooralsnog", "voorbij", "voordat", "voordezen", "voordien", "voorheen", "voorop", "vooruit", "vrij", "vroeg", "waar", "waarom", "wanneer", "want", "waren", "weer", "weg", "wegens", "weldra", "welk", "welke", "wie", "wiens", "wier", "wijzelf", "zelfs", "zichzelf", "zijn", "zijne", "zodra", "zonder", "zouden", "zowat", "zulke", "zullen", "zult", "worden", "wordt", "deze"].freeze
+      STOP_WORDS = %w(aan af al als bij dan dat die dit een en er had heb hem het hij hoe hun ik in is je kan me men met mij nog nu of ons ook te tot uit van was wat we wel wij zal ze zei zij zo zou aan aangaande aangezien achter achterna afgelopen aldaar aldus alhoewel alias alle allebei alleen alsnog altijd altoos ander andere anders anderszins behalve behoudens beide beiden ben beneden bent bepaald betreffende binnen binnenin boven bovenal bovendien bovengenoemd bovenstaand bovenvermeld buiten daar daarheen daarin daarna daarnet daarom daarop daarvanlangs de dikwijls door doorgaand dus echter eer eerdat eerder eerlang eerst elk elke enig enigszins enkel erdoor even eveneens evenwel gauw gedurende geen gehad gekund geleden gelijk gemoeten gemogen geweest gewoon gewoonweg haar hadden hare hebben hebt heeft hen hierbeneden hierboven hoewel hunne ikzelf inmiddels inzake jezelf jij jijzelf jou jouw jouwe juist jullie klaar kon konden krachtens kunnen kunt later liever maar mag meer mezelf mijn mijnent mijner mijzelf misschien mocht mochten moest moesten moet moeten mogen na naar nadat net niet noch nogal ofschoon om omdat omhoog omlaag omstreeks omtrent omver onder ondertussen ongeveer onszelf onze op opnieuw opzij over overeind overigens pas precies reeds rond rondom sedert sinds sindsdien slechts sommige spoedig steeds tamelijk tenzij terwijl thans tijdens toch toen toenmaals toenmalig totdat tussen uitgezonderd vaakwat vandaan vanuit vanwege veeleer verder vervolgens vol volgens voor vooraf vooral vooralsnog voorbij voordat voordezen voordien voorheen voorop vooruit vrij vroeg waar waarom wanneer want waren weer weg wegens weldra welk welke wie wiens wier wijzelf zelfs zichzelf zijn zijne zodra zonder zouden zowat zulke zullen zult worden wordt deze).freeze
       CONTRACTIONS = {}.freeze
     end
   end
-end
+end