pragmatic_tokenizer 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a52658ccd583baac9e69b37649e5403ad6f3549b
-  data.tar.gz: 95335fab907589faa066f7a76bd8a6df4d9f2f70
+  metadata.gz: b8f3d1c236d1faafc68ff8d689de291a59596b1f
+  data.tar.gz: ed010aae17417aa9ea87a1fa817f893c893e5d77
 SHA512:
-  metadata.gz: f7b42942e2676cc22a807938244fc38b7e4d4097bcdb3a6f0c08fe94fccc923cce50e8328e91efdfdd93f758771dda08b3ea432bb67099536630d4a9d9d9f4e9
-  data.tar.gz: d78a40443dc9948866d1f271e47af355a8838e38dbd932d282d7b0c905ad06dfb392f65ba7cc2df6a891098ff52b9a795317b1a458363214930c08650cc6ef96
+  metadata.gz: e10fbb56ec2097ef2854a0517f2b4add98479fecb0318c813c791f0b3872b7ba93a489e409de14e83b5327a8e12ec64747ef64846f57c32e0625d994c50f9331
+  data.tar.gz: e6a8f35c84c6e1029d26dd7981c2f8bd3f43803d095ecd45de0d2ab2132399f1dc652704269217573dbcb4b99aa438b751be57ffc26fc39b8e9b3b4554a2d56e

data/README.md

@@ -70,7 +70,16 @@ Or install it yourself as:
 ##### `remove_numbers`
   **default** = `'false'`
 - `true`  
-  Removes any token that contains a number or Roman numeral.
+  Removes any token that contains a number.
+- `false`   
+  Leaves tokens as is.
+
+<hr>
+
+##### `remove_roman_numerals`
+  **default** = `'false'`
+- `true`  
+  Removes any token that contains a Roman numeral.
 - `false`   
   Leaves tokens as is.
 
@@ -124,145 +133,147 @@ PragmaticTokenizer::Tokenizer.new(text, minimum_length: 6).tokenize
 
 ## Language Support
 
-The following lists the current level of support for different languages. Pull requests or help for any languages that are not fully supported would be greatly appreciated. *N.B. - contractions might not be applicable for all languages below - in that case the CONTRACTIONS hash should stay empty.*  
+The following lists the current level of support for different languages. Pull requests or help for any languages that are not fully supported would be greatly appreciated.  
+
+*N.B. - contractions might not be applicable for all languages below - in that case the CONTRACTIONS hash should stay empty.*  
 
 ##### English
-Specs: Yes
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: Yes
+Specs: Yes  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: Yes  
 
 ##### Arabic
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Bulgarian
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Catalan
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Czech
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Danish
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
-##### German
-Specs: More needed
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+##### Deutsch
+Specs: More needed  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Finnish
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### French
-Specs: More needed
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: More needed  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Greek
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Indonesian
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Italian
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Latvian
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Norwegian
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Persian
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Polish
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Portuguese
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Romanian
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Russian
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Slovak
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Spanish
-Specs: No
-Abbreviations: Yes
-Stop Words: Yes
-Contractions: Yes
+Specs: No  
+Abbreviations: Yes  
+Stop Words: Yes  
+Contractions: Yes  
 
 ##### Swedish
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ##### Turkish
-Specs: No
-Abbreviations: No
-Stop Words: Yes
-Contractions: No
+Specs: No  
+Abbreviations: No  
+Stop Words: Yes  
+Contractions: No  
 
 ## Development
 

data/lib/pragmatic_tokenizer/languages/common.rb

@@ -9,7 +9,15 @@ module PragmaticTokenizer
       ABBREVIATIONS = []
       STOP_WORDS = []
       CONTRACTIONS = {}
+
+      class SingleQuotes
+        def handle_single_quotes(text)
+          text.gsub!(/`(?!`)(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ') || text
+          text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
+          # Separate right single quotes
+          text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+        end
+      end
     end
   end
 end
-

data/lib/pragmatic_tokenizer/languages/english.rb

@@ -86,6 +86,16 @@ module PragmaticTokenizer
         "will-o'-the-wisp"  => "will-of-the-wisp",
         "'twas"             => "it was"
       }
+      class SingleQuotes
+        def handle_single_quotes(text)
+          text.gsub!(/`(?!`)(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ') || text
+          # Convert left quotes to special character except for 'Twas or 'twas
+          text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
+          # Separate right single quotes
+          text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+        end
+      end
     end
   end
 end

data/lib/pragmatic_tokenizer/languages/french.rb

@@ -5,6 +5,16 @@ module PragmaticTokenizer
       ABBREVIATIONS = ["a.c.n", "a.m", "al", "ann", "apr", "art", "auj", "av", "b.p", "boul", "c.-à-d", "c.n", "c.n.s", "c.p.i", "c.q.f.d", "c.s", "ca", "cf", "ch.-l", "chap", "co", "contr", "dir", "e.g", "e.v", "env", "etc", "ex", "fasc", "fig", "fr", "fém", "hab", "i.e", "ibid", "id", "inf", "l.d", "lib", "ll.aa", "ll.aa.ii", "ll.aa.rr", "ll.aa.ss", "ll.ee", "ll.mm", "ll.mm.ii.rr", "loc.cit", "ltd", "masc", "mm", "ms", "n.b", "n.d", "n.d.a", "n.d.l.r", "n.d.t", "n.p.a.i", "n.s", "n/réf", "nn.ss", "p.c.c", "p.ex", "p.j", "p.s", "pl", "pp", "r.-v", "r.a.s", "r.i.p", "r.p", "s.a", "s.a.i", "s.a.r", "s.a.s", "s.e", "s.m", "s.m.i.r", "s.s", "sec", "sect", "sing", "sq", "sqq", "ss", "suiv", "sup", "suppl", "t.s.v.p", "tél", "vb", "vol", "vs", "x.o", "z.i", "éd"]
       STOP_WORDS = ["a", "à", "â", "abord", "afin", "ah", "ai", "aie", "ainsi", "allaient", "allo", "allô", "allons", "après", "assez", "attendu", "au", "aucun", "aucune", "aujourd", "aujourd'hui", "auquel", "aura", "auront", "aussi", "autre", "autres", "aux", "auxquelles", "auxquels", "avaient", "avais", "avait", "avant", "avec", "avoir", "ayant", "b", "bah", "beaucoup", "bien", "bigre", "boum", "bravo", "brrr", "c", "ça", "car", "ce", "ceci", "cela", "celle", "celle-ci", "celle-là", "celles", "celles-ci", "celles-là", "celui", "celui-ci", "celui-là", "cent", "cependant", "certain", "certaine", "certaines", "certains", "certes", "ces", "cet", "cette", "ceux", "ceux-ci", "ceux-là", "chacun", "chaque", "cher", "chère", "chères", "chers", "chez", "chiche", "chut", "ci", "cinq", "cinquantaine", "cinquante", "cinquantième", "cinquième", "clac", "clic", "combien", "comme", "comment", "compris", "concernant", "contre", "couic", "crac", "d", "da", "dans", "de", "debout", "dedans", "dehors", "delà", "depuis", "derrière", "des", "dès", "désormais", "desquelles", "desquels", "dessous", "dessus", "deux", "deuxième", "deuxièmement", "devant", "devers", "devra", "différent", "différente", "différentes", "différents", "dire", "divers", "diverse", "diverses", "dix", "dix-huit", "dixième", "dix-neuf", "dix-sept", "doit", "doivent", "donc", "dont", "douze", "douzième", "dring", "du", "duquel", "durant", "e", "effet", "eh", "elle", "elle-même", "elles", "elles-mêmes", "en", "encore", "entre", "envers", "environ", "es", "ès", "est", "et", "etant", "étaient", "étais", "était", "étant", "etc", "été", "etre", "être", "eu", "euh", "eux", "eux-mêmes", "excepté", "f", "façon", "fais", "faisaient", "faisant", "fait", "feront", "fi", "flac", "floc", "font", "g", "gens", "h", "ha", "hé", "hein", "hélas", "hem", "hep", "hi", "ho", "holà", "hop", "hormis", "hors", "hou", "houp", "hue", "hui", "huit", "huitième", "hum", "hurrah", "i", "il", "ils", "importe", "j", "je", "jusqu", "jusque", "k", "l", "la", "là", "laquelle", "las", "le", "lequel", "les", "lès", "lesquelles", "lesquels", "leur", "leurs", "longtemps", "lorsque", "lui", "lui-même", "m", "ma", "maint", "mais", "malgré", "me", "même", "mêmes", "merci", "mes", "mien", "mienne", "miennes", "miens", "mille", "mince", "moi", "moi-même", "moins", "mon", "moyennant", "n", "na", "ne", "néanmoins", "neuf", "neuvième", "ni", "nombreuses", "nombreux", "non", "nos", "notre", "nôtre", "nôtres", "nous", "nous-mêmes", "nul", "o", "o|", "ô", "oh", "ohé", "olé", "ollé", "on", "ont", "onze", "onzième", "ore", "ou", "où", "ouf", "ouias", "oust", "ouste", "outre", "p", "paf", "pan", "par", "parmi", "partant", "particulier", "particulière", "particulièrement", "pas", "passé", "pendant", "personne", "peu", "peut", "peuvent", "peux", "pff", "pfft", "pfut", "pif", "plein", "plouf", "plus", "plusieurs", "plutôt", "pouah", "pour", "pourquoi", "premier", "première", "premièrement", "près", "proche", "psitt", "puisque", "q", "qu", "quand", "quant", "quanta", "quant-à-soi", "quarante", "quatorze", "quatre", "quatre-vingt", "quatrième", "quatrièmement", "que", "quel", "quelconque", "quelle", "quelles", "quelque", "quelques", "quelqu'un", "quels", "qui", "quiconque", "quinze", "quoi", "quoique", "r", "revoici", "revoilà", "rien", "s", "sa", "sacrebleu", "sans", "sapristi", "sauf", "se", "seize", "selon", "sept", "septième", "sera", "seront", "ses", "si", "sien", "sienne", "siennes", "siens", "sinon", "six", "sixième", "soi", "soi-même", "soit", "soixante", "son", "sont", "sous", "stop", "suis", "suivant", "sur", "surtout", "t", "ta", "tac", "tant", "te", "té", "tel", "telle", "tellement", "telles", "tels", "tenant", "tes", "tic", "tien", "tienne", "tiennes", "tiens", "toc", "toi", "toi-même", "ton", "touchant", "toujours", "tous", "tout", "toute", "toutes", "treize", "trente", "très", "trois", "troisième", "troisièmement", "trop", "tsoin", "tsouin", "tu", "u", "un", "une", "unes", "uns", "v", "va", "vais", "vas", "vé", "vers", "via", "vif", "vifs", "vingt", "vivat", "vive", "vives", "vlan", "voici", "voilà", "vont", "vos", "votre", "vôtre", "vôtres", "vous", "vous-mêmes", "vu", "w", "x", "y", "z", "zut", "alors", "aucuns", "bon", "devrait", "dos", "droite", "début", "essai", "faites", "fois", "force", "haut", "ici", "juste", "maintenant", "mine", "mot", "nommés", "nouveaux", "parce", "parole", "personnes", "pièce", "plupart", "seulement", "soyez", "sujet", "tandis", "valeur", "voie", "voient", "état", "étions", "d'un", "d'une"]
       CONTRACTIONS = {}
+
+      class SingleQuotes
+        def handle_single_quotes(text)
+          text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/`(?!`)(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ') || text
+          text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
+          text.gsub!(/l\'/, '\1 l\' \2') || text
+          text.gsub!(/L\'/, '\1 L\' \2') || text
+        end
+      end
     end
   end
-end
+end

data/lib/pragmatic_tokenizer/processor.rb

@@ -24,18 +24,17 @@ module PragmaticTokenizer
 
     def convert_dbl_quotes(text)
       # Convert left double quotes to special character
-      text.gsub!(/"(?=.*\w)/o, ' ' + convert_punct_to_sym('"') + ' ') || text
+      text.gsub!(/"(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
       # Convert remaining quotes to special character
-      text.gsub!(/"/, ' ' + convert_punct_to_sym('"') + ' ') || text
+      text.gsub!(/"/, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['"'] + ' ') || text
     end
 
     def convert_sgl_quotes(text)
-      text.gsub!(/`(?!`)(?=.*\w)/o, ' ' + convert_punct_to_sym("'") + ' ') || text
-      # Convert left quotes to special character except for 'Twas or 'twas
-      text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + convert_punct_to_sym("'") + ' ' : ' ' + convert_punct_to_sym("'") + ' ' } || text
-      text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + convert_punct_to_sym("'")) || text
-      # Separate right single quotes
-      text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + convert_punct_to_sym("'") + ' ' } || text
+      if defined? @language::SingleQuotes
+        @language::SingleQuotes.new.handle_single_quotes(text)
+      else
+        PragmaticTokenizer::Languages::Common::SingleQuotes.new.handle_single_quotes(text)
+      end
     end
 
     def shift_multiple_dash(text)
@@ -71,7 +70,7 @@ module PragmaticTokenizer
         !(/\A\d+/ == text.partition(':').last[0]) &&
         !(/\A\d+/ == text.partition(':').first[-1])
       # Ignore web addresses
-      text.gsub!(/(?<=[http|https]):(?=\/\/)/, convert_punct_to_sym(":")) || text
+      text.gsub!(/(?<=[http|https]):(?=\/\/)/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[":"]) || text
       text.gsub!(/:/o, ' :') || text
     end
 
@@ -124,10 +123,6 @@ module PragmaticTokenizer
       cleaned_tokens
     end
 
-    def convert_punct_to_sym(punctuation)
-      PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[punctuation]
-    end
-
     def convert_sym_to_punct(token)
       symbol = /[♳ ♴ ♵ ♶ ♷ ♸ ♹ ♺ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☠ ☢ ☣ ☤ ☥ ☦ ☧ ☀ ☁ ☂ ☃ ☄ ☮ ♔ ♕ ♖ ♗ ♘ ♙ ♚]/.match(token)
       if symbol.nil?

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-06 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler