pragmatic_tokenizer 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pragmatic_tokenizer/processor.rb +0 -2
- data/lib/pragmatic_tokenizer/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 45b328926e640455c5a4f02fac9f513aff0a7714
|
|
4
|
+
data.tar.gz: fbcb73dd8c16bd37cef4919487afb294d92fc4aa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f47c0b9ce0d7732848e6836649fa0346c20e30126284a82cfdab6f08facfa138f40479720ca0df7da0fe2dcb44a73164adc3fd7390670721e2a35bcbfc918f9
|
|
7
|
+
data.tar.gz: 28ddafc471573590d93ec7cfb48a45c4ecc5f66e9b6b91653819887cefbf91e6f4551566787efba648e70e5d98164f9fdfdf2fa626de76fc1fb61f6e09d999d2
|
|
@@ -89,11 +89,9 @@ module PragmaticTokenizer
|
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
def shift_colon(text)
|
|
92
|
-
puts "Text: #{text}"
|
|
93
92
|
return text unless text.include?(':') &&
|
|
94
93
|
text.partition(':').last[0] !~ /\A\d+/ &&
|
|
95
94
|
text.partition(':').first[-1] !~ /\A\d+/
|
|
96
|
-
puts "YOYOYO"
|
|
97
95
|
# Ignore web addresses
|
|
98
96
|
text.gsub!(/(?<=[http|https]):(?=\/\/)/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[":"]) || text
|
|
99
97
|
text.gsub!(/:/o, ' :') || text
|