pragmatic_tokenizer 1.2.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pragmatic_tokenizer/pre_processor.rb +5 -0
- data/lib/pragmatic_tokenizer/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 74b115265cde89758fa37abfbbe6e5fbf581819e
|
|
4
|
+
data.tar.gz: 7074d143fd81022852dca5af10800e1abe93f94e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ef9e49e20912c5e3c5324362dfa834bf7ad1d2a9b6ac83f82366b3863724d9a3be42114d672757e19594792b849b1f50bd2f74f1758e40bac0b88e01b1633b85
|
|
7
|
+
data.tar.gz: 79a3cfbd22546a8c7ac90501aa749311f927a8ca4d039d12dc64ec5d8bf5aa3db996ff6c66cbc4b8be8aab512f5d4f7c7b432eb2d2da695d6c586dc373578223
|
|
@@ -20,6 +20,7 @@ module PragmaticTokenizer
|
|
|
20
20
|
shift_semicolon(text)
|
|
21
21
|
shift_caret(text)
|
|
22
22
|
shift_hashtag(text)
|
|
23
|
+
shift_ampersand(text)
|
|
23
24
|
shift_vertical_bar(text)
|
|
24
25
|
convert_dbl_quotes(text)
|
|
25
26
|
convert_sgl_quotes(text)
|
|
@@ -100,6 +101,10 @@ module PragmaticTokenizer
|
|
|
100
101
|
text.gsub!(/(?<=\S)(#|＃)(?=\S)/, ' \1\2') || text
|
|
101
102
|
end
|
|
102
103
|
|
|
104
|
+
def shift_ampersand(text)
|
|
105
|
+
text.gsub!(/\&/, ' & ') || text
|
|
106
|
+
end
|
|
107
|
+
|
|
103
108
|
def shift_vertical_bar(text)
|
|
104
109
|
text.gsub!(/\|/, ' | ') || text
|
|
105
110
|
end
|