pragmatic_tokenizer 1.3.0 → 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pragmatic_tokenizer/pre_processor.rb +5 -0
- data/lib/pragmatic_tokenizer/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 600855c6a883dea597e6abd87b43721579680fe2
|
|
4
|
+
data.tar.gz: f0dc608d45aac9701b6ded9b5de001cde3b987e9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 93ac2871c7e053060289f5fd758a090470c5dc7a98a9d68453c965701e425c63ef0825b83715e081e3c69c2d31617efa1738fdca36411537e622b6409d975416
|
|
7
|
+
data.tar.gz: 2ec0fbdbdfdb29f91324853aea7ccd992ec4e3ca04274d97aead2eccc84b819b83e351e5554caac36211040508dfa01f0f8877c73a87a33c79bf5567f580f3f8
|
|
@@ -18,6 +18,7 @@ module PragmaticTokenizer
|
|
|
18
18
|
shift_colon(text)
|
|
19
19
|
shift_bracket(text)
|
|
20
20
|
shift_semicolon(text)
|
|
21
|
+
shift_percent(text)
|
|
21
22
|
shift_caret(text)
|
|
22
23
|
shift_hashtag(text)
|
|
23
24
|
shift_ampersand(text)
|
|
@@ -93,6 +94,10 @@ module PragmaticTokenizer
|
|
|
93
94
|
text.gsub!(/([;])/o) { ' ' + $1 + ' ' } || text
|
|
94
95
|
end
|
|
95
96
|
|
|
97
|
+
def shift_percent(text)
|
|
98
|
+
text.gsub!(/(?<=\D)%(?=\d+)/, ' %') || text
|
|
99
|
+
end
|
|
100
|
+
|
|
96
101
|
def shift_caret(text)
|
|
97
102
|
text.gsub!(/\^/, ' ^ ') || text
|
|
98
103
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pragmatic_tokenizer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.3.
|
|
4
|
+
version: 1.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kevin S. Dias
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-01-
|
|
11
|
+
date: 2016-01-22 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: unicode
|