pragmatic_tokenizer 0.1.10 → 0.1.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pragmatic_tokenizer/tokenizer.rb +1 -0
- data/lib/pragmatic_tokenizer/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bd6598d349d883708c73d3f51431cf4a516bafdb
|
|
4
|
+
data.tar.gz: ba8ab51aa218c4d5405ec1db2a539e0fffa478ec
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5e3aa17314b8130edf651229048c502e2834517f3f2d793705232248554511937d0157a9ff2dee57191aadeb66ad7dc5303634e9857f6c9fcf51dd9da43072d7
|
|
7
|
+
data.tar.gz: 73af6e7bd5807dc7637d662abebb3a27dbf0e2194a51e30e1876d7c931dbaaeab6bc6ee0ddb3cf915a75385fc6613b2a6480d8686f0e2949b7461334453dff87
|
|
@@ -102,6 +102,7 @@ module PragmaticTokenizer
|
|
|
102
102
|
def find_contractions(tokens)
|
|
103
103
|
return tokens unless expand_contractions && language_module::CONTRACTIONS
|
|
104
104
|
tokens.flat_map { |t| language_module::CONTRACTIONS.has_key?(t) ? language_module::CONTRACTIONS[t].split(' ').flatten : t }
|
|
105
|
+
.flat_map { |t| t.include?("/") ? t.gsub!(/\//, '\1 \2').split(' ').flatten : t }
|
|
105
106
|
end
|
|
106
107
|
end
|
|
107
108
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pragmatic_tokenizer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kevin S. Dias
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-01-
|
|
11
|
+
date: 2016-01-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|