pragmatic_tokenizer 3.0.1 → 3.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pragmatic_tokenizer/tokenizer.rb +1 -1
- data/lib/pragmatic_tokenizer/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cfbd0558c7cf67a00af40c60acf15cbacd856c5a
|
|
4
|
+
data.tar.gz: 8e8823fea384e85ccbefb75ac6ab965fc9375288
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3a8ebd1f301310a2ba6ab053a1b9c94bf54061b08aa826b655313a67cdd6eb21e347683683ea7137a596d693917b6f9e77521aee51630cbe6f071387f6b7c72d
|
|
7
|
+
data.tar.gz: 7d8e5cff9a794669f3b5a2edcdee24c1ba5ff5f1bb5f4f09bfdc21b98ce4ae8277365318dc659c7e835ed73d3797437cd0a746fe1153b56f22cfa7b4ef6e9cb7
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# -*- encoding : utf-8 -*-
|
|
2
|
+
require 'set'
|
|
2
3
|
require 'pragmatic_tokenizer/languages'
|
|
3
4
|
require 'pragmatic_tokenizer/pre_processor'
|
|
4
5
|
require 'pragmatic_tokenizer/post_processor'
|
|
5
6
|
require 'pragmatic_tokenizer/full_stop_separator'
|
|
6
7
|
require 'unicode'
|
|
7
|
-
require 'set'
|
|
8
8
|
|
|
9
9
|
module PragmaticTokenizer
|
|
10
10
|
class Tokenizer
|