kuromoji-ruby 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kuromoji/core.rb +4 -2
- data/lib/kuromoji/version.rb +1 -1
- data/test/kuromoji_test.rb +16 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79695dc07004a11e3cbb3bdf811632947cf60013
|
4
|
+
data.tar.gz: 515f693f83989af2573be3976757119b6e098747
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4cea18296084e75632c32e9fc8d9e3a6a6a94aea35157d1b22607138cc46ceebe7cac2944bf26d78c844983265bd936f58426db15c7bf8e1e048ab72724eeebc
|
7
|
+
data.tar.gz: b4ff9abf67c2c15b2cf97af1c3b4ee83e43e2d72085af1e0a11e4a26bd7605caf7f705ef34a891120561da5f23ec32f563483fbc8ad271f12a093e239f33568f
|
data/lib/kuromoji/core.rb
CHANGED
@@ -37,9 +37,10 @@ module Kuromoji
|
|
37
37
|
end
|
38
38
|
|
39
39
|
def tokenize_with_hash(sentence)
|
40
|
+
result = []
|
41
|
+
return result if sentence.nil?
|
40
42
|
list = @tokenizer.tokenize(sentence)
|
41
43
|
iterator = list.iterator
|
42
|
-
result = []
|
43
44
|
while iterator.has_next
|
44
45
|
item = iterator.next
|
45
46
|
result << item.to_kuromoji_hash
|
@@ -48,9 +49,10 @@ module Kuromoji
|
|
48
49
|
end
|
49
50
|
|
50
51
|
def process(method, sentence)
|
52
|
+
tokenized = {}
|
53
|
+
return tokenized if sentence.nil?
|
51
54
|
list = @tokenizer.tokenize(sentence)
|
52
55
|
iterator = list.iterator
|
53
|
-
tokenized = {}
|
54
56
|
while iterator.has_next
|
55
57
|
item = iterator.next
|
56
58
|
tokenized[item.surface_form] = item.send(method)
|
data/lib/kuromoji/version.rb
CHANGED
data/test/kuromoji_test.rb
CHANGED
@@ -34,4 +34,20 @@ class KuromojiTest < Minitest::Test
|
|
34
34
|
assert_equal token['surface_form'], check[i]
|
35
35
|
end
|
36
36
|
end
|
37
|
+
|
38
|
+
def test_nil
|
39
|
+
tokenized = @no_dictionary.tokenize_with_hash(nil)
|
40
|
+
assert_equal tokenized, []
|
41
|
+
|
42
|
+
tokenized = @no_dictionary.reading(nil)
|
43
|
+
assert_equal tokenized, {}
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_empty
|
47
|
+
tokenized = @use_dictionary.tokenize_with_hash('')
|
48
|
+
assert_equal tokenized, []
|
49
|
+
|
50
|
+
tokenized = @no_dictionary.reading('')
|
51
|
+
assert_equal tokenized, {}
|
52
|
+
end
|
37
53
|
end
|