rseg 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/engines/dict.rb +39 -37
- data/lib/engines/engine.rb +14 -12
- data/lib/engines/english.rb +21 -19
- data/lib/engines/name.rb +44 -42
- data/lib/engines/number.rb +52 -50
- data/lib/filters/conjunction.rb +6 -4
- data/lib/filters/fullwidth.rb +15 -13
- data/lib/filters/symbol.rb +12 -10
- data/lib/rseg.rb +6 -0
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.6
|
data/lib/engines/dict.rb
CHANGED
@@ -1,49 +1,51 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
module RsegEngine
|
2
|
+
class Dict < Engine
|
3
|
+
@@root = nil
|
4
|
+
@@dict_path = File.join(File.dirname(__FILE__), '../../dict/dict.hash')
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
class << self
|
7
|
+
def dict_path=(path)
|
8
|
+
@@dict_path = path
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
11
|
+
def dict_path
|
12
|
+
@@dict_path
|
13
|
+
end
|
12
14
|
end
|
13
|
-
end
|
14
15
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
def initialize
|
17
|
+
@@root ||= load_dict(@@dict_path)
|
18
|
+
@word = ''
|
19
|
+
@node = @@root
|
20
|
+
super
|
21
|
+
end
|
21
22
|
|
22
|
-
|
23
|
-
|
24
|
-
|
23
|
+
def process(char)
|
24
|
+
match = false
|
25
|
+
word = nil
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
else
|
31
|
-
if @node[:end] || @word.chars.to_a.length == 1
|
32
|
-
word = @word
|
27
|
+
if @node[char]
|
28
|
+
@word << char
|
29
|
+
@node = @node[char]
|
30
|
+
match = true
|
33
31
|
else
|
34
|
-
|
35
|
-
|
32
|
+
if @node[:end] || @word.chars.to_a.length == 1
|
33
|
+
word = @word
|
34
|
+
else
|
35
|
+
word = @word.chars.to_a
|
36
|
+
end
|
36
37
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
@node = @@root
|
39
|
+
@word = ''
|
40
|
+
match = false
|
41
|
+
end
|
41
42
|
|
42
|
-
|
43
|
-
|
43
|
+
[match, word]
|
44
|
+
end
|
44
45
|
|
45
|
-
|
46
|
-
|
47
|
-
|
46
|
+
private
|
47
|
+
def load_dict(path)
|
48
|
+
File.open(path, "rb") {|io| Marshal.load(io)}
|
49
|
+
end
|
48
50
|
end
|
49
51
|
end
|
data/lib/engines/engine.rb
CHANGED
@@ -1,17 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
1
|
+
module RsegEngine
|
2
|
+
class Engine
|
3
|
+
def initialize
|
4
|
+
@running = true
|
5
|
+
end
|
5
6
|
|
6
|
-
|
7
|
-
|
8
|
-
|
7
|
+
def stop
|
8
|
+
@running = false
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
def run
|
12
|
+
@running = true
|
13
|
+
end
|
13
14
|
|
14
|
-
|
15
|
-
|
15
|
+
def running?
|
16
|
+
@running
|
17
|
+
end
|
16
18
|
end
|
17
19
|
end
|
data/lib/engines/english.rb
CHANGED
@@ -1,24 +1,26 @@
|
|
1
|
-
|
1
|
+
module RsegEngine
|
2
|
+
LETTER_SYMBOLS = ('a'..'z').to_a + ('A'..'Z').to_a
|
2
3
|
|
3
|
-
class English < Engine
|
4
|
-
|
5
|
-
@word = ''
|
6
|
-
super
|
7
|
-
end
|
8
|
-
|
9
|
-
def process(char)
|
10
|
-
match = false
|
11
|
-
word = nil
|
12
|
-
|
13
|
-
if LETTER_SYMBOLS.include?(char)
|
14
|
-
@word << char
|
15
|
-
match = true
|
16
|
-
else
|
17
|
-
word = @word
|
4
|
+
class English < Engine
|
5
|
+
def initialize
|
18
6
|
@word = ''
|
19
|
-
|
7
|
+
super
|
20
8
|
end
|
9
|
+
|
10
|
+
def process(char)
|
11
|
+
match = false
|
12
|
+
word = nil
|
21
13
|
|
22
|
-
|
23
|
-
|
14
|
+
if LETTER_SYMBOLS.include?(char)
|
15
|
+
@word << char
|
16
|
+
match = true
|
17
|
+
else
|
18
|
+
word = @word
|
19
|
+
@word = ''
|
20
|
+
match = false
|
21
|
+
end
|
22
|
+
|
23
|
+
[match, word]
|
24
|
+
end
|
25
|
+
end
|
24
26
|
end
|
data/lib/engines/name.rb
CHANGED
@@ -1,49 +1,51 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
1
|
+
module RsegEngine
|
2
|
+
class Name < Engine
|
3
|
+
@@last_names = %W(丁 卜 刁 七 弓 干 于 王 尤 孔 方 申 白 甘 田 包 石 左 平 司 皮 史 池 艾 年 匡 充 江 印
|
4
|
+
促 伊 伍 安 任 米 促 牟 向 吉 成 伏 吕 李 吴 沈 何 贝 狄 祁 杜 汪 阮 邢 汲 别 辛 冷 利 沃 谷
|
5
|
+
扶 步 那 沙 周 金 吕 花 孟 和 邵 房 抗 灰 明 屈 松 牧 宓 武 幸 卓 易 尚 邰 空 竺 岳 东 林
|
6
|
+
施 姜 俞 查 封 秋 帅 祖 羿 柯 茅 柳 姚 纪 宣 咸 库 侯 洪 胡 哈 宣 郁 祝 苗 禹 娄
|
7
|
+
秦 奚 倪 度 凌 宰 宦 师 徐 翁 班 马 时 晃 乌 夏 贡 柴 能 家 宫 敖 索 晏 桑 高 凌 桂 容 姬 劳 桑 桂 袁 时 祝 席 徐 高 夏 凌 洪 翁 家 芮 乌 祖 索 贡
|
8
|
+
许 张 曹 戚 梅 屠 盛 崖 章 鱼 国 商 扈 寇 终 冯 苗 康 常 茅 闵 麻 胡 崔 邢 条 符 宿 堵 浦 习 鱼
|
9
|
+
梁 富 曾 程 项 钮 舒 彭 费 童 云 喻 嵇 范 费 贺 毕 付 黄 邵 祁 阮 强 童 邱 解 贲 单 富 钮 荀 惠 邴 焦 班 甯 钭 景 邰 劳 茹 寇 荆
|
10
|
+
莫 际 景 须 杨 詹 郎 雷 贾 路 骆 虞 经 裘 郁 滑 甄 靳 詹 闻 逄 雍 訾 郎 农 路 骆 虞 经 裘 郁 滑 靳 闻 逄 雍
|
11
|
+
赵 黄 褚 凤 郝 齐 臧 熊 管 裴 荣 郗 韶 郜 黎 翟 寿 通
|
12
|
+
卫 葛 鲁 乐 谈 董 樊 万 诸 刘 叶 都 满 广 殴 巩 养
|
13
|
+
郭 钱 陈 陶 鲍 穆 郭 堆 卢 陆 龙 噪 鄂 阴 苍 燕 冀 衡 融 蒯 逯
|
14
|
+
蒋 魏 谢 邹 潘 滕 邬 戴 钟 蔡 缪 应 储 糜 隗 历 蒲 慕 蔚 隆 鞠 关
|
15
|
+
韩 萧 颜 庞 麦 双 璩 濮 聂 丰 看
|
16
|
+
郑 严 蓟 薄 谭 罗 买 蓝 蓬 怀 党 饶 顾 苏 龚 边 栾 权) #:nodoc:
|
16
17
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
@word = ''
|
24
|
-
@last = false
|
25
|
-
super
|
26
|
-
end
|
27
|
-
|
28
|
-
def process(char)
|
29
|
-
match = false
|
30
|
-
word = nil
|
31
|
-
|
32
|
-
if !@last && @@last_names.include?(char)
|
33
|
-
@word << char
|
34
|
-
match = true
|
35
|
-
@last = true
|
36
|
-
elsif @last && @word.chars.to_a.length < 3 && @@first_names.include?(char)
|
37
|
-
@word << char
|
38
|
-
match = true
|
39
|
-
@unit = true
|
40
|
-
else
|
41
|
-
word = @word
|
18
|
+
@@first_names = %W(文 铭 菁 郁 怡 智 德 祥 志 华 孟 庆 雅 佩 晓 蓉 明 仁 宇 青 慧 豪 琪 安
|
19
|
+
惠 宗 信 盈 君 秀 敏 伶 佳 国 荣 忠 宏 育 丽 圣 淑 彦 龙 冠 后 静 娟 子
|
20
|
+
嘉 瑞 柏 弘 芳 正 玮 贞 如 凯 元 士 伟 杰 颍 霖 玲 仪 珮 英 建 政 真 珍
|
21
|
+
美 世 立 秋 婷 贤 瑜 中 玉 维 莹 翔 家 芬 昌 裕 雯 萍 永 成 宜 鸿 珊 民
|
22
|
+
欣 哲 良 伦 燕 梦 磊 丹 元 一 昌 红 健) #:nodoc:
|
23
|
+
def initialize
|
42
24
|
@word = ''
|
43
25
|
@last = false
|
44
|
-
|
26
|
+
super
|
45
27
|
end
|
28
|
+
|
29
|
+
def process(char)
|
30
|
+
match = false
|
31
|
+
word = nil
|
32
|
+
|
33
|
+
if !@last && @@last_names.include?(char)
|
34
|
+
@word << char
|
35
|
+
match = true
|
36
|
+
@last = true
|
37
|
+
elsif @last && @word.chars.to_a.length < 3 && @@first_names.include?(char)
|
38
|
+
@word << char
|
39
|
+
match = true
|
40
|
+
@unit = true
|
41
|
+
else
|
42
|
+
word = @word
|
43
|
+
@word = ''
|
44
|
+
@last = false
|
45
|
+
match = false
|
46
|
+
end
|
46
47
|
|
47
|
-
|
48
|
+
[match, word]
|
49
|
+
end
|
48
50
|
end
|
49
51
|
end
|
data/lib/engines/number.rb
CHANGED
@@ -1,59 +1,61 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
1
|
+
module RsegEngine
|
2
|
+
class Number < Engine
|
3
|
+
@@number_symbols = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
4
|
+
'一', '二', '三', '四', '五', '六', '七', '八', '九', '十',
|
5
|
+
'零', '〇', '百', '千', '壹', '贰', '叁', '肆', '柒', '捌',
|
6
|
+
'玖', '拾', '之', '%', '¥', '分', '$', '.', '点', '第', '每']
|
7
|
+
@@subunit_symbols = ['多', '公', '英', '厘', '毫', '微', '纳', '海', '平', '立',
|
8
|
+
'方', '摄', '华', '氏', '美', '日', '澳', '港', '台', '新',
|
9
|
+
'个', '百', '佰', '千', '仟', '万', '萬', '亿', '兆', '吉']
|
10
|
+
@@unit_symbols = ['刻', '章', '回', '节', '名', '个', '届', '次', '集', '元',
|
11
|
+
'角', '例', '人', '斤', '克', '吨', '米', '里', '升', '码',
|
12
|
+
'尺', '寸', '杆', '顷', '亩', '磅', '镑', '桶', '度', '秒',
|
13
|
+
'分', '卡', '焦', '瓦', '匹', '圆', '币', '年', '月', '日',
|
14
|
+
'时', '秒', '点', '百', '佰', '仟', '千', '万', '萬', '亿',
|
15
|
+
'兆', '吉', '块', '半', '岁', '家', '所', '期', '场', '投',
|
16
|
+
'中', '辆', '只', '头']
|
16
17
|
|
17
|
-
|
18
|
-
@word = ''
|
19
|
-
@number = ''
|
20
|
-
@unit = false
|
21
|
-
@subunit = false
|
22
|
-
super
|
23
|
-
end
|
24
|
-
|
25
|
-
def process(char)
|
26
|
-
match = false
|
27
|
-
word = nil
|
28
|
-
|
29
|
-
if (!@subunit || @unit) && @@number_symbols.include?(char)
|
30
|
-
@number << char
|
31
|
-
match = true
|
32
|
-
@unit = false
|
33
|
-
@subunit = false
|
34
|
-
elsif (@number != '' || @unit) && @@subunit_symbols.include?(char)
|
35
|
-
@number << char
|
36
|
-
match = true
|
37
|
-
@subunit = true
|
38
|
-
end
|
39
|
-
|
40
|
-
if (@number != '' || @subunit) && @@unit_symbols.include?(char)
|
41
|
-
@word << @number
|
42
|
-
@word << char if !match
|
43
|
-
@number = ''
|
44
|
-
@unit = true
|
45
|
-
match = true
|
46
|
-
end
|
47
|
-
|
48
|
-
if !match
|
49
|
-
word = (@word != '') ? @word : @number
|
18
|
+
def initialize
|
50
19
|
@word = ''
|
51
20
|
@number = ''
|
52
|
-
match = false
|
53
21
|
@unit = false
|
54
22
|
@subunit = false
|
23
|
+
super
|
55
24
|
end
|
25
|
+
|
26
|
+
def process(char)
|
27
|
+
match = false
|
28
|
+
word = nil
|
56
29
|
|
57
|
-
|
30
|
+
if (!@subunit || @unit) && @@number_symbols.include?(char)
|
31
|
+
@number << char
|
32
|
+
match = true
|
33
|
+
@unit = false
|
34
|
+
@subunit = false
|
35
|
+
elsif (@number != '' || @unit) && @@subunit_symbols.include?(char)
|
36
|
+
@number << char
|
37
|
+
match = true
|
38
|
+
@subunit = true
|
39
|
+
end
|
40
|
+
|
41
|
+
if (@number != '' || @subunit) && @@unit_symbols.include?(char)
|
42
|
+
@word << @number
|
43
|
+
@word << char if !match
|
44
|
+
@number = ''
|
45
|
+
@unit = true
|
46
|
+
match = true
|
47
|
+
end
|
48
|
+
|
49
|
+
if !match
|
50
|
+
word = (@word != '') ? @word : @number
|
51
|
+
@word = ''
|
52
|
+
@number = ''
|
53
|
+
match = false
|
54
|
+
@unit = false
|
55
|
+
@subunit = false
|
56
|
+
end
|
57
|
+
|
58
|
+
[match, word]
|
59
|
+
end
|
58
60
|
end
|
59
61
|
end
|
data/lib/filters/conjunction.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
module RsegFilter
|
2
|
+
class Conjunction
|
3
|
+
@@conjunctions = %W(给 的 说 对 在 和 是 被 最 所 那 由 这 有 将 你 会 与 他 为 不 没 很 了 啊 哦 呵 把 去 从)
|
3
4
|
|
4
|
-
|
5
|
-
|
5
|
+
def self.filter(char)
|
6
|
+
@@conjunctions.include?(char) ? :conjunction : char
|
7
|
+
end
|
6
8
|
end
|
7
9
|
end
|
data/lib/filters/fullwidth.rb
CHANGED
@@ -1,17 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
1
|
+
module RsegFilter
|
2
|
+
class Fullwidth
|
3
|
+
@@fullwidth_chars = {'1' => '1', '2' => '2', '3' => '3', '4' => '4', '5' => '5', '6' => '6', '7' => '7', '8' => '8',
|
4
|
+
'9' => '9', '0' => '0', 'a' => 'a', 'b' => 'b', 'c' => 'c', 'd' => 'd', 'e' => 'e', 'f' => 'f',
|
5
|
+
'g' => 'g', 'h' => 'h', 'i' => 'i', 'j' => 'j', 'k' => 'k', 'l' => 'l', 'm' => 'm', 'n' => 'n',
|
6
|
+
'o' => 'o', 'p' => 'p', 'q' => 'q', 'r' => 'r', 's' => 's', 't' => 't', 'u' => 'u', 'v' => 'v',
|
7
|
+
'w' => 'w', 'x' => 'x', 'y' => 'y', 'z' => 'z', 'A' => 'A', 'B' => 'B', 'C' => 'C', 'D' => 'D',
|
8
|
+
'E' => 'E', 'F' => 'F', 'G' => 'G', 'H' => 'H', 'I' => 'I', 'J' => 'J', 'K' => 'K', 'L' => 'L',
|
9
|
+
'M' => 'M', 'N' => 'N', 'O' => 'O', 'P' => 'P', 'Q' => 'Q', 'R' => 'R', 'S' => 'S', 'T' => 'T',
|
10
|
+
'U' => 'U', 'V' => 'V', 'W' => 'W', 'X' => 'X', 'Y' => 'Y', 'Z' => 'Z', '-' => '-', '+' => '+',
|
11
|
+
'—' => '-', ',' => ',', '/' => '/', '·' => '.'}
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
class << self
|
14
|
+
def filter(char)
|
15
|
+
@@fullwidth_chars[char].nil? ? char : @@fullwidth_chars[char]
|
16
|
+
end
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
data/lib/filters/symbol.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
1
|
+
module RsegFilter
|
2
|
+
class Symbol
|
3
|
+
@@separators = ['`', '[', ']', '、', '=', '‘', ';', '。', '|', '?', '》',
|
4
|
+
'《', ':', '“', '{', '}', ')', '(', '*', '…', '#', '!',
|
5
|
+
'~', '’', '”', '〕', '〈', '〉', '「', '」', '『', '』', '〖', '〗',
|
6
|
+
'【', '】', '<', '>', '`', '~', '!', '@', '#', '^',
|
7
|
+
'&', '*', '\\', '(', ')', '=', '{', '}', '[', ']',
|
8
|
+
'|', ';', ':', "'", '<', '>', '?', "\n", "\t", "\r",
|
9
|
+
' ', '-', '/', '+', ',', ' ']
|
10
|
+
def self.filter(char)
|
11
|
+
@@separators.include?(char) ? :symbol : char
|
12
|
+
end
|
11
13
|
end
|
12
14
|
end
|
data/lib/rseg.rb
CHANGED
@@ -15,8 +15,14 @@ require File.join(File.dirname(__FILE__), 'filters/conjunction')
|
|
15
15
|
|
16
16
|
class Rseg
|
17
17
|
include Singleton
|
18
|
+
include RsegEngine
|
19
|
+
include RsegFilter
|
18
20
|
|
19
21
|
class << self
|
22
|
+
def dict_path=(path)
|
23
|
+
RsegEngine::Dict.dict_path = path
|
24
|
+
end
|
25
|
+
|
20
26
|
def segment(input)
|
21
27
|
Rseg.instance.input = input
|
22
28
|
Rseg.instance.segment
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rseg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 6
|
10
|
+
version: 0.1.6
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Yuanyi Zhang
|