pragmatic_segmenter 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +730 -0
- data/Rakefile +4 -0
- data/lib/pragmatic_segmenter.rb +2 -0
- data/lib/pragmatic_segmenter/abbreviation.rb +22 -0
- data/lib/pragmatic_segmenter/abbreviation_replacer.rb +149 -0
- data/lib/pragmatic_segmenter/between_punctuation.rb +78 -0
- data/lib/pragmatic_segmenter/cleaner.rb +141 -0
- data/lib/pragmatic_segmenter/ellipsis.rb +36 -0
- data/lib/pragmatic_segmenter/exclamation_words.rb +19 -0
- data/lib/pragmatic_segmenter/languages/amharic.rb +33 -0
- data/lib/pragmatic_segmenter/languages/arabic.rb +83 -0
- data/lib/pragmatic_segmenter/languages/armenian.rb +33 -0
- data/lib/pragmatic_segmenter/languages/burmese.rb +33 -0
- data/lib/pragmatic_segmenter/languages/deutsch.rb +132 -0
- data/lib/pragmatic_segmenter/languages/english.rb +44 -0
- data/lib/pragmatic_segmenter/languages/french.rb +29 -0
- data/lib/pragmatic_segmenter/languages/greek.rb +29 -0
- data/lib/pragmatic_segmenter/languages/hindi.rb +33 -0
- data/lib/pragmatic_segmenter/languages/italian.rb +39 -0
- data/lib/pragmatic_segmenter/languages/japanese.rb +58 -0
- data/lib/pragmatic_segmenter/languages/persian.rb +56 -0
- data/lib/pragmatic_segmenter/languages/russian.rb +60 -0
- data/lib/pragmatic_segmenter/languages/spanish.rb +39 -0
- data/lib/pragmatic_segmenter/languages/urdu.rb +33 -0
- data/lib/pragmatic_segmenter/list.rb +169 -0
- data/lib/pragmatic_segmenter/number.rb +35 -0
- data/lib/pragmatic_segmenter/process.rb +126 -0
- data/lib/pragmatic_segmenter/punctuation.rb +12 -0
- data/lib/pragmatic_segmenter/punctuation_replacer.rb +62 -0
- data/lib/pragmatic_segmenter/rules.rb +38 -0
- data/lib/pragmatic_segmenter/segmenter.rb +81 -0
- data/lib/pragmatic_segmenter/sentence_boundary_punctuation.rb +17 -0
- data/lib/pragmatic_segmenter/single_letter_abbreviation.rb +37 -0
- data/lib/pragmatic_segmenter/types.rb +12 -0
- data/lib/pragmatic_segmenter/version.rb +3 -0
- data/pragmatic_segmenter.gemspec +25 -0
- data/spec/performance_spec.rb +24 -0
- data/spec/pragmatic_segmenter_spec.rb +1906 -0
- data/spec/spec_helper.rb +1 -0
- metadata +150 -0
@@ -0,0 +1,35 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
module PragmaticSegmenter
|
4
|
+
# This class searches for numbers with periods within a string and
|
5
|
+
# replaces the periods.
|
6
|
+
class Number
|
7
|
+
# Rubular: http://rubular.com/r/oNyxBOqbyy
|
8
|
+
PeriodBeforeNumberRule = Rule.new(/\.(?=\d)/, '∯')
|
9
|
+
|
10
|
+
# Rubular: http://rubular.com/r/EMk5MpiUzt
|
11
|
+
NumberAfterPeriodBeforeLetterRule = Rule.new(/(?<=\d)\.(?=\S)/, '∯')
|
12
|
+
|
13
|
+
# Rubular: http://rubular.com/r/rf4l1HjtjG
|
14
|
+
NewLineNumberPeriodSpaceLetterRule = Rule.new(/(?<=\r\d)\.(?=(\s\S)|\))/, '∯')
|
15
|
+
|
16
|
+
# Rubular: http://rubular.com/r/HPa4sdc6b9
|
17
|
+
StartLineNumberPeriodRule = Rule.new(/(?<=^\d)\.(?=(\s\S)|\))/, '∯')
|
18
|
+
|
19
|
+
# Rubular: http://rubular.com/r/NuvWnKleFl
|
20
|
+
StartLineTwoDigitNumberPeriodRule = Rule.new(/(?<=^\d\d)\.(?=(\s\S)|\))/, '∯')
|
21
|
+
|
22
|
+
attr_reader :text
|
23
|
+
def initialize(text:)
|
24
|
+
@text = Text.new(text)
|
25
|
+
end
|
26
|
+
|
27
|
+
def replace
|
28
|
+
@formatted_text = @text.apply(PeriodBeforeNumberRule).
|
29
|
+
apply(NumberAfterPeriodBeforeLetterRule).
|
30
|
+
apply(NewLineNumberPeriodSpaceLetterRule).
|
31
|
+
apply(StartLineNumberPeriodRule).
|
32
|
+
apply(StartLineTwoDigitNumberPeriodRule)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
require 'pragmatic_segmenter/list'
|
3
|
+
require 'pragmatic_segmenter/abbreviation_replacer'
|
4
|
+
require 'pragmatic_segmenter/number'
|
5
|
+
require 'pragmatic_segmenter/ellipsis'
|
6
|
+
require 'pragmatic_segmenter/exclamation_words'
|
7
|
+
require 'pragmatic_segmenter/punctuation_replacer'
|
8
|
+
require 'pragmatic_segmenter/between_punctuation'
|
9
|
+
require 'pragmatic_segmenter/sentence_boundary_punctuation'
|
10
|
+
require 'pragmatic_segmenter/punctuation'
|
11
|
+
|
12
|
+
module PragmaticSegmenter
|
13
|
+
# This class processing segmenting the text.
|
14
|
+
class Process
|
15
|
+
include Rules
|
16
|
+
# Rubular: http://rubular.com/r/NqCqv372Ix
|
17
|
+
QUOTATION_AT_END_OF_SENTENCE_REGEX = /[!?\.][\"\'\u{201d}\u{201c}]\s{1}[A-Z]/
|
18
|
+
|
19
|
+
# Rubular: http://rubular.com/r/JMjlZHAT4g
|
20
|
+
SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = /(?<=[!?\.][\"\'\u{201d}\u{201c}])\s{1}(?=[A-Z])/
|
21
|
+
|
22
|
+
attr_reader :text, :doc_type
|
23
|
+
def initialize(text:, doc_type:)
|
24
|
+
@text = text
|
25
|
+
@doc_type = doc_type
|
26
|
+
end
|
27
|
+
|
28
|
+
def process
|
29
|
+
reformatted_text = PragmaticSegmenter::List.new(text: text).add_line_break
|
30
|
+
reformatted_text = replace_abbreviations(reformatted_text)
|
31
|
+
reformatted_text = replace_numbers(reformatted_text)
|
32
|
+
reformatted_text = reformatted_text.apply(GeoLocationRule)
|
33
|
+
split_lines(reformatted_text)
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def split_lines(txt)
|
39
|
+
segments = []
|
40
|
+
lines = txt.split("\r")
|
41
|
+
lines.each do |l|
|
42
|
+
next if l.eql?('')
|
43
|
+
analyze_lines(l, segments, punctuation_array)
|
44
|
+
end
|
45
|
+
sentence_array = []
|
46
|
+
segments.each_with_index do |line|
|
47
|
+
next if line.gsub(/_{3,}/, '').length.eql?(0) || line.length < 2
|
48
|
+
line = reinsert_ellipsis(line)
|
49
|
+
line = line.apply(ExtraWhiteSpaceRule)
|
50
|
+
if line =~ QUOTATION_AT_END_OF_SENTENCE_REGEX
|
51
|
+
subline = line.split(SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX)
|
52
|
+
subline.each do |s|
|
53
|
+
sentence_array << s
|
54
|
+
end
|
55
|
+
else
|
56
|
+
sentence_array << line.tr("\n", '').strip
|
57
|
+
end
|
58
|
+
end
|
59
|
+
sentence_array.reject(&:empty?)
|
60
|
+
end
|
61
|
+
|
62
|
+
def analyze_lines(line, segments, punctuation)
|
63
|
+
line = line.apply(SingleNewLineRule, EllipsisRules::All, EmailRule)
|
64
|
+
clause_1 = false
|
65
|
+
end_punc_check = false
|
66
|
+
punctuation.each do |p|
|
67
|
+
end_punc_check = true if line[-1].include?(p)
|
68
|
+
clause_1 = true if line.include?(p)
|
69
|
+
end
|
70
|
+
if clause_1
|
71
|
+
segments = process_text(line, end_punc_check, segments)
|
72
|
+
else
|
73
|
+
line.gsub!(/ȹ/, "\n")
|
74
|
+
line.gsub!(/∯/, '.')
|
75
|
+
segments << line
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def process_text(line, end_punc_check, segments)
|
80
|
+
line << 'ȸ' if !end_punc_check
|
81
|
+
PragmaticSegmenter::ExclamationWords.apply_rules(line)
|
82
|
+
between_punctutation(line)
|
83
|
+
line = line.apply(
|
84
|
+
DoublePuctationRules::All,
|
85
|
+
QuestionMarkInQuotationRule,
|
86
|
+
ExclamationPointRules::All
|
87
|
+
)
|
88
|
+
subline = sentence_boundary_punctuation(line)
|
89
|
+
subline.each_with_index do |s_l|
|
90
|
+
segments << sub_symbols(s_l)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def replace_numbers(txt)
|
95
|
+
PragmaticSegmenter::Number.new(text: txt).replace
|
96
|
+
end
|
97
|
+
|
98
|
+
def replace_abbreviations(txt)
|
99
|
+
PragmaticSegmenter::AbbreviationReplacer.new(text: txt).replace
|
100
|
+
end
|
101
|
+
|
102
|
+
def punctuation_array
|
103
|
+
PragmaticSegmenter::Punctuation.new.punct
|
104
|
+
end
|
105
|
+
|
106
|
+
def between_punctutation(txt)
|
107
|
+
PragmaticSegmenter::BetweenPunctuation.new(text: txt).replace
|
108
|
+
end
|
109
|
+
|
110
|
+
def sentence_boundary_punctuation(txt)
|
111
|
+
PragmaticSegmenter::SentenceBoundaryPunctuation.new(text: txt).split
|
112
|
+
end
|
113
|
+
|
114
|
+
def sub_symbols(txt)
|
115
|
+
txt.gsub(/∯/, '.').gsub(/♬/, '،').gsub(/♭/, ':').gsub(/ᓰ/, '。').gsub(/ᓱ/, '.')
|
116
|
+
.gsub(/ᓳ/, '!').gsub(/ᓴ/, '!').gsub(/ᓷ/, '?').gsub(/ᓸ/, '?').gsub(/☉/, '?!')
|
117
|
+
.gsub(/☈/, '!?').gsub(/☇/, '??').gsub(/☄/, '!!').delete('ȸ').gsub(/ȹ/, "\n")
|
118
|
+
end
|
119
|
+
|
120
|
+
def reinsert_ellipsis(line)
|
121
|
+
line.gsub(/ƪ/, '...').gsub(/♟/, ' . . . ')
|
122
|
+
.gsub(/♝/, '. . . .').gsub(/☏/, '..')
|
123
|
+
.gsub(/∮/, '.')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
module PragmaticSegmenter
|
4
|
+
# This class replaces punctuation that is typically a sentence boundary
|
5
|
+
# but in this case is not a sentence boundary.
|
6
|
+
class PunctuationReplacer
|
7
|
+
attr_reader :matches_array, :text
|
8
|
+
def initialize(text:, matches_array:)
|
9
|
+
@text = text
|
10
|
+
@matches_array = matches_array
|
11
|
+
end
|
12
|
+
|
13
|
+
def replace
|
14
|
+
replace_punctuation(matches_array, text)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def replace_punctuation(array, txt)
|
20
|
+
return if !array || array.empty?
|
21
|
+
txt.gsub!('(', '\\(')
|
22
|
+
txt.gsub!(')', '\\)')
|
23
|
+
txt.gsub!(']', '\\]')
|
24
|
+
txt.gsub!('[', '\\[')
|
25
|
+
txt.gsub!('-', '\\-')
|
26
|
+
array.each do |a|
|
27
|
+
a.gsub!('(', '\\(')
|
28
|
+
a.gsub!(')', '\\)')
|
29
|
+
a.gsub!(']', '\\]')
|
30
|
+
a.gsub!('[', '\\[')
|
31
|
+
a.gsub!('-', '\\-')
|
32
|
+
|
33
|
+
sub = a.gsub('.', '∯')
|
34
|
+
txt.gsub!(/#{Regexp.escape(a)}/, "#{sub}")
|
35
|
+
|
36
|
+
sub_1 = sub.gsub('。', 'ᓰ')
|
37
|
+
txt.gsub!(/#{Regexp.escape(sub)}/, "#{sub_1}")
|
38
|
+
|
39
|
+
sub_2 = sub_1.gsub('.', 'ᓱ')
|
40
|
+
txt.gsub!(/#{Regexp.escape(sub_1)}/, "#{sub_2}")
|
41
|
+
|
42
|
+
sub_3 = sub_2.gsub('!', 'ᓳ')
|
43
|
+
txt.gsub!(/#{Regexp.escape(sub_2)}/, "#{sub_3}")
|
44
|
+
|
45
|
+
sub_4 = sub_3.gsub('!', 'ᓴ')
|
46
|
+
txt.gsub!(/#{Regexp.escape(sub_3)}/, "#{sub_4}")
|
47
|
+
|
48
|
+
sub_5 = sub_4.gsub('?', 'ᓷ')
|
49
|
+
txt.gsub!(/#{Regexp.escape(sub_4)}/, "#{sub_5}")
|
50
|
+
|
51
|
+
sub_6 = sub_5.gsub('?', 'ᓸ')
|
52
|
+
txt.gsub!(/#{Regexp.escape(sub_5)}/, "#{sub_6}")
|
53
|
+
end
|
54
|
+
txt.gsub!('\\(', '(')
|
55
|
+
txt.gsub!('\\)', ')')
|
56
|
+
txt.gsub!('\\[', '[')
|
57
|
+
txt.gsub!('\\]', ']')
|
58
|
+
txt.gsub!('\\-', '-')
|
59
|
+
txt
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module PragmaticSegmenter
|
2
|
+
module Rules
|
3
|
+
# Rubular: http://rubular.com/r/EUbZCNfgei
|
4
|
+
EmailRule = Rule.new(/(\w)(\.)(\w)/, '\1∮\3')
|
5
|
+
|
6
|
+
# Rubular: http://rubular.com/r/G2opjedIm9
|
7
|
+
GeoLocationRule = Rule.new(/(?<=[a-zA-z]°)\.(?=\s*\d+)/, '∯')
|
8
|
+
|
9
|
+
SingleNewLineRule = Rule.new(/\n/, 'ȹ')
|
10
|
+
|
11
|
+
ExtraWhiteSpaceRule = Rule.new(/\s{3,}/, ' ')
|
12
|
+
|
13
|
+
# Rubular: http://rubular.com/r/aXPUGm6fQh
|
14
|
+
QuestionMarkInQuotationRule = Rule.new(/\?(?=(\'|\"))/, 'ᓷ')
|
15
|
+
|
16
|
+
module ExclamationPointRules
|
17
|
+
# Rubular: http://rubular.com/r/XS1XXFRfM2
|
18
|
+
InQuotationRule = Rule.new(/\!(?=(\'|\"))/, 'ᓴ')
|
19
|
+
|
20
|
+
# Rubular: http://rubular.com/r/sl57YI8LkA
|
21
|
+
BeforeCommaMidSentenceRule = Rule.new(/\!(?=\,\s[a-z])/, 'ᓴ')
|
22
|
+
|
23
|
+
# Rubular: http://rubular.com/r/f9zTjmkIPb
|
24
|
+
MidSentenceRule = Rule.new(/\!(?=\s[a-z])/, 'ᓴ')
|
25
|
+
|
26
|
+
All = [ InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule ]
|
27
|
+
end
|
28
|
+
|
29
|
+
module DoublePuctationRules
|
30
|
+
FirstRule = Rule.new(/\?!/, '☉')
|
31
|
+
SecondRule = Rule.new(/!\?/, '☈')
|
32
|
+
ThirdRule = Rule.new(/\?\?/, '☇')
|
33
|
+
ForthRule = Rule.new(/!!/, '☄')
|
34
|
+
|
35
|
+
All = [ FirstRule, SecondRule, ThirdRule, ForthRule ]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
require 'pragmatic_segmenter/types'
|
3
|
+
require 'pragmatic_segmenter/process'
|
4
|
+
require 'pragmatic_segmenter/cleaner'
|
5
|
+
require 'pragmatic_segmenter/languages/english'
|
6
|
+
require 'pragmatic_segmenter/languages/deutsch'
|
7
|
+
require 'pragmatic_segmenter/languages/hindi'
|
8
|
+
require 'pragmatic_segmenter/languages/persian'
|
9
|
+
require 'pragmatic_segmenter/languages/amharic'
|
10
|
+
require 'pragmatic_segmenter/languages/arabic'
|
11
|
+
require 'pragmatic_segmenter/languages/greek'
|
12
|
+
require 'pragmatic_segmenter/languages/armenian'
|
13
|
+
require 'pragmatic_segmenter/languages/burmese'
|
14
|
+
require 'pragmatic_segmenter/languages/urdu'
|
15
|
+
require 'pragmatic_segmenter/languages/french'
|
16
|
+
require 'pragmatic_segmenter/languages/italian'
|
17
|
+
require 'pragmatic_segmenter/languages/spanish'
|
18
|
+
require 'pragmatic_segmenter/languages/russian'
|
19
|
+
require 'pragmatic_segmenter/languages/japanese'
|
20
|
+
require 'pragmatic_segmenter/rules'
|
21
|
+
|
22
|
+
module PragmaticSegmenter
|
23
|
+
# This class segments a text into an array of sentences.
|
24
|
+
class Segmenter
|
25
|
+
include Rules
|
26
|
+
attr_reader :text, :language, :doc_type
|
27
|
+
def initialize(text:, **args)
|
28
|
+
return [] unless text
|
29
|
+
@language = args[:language] || 'en'
|
30
|
+
@doc_type = args[:doc_type]
|
31
|
+
if args[:clean].eql?(false)
|
32
|
+
@text = text.dup
|
33
|
+
else
|
34
|
+
case @language
|
35
|
+
when 'en'
|
36
|
+
@text = PragmaticSegmenter::Languages::English::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
|
37
|
+
when 'ja'
|
38
|
+
@text = PragmaticSegmenter::Languages::Japanese::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
|
39
|
+
else
|
40
|
+
@text = PragmaticSegmenter::Cleaner.new(text: text.dup, doc_type: args[:doc_type]).clean
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def segment
|
46
|
+
return [] unless text
|
47
|
+
case language
|
48
|
+
when 'en'
|
49
|
+
PragmaticSegmenter::Process.new(text: text, doc_type: doc_type).process
|
50
|
+
when 'de'
|
51
|
+
PragmaticSegmenter::Languages::Deutsch::Process.new(text: text, doc_type: doc_type).process
|
52
|
+
when 'es'
|
53
|
+
PragmaticSegmenter::Languages::Spanish::Process.new(text: text, doc_type: doc_type).process
|
54
|
+
when 'it'
|
55
|
+
PragmaticSegmenter::Languages::Italian::Process.new(text: text, doc_type: doc_type).process
|
56
|
+
when 'ja'
|
57
|
+
PragmaticSegmenter::Languages::Japanese::Process.new(text: text, doc_type: doc_type).process
|
58
|
+
when 'el'
|
59
|
+
PragmaticSegmenter::Languages::Greek::Process.new(text: text, doc_type: doc_type).process
|
60
|
+
when 'ru'
|
61
|
+
PragmaticSegmenter::Languages::Russian::Process.new(text: text, doc_type: doc_type).process
|
62
|
+
when 'ar'
|
63
|
+
PragmaticSegmenter::Languages::Arabic::Process.new(text: text, doc_type: doc_type).process
|
64
|
+
when 'am'
|
65
|
+
PragmaticSegmenter::Languages::Amharic::Process.new(text: text, doc_type: doc_type).process
|
66
|
+
when 'hi'
|
67
|
+
PragmaticSegmenter::Languages::Hindi::Process.new(text: text, doc_type: doc_type).process
|
68
|
+
when 'hy'
|
69
|
+
PragmaticSegmenter::Languages::Armenian::Process.new(text: text, doc_type: doc_type).process
|
70
|
+
when 'fa'
|
71
|
+
PragmaticSegmenter::Languages::Persian::Process.new(text: text, doc_type: doc_type).process
|
72
|
+
when 'my'
|
73
|
+
PragmaticSegmenter::Languages::Burmese::Process.new(text: text, doc_type: doc_type).process
|
74
|
+
when 'ur'
|
75
|
+
PragmaticSegmenter::Languages::Urdu::Process.new(text: text, doc_type: doc_type).process
|
76
|
+
else
|
77
|
+
PragmaticSegmenter::Process.new(text: text, doc_type: doc_type).process
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
module PragmaticSegmenter
|
4
|
+
# This class splits text at sentence boundary punctuation marks
|
5
|
+
class SentenceBoundaryPunctuation
|
6
|
+
SENTENCE_BOUNDARY_REGEX = /\u{ff08}(?:[^\u{ff09}])*\u{ff09}(?=\s?[A-Z])|\u{300c}(?:[^\u{300d}])*\u{300d}(?=\s[A-Z])|\((?:[^\)])*\)(?=\s[A-Z])|'(?:[^'])*'(?=\s[A-Z])|"(?:[^"])*"(?=\s[A-Z])|“(?:[^”])*”(?=\s[A-Z])|\S.*?[。..!!??ȸȹ☉☈☇☄]/
|
7
|
+
|
8
|
+
attr_reader :text
|
9
|
+
def initialize(text:)
|
10
|
+
@text = text
|
11
|
+
end
|
12
|
+
|
13
|
+
def split
|
14
|
+
text.scan(SENTENCE_BOUNDARY_REGEX)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
|
3
|
+
module PragmaticSegmenter
|
4
|
+
# This class searches for periods within an abbreviation and
|
5
|
+
# replaces the periods.
|
6
|
+
class SingleLetterAbbreviation
|
7
|
+
# Rubular: http://rubular.com/r/e3H6kwnr6H
|
8
|
+
SingleUpperCaseLetterAtStartOfLineRule = Rule.new(/(?<=^[A-Z])\.(?=\s)/, '∯')
|
9
|
+
|
10
|
+
# Rubular: http://rubular.com/r/gitvf0YWH4
|
11
|
+
SingleUpperCaseLetterRule = Rule.new(/(?<=\s[A-Z])\.(?=\s)/, '∯')
|
12
|
+
|
13
|
+
attr_reader :text
|
14
|
+
def initialize(text:)
|
15
|
+
@text = text
|
16
|
+
end
|
17
|
+
|
18
|
+
def replace
|
19
|
+
@formatted_text = replace_single_letter_abbreviations(text)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def replace_single_letter_abbreviations(txt)
|
25
|
+
new_text = replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
|
26
|
+
replace_single_uppercase_letter_abbreviation(new_text)
|
27
|
+
end
|
28
|
+
|
29
|
+
def replace_single_uppercase_letter_abbreviation_at_start_of_line(txt)
|
30
|
+
txt.apply(SingleUpperCaseLetterAtStartOfLineRule)
|
31
|
+
end
|
32
|
+
|
33
|
+
def replace_single_uppercase_letter_abbreviation(txt)
|
34
|
+
txt.apply(SingleUpperCaseLetterRule)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'pragmatic_segmenter/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "pragmatic_segmenter"
|
8
|
+
spec.version = PragmaticSegmenter::VERSION
|
9
|
+
spec.authors = ["Kevin S. Dias"]
|
10
|
+
spec.email = ["diasks2@gmail.com"]
|
11
|
+
spec.summary = %q{A rule-based sentence boundary detection gem that works out-of-the-box across many languages}
|
12
|
+
spec.description = %q{Pragmatic Segmenter is a sentence segmentation tool for Ruby. It allows you to split a text into an array of sentences. This gem provides 2 main benefits over other segmentation gems - 1) It works well even with ill-formatted text 2) It works for multiple languages }
|
13
|
+
spec.homepage = "https://github.com/diasks2/pragmatic_segmenter"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_development_dependency "rspec"
|
24
|
+
spec.add_development_dependency "rubocop"
|
25
|
+
end
|