tefil 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES +10 -1
- data/VERSION +1 -1
- data/bin/columnform +1 -0
- data/bin/linesplit +48 -0
- data/example/linesplit/run.sh +6 -0
- data/example/linesplit/sample1.txt +1 -0
- data/example/linesplit/sample2.txt +2 -0
- data/example/linesplit/sample3.txt +2 -0
- data/example/linesplit/sample4.txt +5 -0
- data/lib/tefil.rb +1 -2
- data/lib/tefil/columnformer.rb +5 -0
- data/lib/tefil/fswikitomd.rb +11 -1
- data/lib/tefil/linesplitter.rb +50 -0
- data/lib/tefil/mdtofswiki.rb +5 -5
- data/lib/tefil/textfilterbase.rb +9 -0
- data/tefil.gemspec +12 -11
- data/test/test_columnformer.rb +20 -3
- data/test/test_fswikitomd.rb +130 -80
- data/test/test_linesplitter.rb +129 -0
- data/test/test_mdtofswiki.rb +55 -30
- data/test/test_textfilterbase.rb +11 -3
- metadata +11 -11
- data/bin/eachsentence +0 -24
- data/bin/statistics +0 -24
- data/example/eachsentence/sample.txt +0 -14
- data/lib/tefil/eachsentence.rb +0 -35
- data/lib/tefil/statistics.rb +0 -38
- data/test/test_eachsentence.rb +0 -123
- data/test/test_statistics.rb +0 -38
@@ -0,0 +1,129 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
# coding: utf-8
|
3
|
+
|
4
|
+
require "helper"
|
5
|
+
require "stringio"
|
6
|
+
|
7
|
+
# 元々の行末は保存する。
|
8
|
+
# 消すようにすると、空行などの処理が面倒になる。
|
9
|
+
class TC_LineSplitter < Test::Unit::TestCase
|
10
|
+
def setup
|
11
|
+
@test00 = Tefil::LineSplitter.new(separators: %w(.))
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_process_stream
|
15
|
+
# divide
|
16
|
+
in_io = StringIO.new
|
17
|
+
in_io.puts "Abc def. Ghi jhk."
|
18
|
+
in_io.rewind
|
19
|
+
out_io = StringIO.new
|
20
|
+
@test00.process_stream(in_io, out_io)
|
21
|
+
out_io.rewind
|
22
|
+
result = out_io.read
|
23
|
+
correct = "Abc def.\n Ghi jhk.\n"
|
24
|
+
assert_equal(correct, result)
|
25
|
+
|
26
|
+
# 行末の保存
|
27
|
+
in_io = StringIO.new
|
28
|
+
in_io.puts "Abc def\nGhi jhk."
|
29
|
+
in_io.rewind
|
30
|
+
out_io = StringIO.new
|
31
|
+
@test00.process_stream(in_io, out_io)
|
32
|
+
out_io.rewind
|
33
|
+
result = out_io.read
|
34
|
+
correct = "Abc def\nGhi jhk.\n"
|
35
|
+
assert_equal(correct, result)
|
36
|
+
|
37
|
+
# indent
|
38
|
+
in_io = StringIO.new
|
39
|
+
in_io.puts " Abc def. Ghi jhk.\n"
|
40
|
+
in_io.rewind
|
41
|
+
out_io = StringIO.new
|
42
|
+
@test00.process_stream(in_io, out_io)
|
43
|
+
out_io.rewind
|
44
|
+
result = out_io.read
|
45
|
+
correct = " Abc def.\n Ghi jhk.\n"
|
46
|
+
assert_equal(correct, result)
|
47
|
+
|
48
|
+
# empty line
|
49
|
+
in_io = StringIO.new
|
50
|
+
in_io.puts "Abc def.\n\nGhi jhk.\n"
|
51
|
+
in_io.rewind
|
52
|
+
out_io = StringIO.new
|
53
|
+
@test00.process_stream(in_io, out_io)
|
54
|
+
out_io.rewind
|
55
|
+
result = out_io.read
|
56
|
+
correct = "Abc def.\n\n\nGhi jhk.\n"
|
57
|
+
assert_equal(correct, result)
|
58
|
+
|
59
|
+
# Fig.
|
60
|
+
in_io = StringIO.new
|
61
|
+
in_io.puts "Including Fig. 3. Fig. 3? Fig.\n4 does' not exist."
|
62
|
+
in_io.rewind
|
63
|
+
out_io = StringIO.new
|
64
|
+
@test00.process_stream(in_io, out_io)
|
65
|
+
out_io.rewind
|
66
|
+
result = out_io.read
|
67
|
+
correct = "Including Fig.\n 3.\n Fig.\n 3? Fig.\n\n4 does' not exist.\n"
|
68
|
+
assert_equal(correct, result)
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_process_stream_strip
|
72
|
+
# strip
|
73
|
+
test10 = Tefil::LineSplitter.new(separators: %w(.), indent_mode: :strip)
|
74
|
+
in_io = StringIO.new
|
75
|
+
in_io.puts " Abc def. Ghi jhk.\n"
|
76
|
+
in_io.rewind
|
77
|
+
out_io = StringIO.new
|
78
|
+
test10.process_stream(in_io, out_io)
|
79
|
+
out_io.rewind
|
80
|
+
result = out_io.read
|
81
|
+
correct = "Abc def.\nGhi jhk.\n"
|
82
|
+
assert_equal(correct, result)
|
83
|
+
end
|
84
|
+
|
85
|
+
#def test_process_stream_indent
|
86
|
+
# # strip
|
87
|
+
# test10 = Tefil::LineSplitter.new(separators: %w(.), indent_mode: :indent)
|
88
|
+
# in_io = StringIO.new
|
89
|
+
# in_io.puts " Abc def. Ghi jhk.\n"
|
90
|
+
# in_io.rewind
|
91
|
+
# out_io = StringIO.new
|
92
|
+
# test10.process_stream(in_io, out_io)
|
93
|
+
# out_io.rewind
|
94
|
+
# result = out_io.read
|
95
|
+
# correct = " Abc def.\n Ghi jhk.\n"
|
96
|
+
# assert_equal(correct, result)
|
97
|
+
#end
|
98
|
+
|
99
|
+
def test_process_stream_except
|
100
|
+
# except
|
101
|
+
in_io = StringIO.new
|
102
|
+
in_io.puts "Including Fig.3. Fig. 3? Fig.\n4 does' not exist.\n"
|
103
|
+
in_io.rewind
|
104
|
+
out_io = StringIO.new
|
105
|
+
test10 = Tefil::LineSplitter.new(separators: ["."], except_words: ["FIG.", "Fig."])
|
106
|
+
test10.process_stream(in_io, out_io)
|
107
|
+
out_io.rewind
|
108
|
+
result = out_io.read
|
109
|
+
correct = "Including Fig.3.\n Fig. 3? Fig.\n4 does' not exist.\n"
|
110
|
+
assert_equal(correct, result)
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_process_stream_japanese
|
114
|
+
# Japanese kutouten
|
115
|
+
test10 = Tefil::LineSplitter.new(separators: %w(. 。))
|
116
|
+
in_io = StringIO.new
|
117
|
+
in_io.puts "あいうえお。かき\nくけこ。"
|
118
|
+
in_io.rewind
|
119
|
+
out_io = StringIO.new
|
120
|
+
test10.process_stream(in_io, out_io)
|
121
|
+
out_io.rewind
|
122
|
+
result = out_io.read
|
123
|
+
correct = "あいうえお。\nかき\nくけこ。\n"
|
124
|
+
assert_equal(correct, result)
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
|
129
|
+
|
data/test/test_mdtofswiki.rb
CHANGED
@@ -5,7 +5,7 @@ require "helper"
|
|
5
5
|
require "stringio"
|
6
6
|
|
7
7
|
class Tefil::MdToFswiki
|
8
|
-
public :process_stream
|
8
|
+
public :process_stream, :process_string
|
9
9
|
end
|
10
10
|
|
11
11
|
class TC_MdToFswiki < Test::Unit::TestCase
|
@@ -14,39 +14,64 @@ class TC_MdToFswiki < Test::Unit::TestCase
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_process_stream
|
17
|
-
|
18
|
-
#
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
|
18
|
+
assert_equal("!!! head1" , @f00.process_string( "# head1" ))
|
19
|
+
assert_equal("!! head2" , @f00.process_string( "## head2" ))
|
20
|
+
assert_equal("! head3" , @f00.process_string( "### head3" ))
|
21
|
+
assert_equal("abc ''italic'' def" , @f00.process_string( "abc *italic* def" ))
|
22
|
+
assert_equal("* item" , @f00.process_string( "* item" ))
|
23
|
+
assert_equal("** item" , @f00.process_string( " * item" ))
|
24
|
+
assert_equal("*** item" , @f00.process_string( " * item" ))
|
25
|
+
assert_equal("**** item" , @f00.process_string( " * item" ))
|
26
|
+
assert_equal("+ enum" , @f00.process_string( "0. enum" ))
|
27
|
+
assert_equal("++ enum" , @f00.process_string( " 0. enum" ))
|
28
|
+
assert_equal("+++ enum" , @f00.process_string( " 0. enum" ))
|
29
|
+
assert_equal("++++ enum" , @f00.process_string( " 0. enum" ))
|
30
|
+
|
31
|
+
assert_equal("64 str" , @f00.process_string( "64 str" ))
|
32
|
+
|
33
|
+
assert_equal(" formatted text" , @f00.process_string( " formatted text" ))
|
34
|
+
assert_equal("----" , @f00.process_string( "---" ))
|
35
|
+
assert_equal("[Google|http://www.google.co.jp/]", @f00.process_string( "[Google](http://www.google.co.jp/)"))
|
36
|
+
|
25
37
|
#[ "abc **bold** def" , "abc '''bold''' def" ],
|
26
38
|
#[ , "abc ==strike== def" ],
|
27
39
|
#[ , "abc __underline__ def" ],
|
28
40
|
#[ , '"" quotation' ],
|
29
|
-
[ "* item" , "* item" ],
|
30
|
-
[ " * item" , "** item" ],
|
31
|
-
[ " * item" , "*** item" ],
|
32
|
-
[ " * item" , "**** item" ],
|
33
|
-
[ "0. enum" , "+ enum" ],
|
34
|
-
[ " 0. enum" , "++ enum" ],
|
35
|
-
[ " 0. enum" , "+++ enum" ],
|
36
|
-
[ " 0. enum" , "++++ enum" ],
|
37
41
|
#[ , "*http://www.yahoo.co.jp/" ],
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
42
|
+
|
43
|
+
|
44
|
+
|
45
|
+
#[
|
46
|
+
# [ "# head1" , "!!! head1" ],
|
47
|
+
# [ "## head2" , "!! head2" ],
|
48
|
+
# [ "### head3" , "! head3" ],
|
49
|
+
# [ "abc *italic* def" , "abc ''italic'' def" ],
|
50
|
+
# #[ "abc **bold** def" , "abc '''bold''' def" ],
|
51
|
+
# #[ , "abc ==strike== def" ],
|
52
|
+
# #[ , "abc __underline__ def" ],
|
53
|
+
# #[ , '"" quotation' ],
|
54
|
+
# [ "* item" , "* item" ],
|
55
|
+
# [ " * item" , "** item" ],
|
56
|
+
# [ " * item" , "*** item" ],
|
57
|
+
# [ " * item" , "**** item" ],
|
58
|
+
# [ "0. enum" , "+ enum" ],
|
59
|
+
# [ " 0. enum" , "++ enum" ],
|
60
|
+
# [ " 0. enum" , "+++ enum" ],
|
61
|
+
# [ " 0. enum" , "++++ enum" ],
|
62
|
+
# #[ , "*http://www.yahoo.co.jp/" ],
|
63
|
+
# [ "[Google](http://www.google.co.jp/)", "[Google|http://www.google.co.jp/]" ],
|
64
|
+
# [ " formatted text" , " formatted text" ],
|
65
|
+
# [ "---" , "----" ],
|
66
|
+
# #[ "<!-- comment-->" , "// comment" ],
|
67
|
+
#].each do |i|
|
68
|
+
# $stdin = StringIO.new
|
69
|
+
# $stdin.puts i[0]
|
70
|
+
# $stdin.rewind
|
71
|
+
# #str = capture_stdout{}
|
72
|
+
# result = capture_stdout{ @f00.filter([])}
|
73
|
+
# correct = sprintf("#{i[1]}\n")
|
74
|
+
# assert_equal(correct, result)
|
75
|
+
#end
|
51
76
|
end
|
52
77
|
end
|
data/test/test_textfilterbase.rb
CHANGED
@@ -9,10 +9,12 @@ require "tempfile"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
class SampleFilter < Tefil::TextFilterBase
|
12
|
-
|
12
|
+
public :process_string
|
13
|
+
|
14
|
+
def process_stream(in_io, out_io)
|
13
15
|
#results = []
|
14
|
-
|
15
|
-
|
16
|
+
in_io.each do |line|
|
17
|
+
out_io.print line.sub('a', 'A')
|
16
18
|
end
|
17
19
|
end
|
18
20
|
end
|
@@ -134,6 +136,12 @@ class TestTefil < Test::Unit::TestCase
|
|
134
136
|
assert_equal(["Abc\n", "def\n", "cAb\n"], tmp)
|
135
137
|
end
|
136
138
|
|
139
|
+
def test_process_string
|
140
|
+
result = @t00.process_string("abc\naabbcc")
|
141
|
+
correct = "Abc\nAabbcc"
|
142
|
+
assert_equal(correct, result)
|
143
|
+
end
|
144
|
+
|
137
145
|
# def test_self_filter
|
138
146
|
# $stdin = StringIO.new
|
139
147
|
# $stdin.puts "abc"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tefil
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ippei94da
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-05-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -115,14 +115,13 @@ executables:
|
|
115
115
|
- calc
|
116
116
|
- columnanalyze
|
117
117
|
- columnform
|
118
|
-
- eachsentence
|
119
118
|
- fswiki2md
|
120
119
|
- indentconv
|
121
120
|
- indentstat
|
121
|
+
- linesplit
|
122
122
|
- linesub
|
123
123
|
- md2fswiki
|
124
124
|
- percentpack
|
125
|
-
- statistics
|
126
125
|
- zshescape
|
127
126
|
extensions: []
|
128
127
|
extra_rdoc_files:
|
@@ -139,14 +138,13 @@ files:
|
|
139
138
|
- bin/calc
|
140
139
|
- bin/columnanalyze
|
141
140
|
- bin/columnform
|
142
|
-
- bin/eachsentence
|
143
141
|
- bin/fswiki2md
|
144
142
|
- bin/indentconv
|
145
143
|
- bin/indentstat
|
144
|
+
- bin/linesplit
|
146
145
|
- bin/linesub
|
147
146
|
- bin/md2fswiki
|
148
147
|
- bin/percentpack
|
149
|
-
- bin/statistics
|
150
148
|
- bin/zshescape
|
151
149
|
- doc/memo.txt
|
152
150
|
- example/calc/run.zsh
|
@@ -155,26 +153,29 @@ files:
|
|
155
153
|
- example/columnanalyze/run.zsh
|
156
154
|
- example/columnformer/indent.txt
|
157
155
|
- example/columnformer/sample.txt
|
158
|
-
- example/eachsentence/sample.txt
|
159
156
|
- example/indentconv/sample0.txt
|
160
157
|
- example/indentconv/sample1.txt
|
161
158
|
- example/indentstat/indent4.txt
|
162
159
|
- example/indentstat/sample0.txt
|
163
160
|
- example/indentstat/sample1.txt
|
161
|
+
- example/linesplit/run.sh
|
162
|
+
- example/linesplit/sample1.txt
|
163
|
+
- example/linesplit/sample2.txt
|
164
|
+
- example/linesplit/sample3.txt
|
165
|
+
- example/linesplit/sample4.txt
|
164
166
|
- example/percentpack/sample.txt
|
165
167
|
- example/zshescape/sample.txt
|
166
168
|
- lib/tefil.rb
|
167
169
|
- lib/tefil/calculator.rb
|
168
170
|
- lib/tefil/columnanalyzer.rb
|
169
171
|
- lib/tefil/columnformer.rb
|
170
|
-
- lib/tefil/eachsentence.rb
|
171
172
|
- lib/tefil/fswikitomd.rb
|
172
173
|
- lib/tefil/indentconverter.rb
|
173
174
|
- lib/tefil/indentstatistics.rb
|
175
|
+
- lib/tefil/linesplitter.rb
|
174
176
|
- lib/tefil/linesubstituter.rb
|
175
177
|
- lib/tefil/mdtofswiki.rb
|
176
178
|
- lib/tefil/percentpacker.rb
|
177
|
-
- lib/tefil/statistics.rb
|
178
179
|
- lib/tefil/textfilterbase.rb
|
179
180
|
- lib/tefil/zshescaper.rb
|
180
181
|
- tefil.gemspec
|
@@ -185,14 +186,13 @@ files:
|
|
185
186
|
- test/test_calculator.rb
|
186
187
|
- test/test_columnanalyzer.rb
|
187
188
|
- test/test_columnformer.rb
|
188
|
-
- test/test_eachsentence.rb
|
189
189
|
- test/test_fswikitomd.rb
|
190
190
|
- test/test_indentconverter.rb
|
191
191
|
- test/test_indentstatistics.rb
|
192
|
+
- test/test_linesplitter.rb
|
192
193
|
- test/test_linesubstituter.rb
|
193
194
|
- test/test_mdtofswiki.rb
|
194
195
|
- test/test_percentpacker.rb
|
195
|
-
- test/test_statistics.rb
|
196
196
|
- test/test_textfilterbase.rb
|
197
197
|
- test/test_zshescaper.rb
|
198
198
|
homepage: http://github.com/ippei94da/tefil
|
data/bin/eachsentence
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
# coding: utf-8
|
3
|
-
|
4
|
-
# convert to text with each line / each sentence.
|
5
|
-
|
6
|
-
require "pp"
|
7
|
-
require "optparse"
|
8
|
-
require "rubygems"
|
9
|
-
require "tefil"
|
10
|
-
|
11
|
-
|
12
|
-
# option analysis
|
13
|
-
options = {}
|
14
|
-
op = OptionParser.new
|
15
|
-
#op.banner = [
|
16
|
-
# "Usage: #{File.basename("#{__FILE__}")} [options] [files]",
|
17
|
-
#].join("\n")
|
18
|
-
op.on("-o" , "--overwrite" , "Overwrite."){ options[:overwrite] = true}
|
19
|
-
op.parse!(ARGV)
|
20
|
-
|
21
|
-
options[:overwrite] ||= false
|
22
|
-
|
23
|
-
is = Tefil::EachSentence.new options
|
24
|
-
is.filter(ARGV)
|
data/bin/statistics
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
# coding: utf-8
|
3
|
-
|
4
|
-
require "pp"
|
5
|
-
require "optparse"
|
6
|
-
require "rubygems"
|
7
|
-
require "tefil"
|
8
|
-
|
9
|
-
|
10
|
-
# option analysis
|
11
|
-
options = {}
|
12
|
-
op = OptionParser.new
|
13
|
-
#op.banner = [
|
14
|
-
# "Usage: #{File.basename("#{__FILE__}")} [options] [files]",
|
15
|
-
#].join("\n")
|
16
|
-
op.on("-o" , "--overwrite" , "Overwrite."){ options[:overwrite] = true}
|
17
|
-
op.parse!(ARGV)
|
18
|
-
|
19
|
-
|
20
|
-
options[:overwrite] ||= false
|
21
|
-
|
22
|
-
stat = Tefil::Statistics.new options
|
23
|
-
stat.filter(ARGV)
|
24
|
-
|
@@ -1,14 +0,0 @@
|
|
1
|
-
After the loop is broken, a pair of nodes, comprising an OSS
|
2
|
-
and its qe-OSS, is obtained; OSS $(s_4^{-}, s_2)$ and
|
3
|
-
$(s_2, s_4)$ in Fig. \ref{fig1}. Each of the OSSs can be
|
4
|
-
elevated to $\vstart$ by using its stored data. An MEP that
|
5
|
-
connects $\vstart$ and qe-OSS of $\vstart$.
|
6
|
-
|
7
|
-
あいうえお。かき
|
8
|
-
くけこ。
|
9
|
-
あいうえお.かき
|
10
|
-
くけこ.
|
11
|
-
あいうえお、かき
|
12
|
-
くけこ、
|
13
|
-
あいうえお,かき
|
14
|
-
くけこ,
|
data/lib/tefil/eachsentence.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
class Tefil::EachSentence < Tefil::TextFilterBase
|
3
|
-
|
4
|
-
END_CHAR = %w(. ? . 。)
|
5
|
-
NOT_END_WORDS = ["Fig.", "FIG."]
|
6
|
-
|
7
|
-
def initialize(options = {})
|
8
|
-
options[:smart_filename] = true
|
9
|
-
@minimum = options[:minimum]
|
10
|
-
super(options)
|
11
|
-
end
|
12
|
-
|
13
|
-
def process_stream(in_io, out_io)
|
14
|
-
results = []
|
15
|
-
#words = []
|
16
|
-
in_io.read.strip.split("\n").each do |line|
|
17
|
-
new_line = ''
|
18
|
-
#line.gsub!("\n", ' ')
|
19
|
-
line.chars.each do |char|
|
20
|
-
new_line += char
|
21
|
-
new_line += "\n" if (END_CHAR.include?(char))
|
22
|
-
end
|
23
|
-
NOT_END_WORDS.each do |word|
|
24
|
-
new_line.gsub!(/#{word}\n/, word)
|
25
|
-
end
|
26
|
-
new_line.gsub!(/\n */, "\n")
|
27
|
-
new_line.strip!
|
28
|
-
new_line.gsub!(/ */, " ")
|
29
|
-
results << new_line
|
30
|
-
end
|
31
|
-
out_io.puts results.join("\n")
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
|