tefil 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a9f33d97e6372c4513f17026dab55e47d8452db9
-  data.tar.gz: 658c279c7e6a680f391835939078731c54f59d64
+  metadata.gz: b17ca385c9007e407470b7bc20a352f1e218a46f
+  data.tar.gz: 61dc4ec8abad83dc9e306a60347a383b99c90628
 SHA512:
-  metadata.gz: 7f5c7690d8c562c88f7337553d793d66529660a6ee661ea9f50a1a19f2f08c5c6efbe3c601bbb0018b39dd1f9b9c150742b539a917ad43daa9bf3ba323a5eb12
-  data.tar.gz: 234e09896e7d87602852e0c5469dcb28206a7897db63ecf485ad7b1a6c429dd31284dfe309cf0fe4b9d5617e650e565f46c86adf57c512129f18dcd7434018be
+  metadata.gz: ca0479bb7601d8f976bed6e8db4aaa9722c22721a8e8cdde0d53f177d5d26b48f8609dd95c9f845316359eb6d0aadc436777b33aa1770a009dc85317f4f8fd55
+  data.tar.gz: b8544dd354cf217ebcead34b7db977184dd0e24329cce552dd02c1df4aaa0a5ef1ab36b926860b2823f79b9804b30ee936e0d1dc6cefcd7a9b7a4071f0ecee29

data/CHANGES

@@ -1,6 +1,15 @@
 = tefil changelog
 
-== Master (for 0.1.5)
+== Version 0.1.5 [2018-05-11] released
+
+* bin/fswiki2md: add new line after header
+* bugfix for new line at the end of the stream
+* Enable to use not String in the matrix for Tefil::ColumnFormer#form
+* bugfix for bin/md2fswiki
+* delete bin/statistics # move to gem 'malge'
+* catch broken pipe and do nothing
+* bin/columnform: add --transpose option
+* rename: bin/eachsentence to bin/linesplit with adding some functions
 
 == Version 0.1.4 [2017-05-31] released
 

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.5

data/bin/columnform

@@ -19,6 +19,7 @@ op.on("-o"     , "--overwrite"     , "Overwrite"){    OPTIONS[:overwrite] = true
 op.on("-s char", "--separator=char", "Indicate separator"){|val| OPTIONS[:separator] = val}
 op.on("-l"     , "--left-just"   , "Left justified (Default)" ){ OPTIONS[:just] = :left}
 op.on("-r"     , "--right-just"  , "Right justified"){ OPTIONS[:just] = :right}
+op.on("-t"     , "--transpose"  , "Transpose matrix"){ OPTIONS[:transpose] = true}
 op.parse!(ARGV)
 
 ## default settings

data/bin/linesplit

@@ -0,0 +1,48 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+# split a line to lines.
+
+require "pp"
+require "optparse"
+require "rubygems"
+require "tefil"
+
+
+# option analysis
+options = {
+  #:period => true,
+  :separator => ".",
+  :except => "" 
+}
+op = OptionParser.new
+#op.banner = [
+#  "Usage: #{File.basename("#{__FILE__}")} [options] [files]",
+#].join("\n")
+op.on("-o"     , "--overwrite" , "Overwrite."){ options[:overwrite] =    true}
+op.on("-t str" , "--separator=str"  , "Target strings. Default value is '.'"){ |v| options[:separator] = v}
+op.on("-e str" , "--except=str"  , "Except strings."){ |v| options[:except] = v}
+op.on("-s"     , "--space"     , "Add space to separators."){ options[:space] = true}
+op.on("-S"     , "--strip"     , "Strip extra space at head and tail of line."){ options[:strip] = true}
+op.parse!(ARGV)
+
+options[:overwrite] ||= false
+
+#EXCEPT_WORDS = ["Fig.", "FIG."]
+
+separators = []
+separators += options[:separator].split(" ")
+separators << ' ' if options[:space]
+separators.uniq!
+
+indent_mode = :no
+indent_mode = :strip if options[:strip]
+
+#pp separators; exit
+
+is = Tefil::LineSplitter.new(separators: separators,
+                             except_words: options[:except].split(" "),
+                             indent_mode: indent_mode,
+                             options: options)
+#pp is; exit
+is.filter(ARGV)

data/example/linesplit/run.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+
+linesplit sample1.txt --separator="." --except="Fig."
+linesplit sample2.txt --separator="。 ． 、 ，"
+linesplit sample3.txt --separator="、 。" --except="Fig."
+linesplit sample4.txt --strip

data/example/linesplit/sample1.txt

@@ -0,0 +1 @@
+A bad workman always blames his tools. Once a beggar, always a beggar.

data/example/linesplit/sample2.txt

@@ -0,0 +1,2 @@
+あいうえお。かきくけこ。あいうえお．かきくけこ．あいうえお、かきくけこ、あいうえお，かきくけこ，
+

data/example/linesplit/sample3.txt

@@ -0,0 +1,2 @@
+本日は晴天なり。晴天の写真を Fig. 1 に示す。かるいきびんなこねこなんびきいるかといわれれば、たけやぶやけた、しんぶんし、きつつき、とまとと返された。
+昨日は曇天なり。

data/example/linesplit/sample4.txt

@@ -0,0 +1,5 @@
+indent0. abcde.
+  indent2. abcde.
+    indent4. abcde.
+  indent2. abcde.
+indent0. abcde.

data/lib/tefil.rb

@@ -11,11 +11,10 @@ require 'tefil/columnformer'
 require 'tefil/columnanalyzer'
 require 'tefil/indentconverter'
 require 'tefil/indentstatistics'
-require 'tefil/statistics'
 require 'tefil/linesubstituter'
 require 'tefil/percentpacker'
 require 'tefil/zshescaper'
-require 'tefil/eachsentence.rb'
+require 'tefil/linesplitter.rb'
 require 'tefil/fswikitomd.rb'
 require 'tefil/mdtofswiki.rb'
 

data/lib/tefil/columnformer.rb

@@ -29,12 +29,16 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
   def initialize(options = {})
     @just = options[:just] || :left
     @separator = options[:separator] || ' '
+    @transpose = options[:transpose]
     super(options)
   end
 
   #def form(matrix, io = $stdout, separator = " ", left = false)
   def form(matrix, io = $stdout, indent = 0)
     #Obtain max length for each column.
+
+    matrix = matrix.transpose if @transpose
+
     max_lengths = []
     matrix.each do |row|
       row.each_with_index do |item, index|
@@ -49,6 +53,7 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
     matrix.each do |row|
       new_items = []
       row.each_with_index do |item, index|
+        item = item.to_s
         method = (@just.to_s + "_just").to_sym
         new_items[index] = item.send(method, max_lengths[index])
       end

data/lib/tefil/fswikitomd.rb

@@ -57,7 +57,17 @@ class Tefil::FswikiToMd < Tefil::TextFilterBase
       end
 
       # 出力
-      out_io.puts line
+      out_io.print line
+
+      #p "test"
+      ## 空行処理
+      case
+      when type == :head1   then out_io.puts
+      when type == :head2   then out_io.puts
+      when type == :head3   then out_io.puts
+      else # type == :pain  then  'do nothing'
+      end
+
     end
   end
 end

data/lib/tefil/linesplitter.rb

@@ -0,0 +1,50 @@
+# coding: utf-8
+class Tefil::LineSplitter < Tefil::TextFilterBase
+
+  def initialize(separators: , indent_mode: :no, except_words: [], options: {})
+    options[:smart_filename] = true
+    @minimum = options[:minimum]
+    @separators = separators
+    @except_words = except_words
+    @indent_mode = indent_mode
+    super(options)
+  end
+
+  def process_stream(in_io, out_io)
+    results = []
+ 
+    # prepare substitute info to get back to original for except rule.
+    sub_except_words = Marshal.load(Marshal.dump(@except_words))
+    @separators.each do |str|
+      sub_except_words.map do |word|
+        word.gsub!(str, str + "\n")
+      end
+    end
+
+    in_io.read.split("\n").each do |line|
+      #if @indent_mode == :indent
+      #  /^( *)/ =~ line
+      #  head_spaces = $1
+      #end
+      @separators.each do |str|
+        line.gsub!(str, str + "\n")
+      end
+      @except_words.each_with_index do |word, index|
+        line.gsub!(sub_except_words[index], word)
+      end
+      # 行の頭と末尾の空白 strip
+      if @indent_mode == :strip
+        line.gsub!(/\n  */, "\n")
+        line.strip!
+        line.gsub!(/  */, " ")
+      end
+      #if @indent_mode == :indent
+      #  results << head_spaces + line
+      #end
+      results << line
+    end
+    out_io.puts results.join("\n")
+  end
+
+end
+

data/lib/tefil/mdtofswiki.rb

@@ -18,10 +18,10 @@ class Tefil::MdToFswiki < Tefil::TextFilterBase
       when line.sub!(/^    \*/  , '') then type = :item3
       when line.sub!(/^  \*/    , '') then type = :item2
       when line.sub!(/^\*/      , '') then type = :item1
-      when line.sub!(/^      \d+./, '') then type = :enum4
-      when line.sub!(/^    \d+./  , '') then type = :enum3
-      when line.sub!(/^  \d+./    , '') then type = :enum2
-      when line.sub!(/^\d+./      , '') then type = :enum1
+      when line.sub!(/^      \d+\./, '') then type = :enum4
+      when line.sub!(/^    \d+\./  , '') then type = :enum3
+      when line.sub!(/^  \d+\./    , '') then type = :enum2
+      when line.sub!(/^\d+\./      , '') then type = :enum1
       when line.sub!(/^    /    , '') then type = :pre
       when line.sub!(/^---/     , '') then type = :hline
       else                                 type = :plain
@@ -54,7 +54,7 @@ class Tefil::MdToFswiki < Tefil::TextFilterBase
       end
 
       # 出力
-      out_io.puts line
+      out_io.print line
     end
   end
 end

data/lib/tefil/textfilterbase.rb

@@ -71,6 +71,7 @@ class Tefil::TextFilterBase
         rescue ArgumentError, Errno::EISDIR
           $stderr.puts $!
           next
+        rescue Errno::EPIPE
         end
 
         if @overwrite
@@ -105,5 +106,13 @@ class Tefil::TextFilterBase
     raise NotRedefinedMethodError
   end
 
+  def process_string(in_str)
+    in_io = StringIO.open in_str
+    out_io = StringIO.new
+    process_stream(in_io, out_io)
+    out_io.rewind
+    out_io.read
+  end
+
 end
 

data/tefil.gemspec

@@ -2,19 +2,19 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: tefil 0.1.4 ruby lib
+# stub: tefil 0.1.5 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "tefil"
-  s.version = "0.1.4"
+  s.version = "0.1.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["ippei94da"]
-  s.date = "2017-05-31"
+  s.date = "2018-05-11"
   s.description = "This gem provides a framework of text filter.\n    Tefil eneable to make text filter commands which have overwrite option easily.\n  "
   s.email = "ippei94da@gmail.com"
-  s.executables = ["calc", "columnanalyze", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "statistics", "zshescape"]
+  s.executables = ["calc", "columnanalyze", "columnform", "fswiki2md", "indentconv", "indentstat", "linesplit", "linesub", "md2fswiki", "percentpack", "zshescape"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -30,14 +30,13 @@ Gem::Specification.new do |s|
     "bin/calc",
     "bin/columnanalyze",
     "bin/columnform",
-    "bin/eachsentence",
     "bin/fswiki2md",
     "bin/indentconv",
     "bin/indentstat",
+    "bin/linesplit",
     "bin/linesub",
     "bin/md2fswiki",
     "bin/percentpack",
-    "bin/statistics",
     "bin/zshescape",
     "doc/memo.txt",
     "example/calc/run.zsh",
@@ -46,26 +45,29 @@ Gem::Specification.new do |s|
     "example/columnanalyze/run.zsh",
     "example/columnformer/indent.txt",
     "example/columnformer/sample.txt",
-    "example/eachsentence/sample.txt",
     "example/indentconv/sample0.txt",
     "example/indentconv/sample1.txt",
     "example/indentstat/indent4.txt",
     "example/indentstat/sample0.txt",
     "example/indentstat/sample1.txt",
+    "example/linesplit/run.sh",
+    "example/linesplit/sample1.txt",
+    "example/linesplit/sample2.txt",
+    "example/linesplit/sample3.txt",
+    "example/linesplit/sample4.txt",
     "example/percentpack/sample.txt",
     "example/zshescape/sample.txt",
     "lib/tefil.rb",
     "lib/tefil/calculator.rb",
     "lib/tefil/columnanalyzer.rb",
     "lib/tefil/columnformer.rb",
-    "lib/tefil/eachsentence.rb",
     "lib/tefil/fswikitomd.rb",
     "lib/tefil/indentconverter.rb",
     "lib/tefil/indentstatistics.rb",
+    "lib/tefil/linesplitter.rb",
     "lib/tefil/linesubstituter.rb",
     "lib/tefil/mdtofswiki.rb",
     "lib/tefil/percentpacker.rb",
-    "lib/tefil/statistics.rb",
     "lib/tefil/textfilterbase.rb",
     "lib/tefil/zshescaper.rb",
     "tefil.gemspec",
@@ -76,14 +78,13 @@ Gem::Specification.new do |s|
     "test/test_calculator.rb",
     "test/test_columnanalyzer.rb",
     "test/test_columnformer.rb",
-    "test/test_eachsentence.rb",
     "test/test_fswikitomd.rb",
     "test/test_indentconverter.rb",
     "test/test_indentstatistics.rb",
+    "test/test_linesplitter.rb",
     "test/test_linesubstituter.rb",
     "test/test_mdtofswiki.rb",
     "test/test_percentpacker.rb",
-    "test/test_statistics.rb",
     "test/test_textfilterbase.rb",
     "test/test_zshescaper.rb"
   ]

data/test/test_columnformer.rb

@@ -15,6 +15,7 @@ class TC_ColumnFormer < Test::Unit::TestCase
     @cf00 = Tefil::ColumnFormer.new
     @cf01 = Tefil::ColumnFormer.new({:just => :right})
     @cf02 = Tefil::ColumnFormer.new({:separator => ','})
+    @cf03 = Tefil::ColumnFormer.new({:transpose => true})
   end
 
   def test_print_size
@@ -38,7 +39,6 @@ class TC_ColumnFormer < Test::Unit::TestCase
     io.rewind
     assert_equal("  a   ab\n  abc a\n", io.read)
 
-
     io = StringIO.new
     @cf01.form(matrix, io)
     io.rewind
@@ -60,8 +60,25 @@ class TC_ColumnFormer < Test::Unit::TestCase
     io.rewind
     assert_equal("abc    def\nあいう えおか\n", io.read)
 
-    ####
-    
+    ##### not string
+    io = StringIO.new
+    matrix = [
+      [0, 1],
+      [2, 3],
+    ]
+    @cf00.form(matrix, io)
+    io.rewind
+    assert_equal("0 1\n2 3\n", io.read)
+
+    #### transpose
+    io = StringIO.new
+    matrix = [
+      [0, 1],
+      [2, 3],
+    ]
+    @cf03.form(matrix, io)
+    io.rewind
+    assert_equal("0 2\n1 3\n", io.read)
   end
 end
 

data/test/test_fswikitomd.rb

@@ -5,7 +5,8 @@ require "helper"
 require "stringio"
 
 class Tefil::FswikiToMd
-  public :process_stream
+  public :process_stream,
+   :process_string
 end
 
 class TC_FswikiToMd < Test::Unit::TestCase
@@ -13,91 +14,140 @@ class TC_FswikiToMd < Test::Unit::TestCase
     @f00 = Tefil::FswikiToMd.new()
   end
 
-  def test_process_stream
-    [
-      ["!!! head1"                         , "# head1"                            ],
-      ["!! head2"                          , "## head2"                           ],
-      ["! head3"                           , "### head3"                          ],
-      ["abc ''italic'' def"                , "abc *italic* def"                   ],
-      ["abc '''bold''' def"                , "abc **bold** def"                   ],
-      #["abc ==strike== def"               ,                                      ],
-      #["abc __underline__ def"            ,                                      ],
-      #['"" quotation'                     ,                                      ],
-      ["* item"                            , "* item"                             ],
-      ["** item"                           , "    * item"                           ],
-      ["*** item"                          , "        * item"                         ],
-      ["**** item"                         , "            * item"                       ],
-      ["+ enum"                            , "1. enum"                            ],
-      ["++ enum"                           , "    1. enum"                          ],
-      ["+++ enum"                          , "        1. enum"                        ],
-      ["++++ enum"                         , "            1. enum"                      ],
-      #["*http://www.yahoo.co.jp/"         ,                                      ],
-      ["[Google|http://www.google.co.jp/]" , "[Google](http://www.google.co.jp/)" ],
-      [" formatted text"                   , "    formatted text"                 ],
-      ["----"                              , "---"                                ],
-      ["// comment"                        , "<!-- comment-->"                    ],
-    ].each do |i|
-      $stdin = StringIO.new
-      $stdin.puts i[0]
-      $stdin.rewind
-      #str = capture_stdout{}
-      result = capture_stdout{ @f00.filter([])}
-      correct = sprintf("#{i[1]}\n")
-      assert_equal(correct, result)
-    end
+  def test_process_string
+    assert_equal("# head1\n\n", @f00.process_string("!!! head1\n"))
+    assert_equal("# head2\n\n", @f00.process_string("!!! head2\n"))
+    assert_equal("# head3\n\n", @f00.process_string("!!! head3\n"))
 
-    ## stdin -> stdout
-    #$stdin = StringIO.new
-    #$stdin.puts "!!! head1"
-    #$stdin.puts "!! head2"
-    #$stdin.puts "! head3"
-    #$stdin.puts "abc ''italic'' def"
-    #$stdin.puts "abc '''bold''' def"
-    ##$stdin.puts "abc ==strike== def"
-    ##$stdin.puts "abc __underline__ def"
-    ##$stdin.puts '"" quotation'
-    #$stdin.puts "* item"
-    #$stdin.puts "** item"
-    #$stdin.puts "*** item"
-    #$stdin.puts "**** item"
-    #$stdin.puts "+ enum"
-    #$stdin.puts "++ enum"
-    #$stdin.puts "+++ enum"
-    #$stdin.puts "++++ enum"
-    ##$stdin.puts "*http://www.yahoo.co.jp/"
-    #$stdin.puts "[Google|http://www.google.co.jp/]"
-    #$stdin.puts " formatted text"
-    #$stdin.puts "----" #horizontal line
-    ##$stdin.puts "abc // commented out"
-    #$stdin.rewind
-    #str = capture_stdout{}
-    #result = capture_stdout{ @f00.filter([])}
-    #correct = [
-    #  "# head1",
-    #  "## head2",
-    #  "### head3",
-    #  "abc *italic* def",
-    #  "abc **bold** def",
-    #  "* item",
-    #  "  * item",
-    #  "    * item",
-    #  "      * item",
-    #  "0. enum",
-    #  "  0. enum",
-    #  "    0. enum",
-    #  "      0. enum",
-    #  "[Google](http://www.google.co.jp/)",
-    #  "    formatted text",
-    #  "---",
-    #  ""
-    #].join("\n")
-    #assert_equal(correct, result)
+    assert_equal("abc *italic* def",
+                 @f00.process_string("abc ''italic'' def"))
+    assert_equal("abc **bold** def",
+                 @f00.process_string("abc '''bold''' def"))
+    assert_equal( "* item",
+                 @f00.process_string("* item"))
+    assert_equal("    * item",
+                 @f00.process_string("** item"))
+    assert_equal("        * item",
+                 @f00.process_string("*** item"))
+    assert_equal("            * item",
+                 @f00.process_string("**** item"))
+    assert_equal("1. enum",
+                 @f00.process_string("+ enum"))
+    assert_equal("    1. enum",
+                 @f00.process_string("++ enum"))
+    assert_equal("        1. enum",
+                 @f00.process_string("+++ enum"))
+    assert_equal("            1. enum",
+                 @f00.process_string("++++ enum"))
+    assert_equal("[Google](http://www.google.co.jp/)",
+                 @f00.process_string("[Google|http://www.google.co.jp/]" ))
+    assert_equal("    formatted text",
+                 @f00.process_string(" formatted text"))
+    assert_equal("---",
+                 @f00.process_string("----"))
+    assert_equal("<!-- comment-->",
+                 @f00.process_string("// comment"))
   end
-end
 
+  def test_process_stream
+    input = <<HERE
+!!! head1
+!! head2
+! head3
+abc ''italic'' def
+abc '''bold''' def
+* item
+** item
+*** item
+**** item
++ enum
+++ enum
++++ enum
+++++ enum
+[Google|http://www.google.co.jp/]" ,
+ formatted text"
+----"
+// comment"
+HERE
+
+#abc ==strike== def
+#abc __underline__ def
+#"" quotation'
+#*http://www.yahoo.co.jp/"
 
+#    correct = <<HERE
+## head1
+#
+## head1
+#
+### head2
+#
+#### head3
+#
+#abc *italic* def
+#abc **bold** def
+#
+#* item
+#    * item
+#        * item
+#            * item
+#1. enum
+#    1. enum
+#        1. enum
+#            1. enum
+#
+#[Google](http://www.google.co.jp/)
+#    formatted text
+#---
+#<!-- comment-->
+#HERE
+#
+#
+#      in_io = StringIO.new
+#      in_io.puts input
+#      in_io.rewind
+#      out_io = StringIO.new
+#      @f00.process_stream(in_io, out_io)
+#      out_io.rewind
+#      result = out_io.read
+#
+#      assert_equal(correct, result)
+    #end
 
+    #[
+    #  ["!!! head1"                         , "# head1"                            ],
+    #  ["!! head2"                          , "## head2"                           ],
+    #  ["! head3"                           , "### head3"                          ],
+    #  ["abc ''italic'' def"                , "abc *italic* def"                   ],
+    #  ["abc '''bold''' def"                , "abc **bold** def"                   ],
+    #  #["abc ==strike== def"               ,                                      ],
+    #  #["abc __underline__ def"            ,                                      ],
+    #  #['"" quotation'                     ,                                      ],
+    #  ["* item"                            , "* item"                             ],
+    #  ["** item"                           , "    * item"                           ],
+    #  ["*** item"                          , "        * item"                         ],
+    #  ["**** item"                         , "            * item"                       ],
+    #  ["+ enum"                            , "1. enum"                            ],
+    #  ["++ enum"                           , "    1. enum"                          ],
+    #  ["+++ enum"                          , "        1. enum"                        ],
+    #  ["++++ enum"                         , "            1. enum"                      ],
+    #  #["*http://www.yahoo.co.jp/"         ,                                      ],
+    #  ["[Google|http://www.google.co.jp/]" , "[Google](http://www.google.co.jp/)" ],
+    #  [" formatted text"                   , "    formatted text"                 ],
+    #  ["----"                              , "---"                                ],
+    #  ["// comment"                        , "<!-- comment-->"                    ],
+    #].each do |i|
+    #  $stdin = StringIO.new
+    #  $stdin.puts i[0]
+    #  $stdin.rewind
+    #  #str = capture_stdout{}
+    #  result = capture_stdout{ @f00.filter([])}
+    #  correct = sprintf("#{i[1]}\n")
+    #  assert_equal(correct, result)
+    #end
 
 
 
 
+  end
+end