tefil 0.1.3 → 0.1.4

data/example/columnanalyze/run.zsh

@@ -0,0 +1,13 @@
+#! /usr/bin/zsh
+
+cat qstat.out | columnanalyze
+echo ------------------------------------------------------------
+cat qstat.out | columnanalyze 4
+echo ------------------------------------------------------------
+cat qstat.out | columnanalyze 4 5
+echo ------------------------------------------------------------
+cat qstat.out | columnanalyze 4=bob
+echo ------------------------------------------------------------
+cat qstat.out | columnanalyze 4=bob 5=r
+echo ------------------------------------------------------------
+cat qstat.out | columnanalyze 4=bob 5=r 3

data/example/columnformer/indent.txt

@@ -0,0 +1,2 @@
+  a ab
+    abc a

data/lib/tefil.rb

@@ -8,8 +8,10 @@ require 'tefil/textfilterbase'
 
 require 'tefil/calculator.rb'
 require 'tefil/columnformer'
+require 'tefil/columnanalyzer'
 require 'tefil/indentconverter'
 require 'tefil/indentstatistics'
+require 'tefil/statistics'
 require 'tefil/linesubstituter'
 require 'tefil/percentpacker'
 require 'tefil/zshescaper'

data/lib/tefil/columnanalyzer.rb

@@ -0,0 +1,166 @@
+#! /usr/bin/env ruby
+# coding: utf-8
+
+#INPUT_SEPARATOR = /\s+/
+
+#class String
+#  #http://www.techscore.com/blog/2012/12/25/
+#  #def mb_ljust(width, padding=' ')
+#  def left_just(width, padding=' ')
+#    output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
+#    padding_size = [0, width - output_width].max
+#    self + padding * padding_size
+#  end
+#
+#  #def mb_rjust(width, padding=' ')
+#  def right_just(width, padding=' ')
+#    output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
+#    padding_size = [0, width - output_width].max
+#    padding * padding_size + self
+#  end
+#end
+
+
+#
+#
+#
+class Tefil::ColumnAnalyzer < Tefil::TextFilterBase
+
+  # keys should be array including keys or key=value items)  as like:
+  #   ['1', '2']
+  #   ['1=str1', '2=str2']
+  #   ['1=str1', '2']
+  def initialize(keys = [])
+    @nums_values = {}
+    @keys = []
+    keys.each do |str|
+      if str.include? '='
+        key, value = str.split('=')
+        @nums_values[key.to_i-1] = value
+      else
+        @keys << str
+      end
+    end
+    super({})
+  end
+
+  private
+
+  def process_stream(in_io, out_io)
+    lines = in_io.readlines
+
+    # delete line consist of one character 
+    lines.delete_if {|line| line.split.uniq.size == 1}
+
+    ranges = get_ranges(projection_ary(lines))
+    items_list = lines.map do |line|
+      ranges.map { |range| line[range].strip }
+    end
+
+    # screen items
+    items_head = items_list[0]
+    items_list.select! do |items|
+      flag = true
+      @nums_values.each do |key, value|
+        if items[key] != value
+          flag = false
+          break
+        end
+      end
+      flag
+    end
+
+    # output head
+    results = []
+    results << (1..(items_head.size)).to_a.map{|v| v.to_s}
+    #pp items_list[0]
+    #pp items_head
+    #exit
+    results << items_head unless items_list[0] == items_head
+    results += items_list
+    Tefil::ColumnFormer.new.form(results, out_io)
+
+    out_io.puts
+    out_io.puts "All:       #{lines.size}"
+    out_io.print "Extracted: #{items_list.size}"
+    conditions = []
+    @nums_values.each do |key, val|
+      conditions << "#{key}=#{val}"
+    end
+    out_io.puts " (#{conditions.join(' ')})"
+    out_io.puts
+
+    if items_list.size != 0
+      results = []
+      results << %w(key head types)
+
+      ranges.each_with_index do |range, i|
+        results << [(i+1).to_s, lines[0][range].strip, 
+          items_list.map {|items| items[i]}.sort.uniq.size.to_s
+        ]
+      end
+
+      Tefil::ColumnFormer.new.form(results, out_io)
+    end
+
+    unless @keys.empty?
+      out_io.puts
+      out_io.puts "key analysis"
+      @keys.each do |key|
+        out_io.puts "(key=#{key})"
+        values = items_list.map{|items| items[key.to_i-1]}
+        names = values.sort.uniq
+        results = []
+        names.each do |name|
+          results << [name, values.count(name).to_s]
+        end
+        results.sort_by!{|v| v[1].to_i}
+        Tefil::ColumnFormer.new.form(results, out_io)
+        out_io.puts
+      end
+    end
+  end
+
+  # true の範囲を示す二重配列を返す。
+  # 各要素は 始点..終点 の各インデックスで出来た範囲。
+  ## 各要素は[始点, 終点] の各インデックス。
+  def get_ranges(ary)
+    results = []
+    start = nil
+    prev = false
+    ary << false # for true in final item
+    ary.each_with_index do |cur, i|
+      if prev == false && cur == true
+        start = i
+        prev = cur
+      elsif prev == true && cur == false
+        results << (start..(i - 1))
+        prev = cur
+      else
+        next
+      end
+    end
+    results
+  end
+
+  # 全ての文字列の最大長を要素数とする配列で、
+  # 空白文字以外があれば true, 全て空白文字ならば false にした配列。
+  def projection_ary(lines)
+    max_length = lines.max_by{|line| line.size}.size
+    results = Array.new(max_length).fill(false)
+    lines.each do |line|
+      line.chomp.size.times do |i|
+        c = line[i]
+        next if results[i] == true
+        if c == ' '
+          next
+        else
+          results[i] = true
+        end
+      end
+    end
+    results
+  end
+
+end
+

data/lib/tefil/columnformer.rb

@@ -5,13 +5,15 @@ INPUT_SEPARATOR = /\s+/
 
 class String
   #http://www.techscore.com/blog/2012/12/25/
-  def mb_ljust(width, padding=' ')
+  #def mb_ljust(width, padding=' ')
+  def left_just(width, padding=' ')
     output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
     padding_size = [0, width - output_width].max
     self + padding * padding_size
   end
 
-  def mb_rjust(width, padding=' ')
+  #def mb_rjust(width, padding=' ')
+  def right_just(width, padding=' ')
     output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
     padding_size = [0, width - output_width].max
     padding * padding_size + self
@@ -24,7 +26,14 @@ end
 #
 class Tefil::ColumnFormer < Tefil::TextFilterBase
 
-  def form(matrix, io = $stdout, separator = " ", left = false)
+  def initialize(options = {})
+    @just = options[:just] || :left
+    @separator = options[:separator] || ' '
+    super(options)
+  end
+
+  #def form(matrix, io = $stdout, separator = " ", left = false)
+  def form(matrix, io = $stdout, indent = 0)
     #Obtain max length for each column.
     max_lengths = []
     matrix.each do |row|
@@ -40,11 +49,11 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
     matrix.each do |row|
       new_items = []
       row.each_with_index do |item, index|
-        method = :mb_rjust
-        method = :mb_ljust if left
+        method = (@just.to_s + "_just").to_sym
         new_items[index] = item.send(method, max_lengths[index])
       end
-      io.puts new_items.join(separator).sub(/ +$/, "")
+      io.print(" " * indent)
+      io.puts new_items.join(@separator).sub(/ +$/, "")
     end
   end
 
@@ -52,10 +61,15 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
   private
 
   def process_stream(in_io, out_io)
+    space_width = []
     rows = in_io.readlines.map do |line|
+      #pp line
+      line =~ /^(\s*)/
+      space_width << $1.length
       line.strip.split(INPUT_SEPARATOR)
     end
-    form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
+    #form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
+    form(rows, out_io, space_width.min)
   end
 
   def print_size(string)

data/lib/tefil/fswikitomd.rb

@@ -39,19 +39,19 @@ class Tefil::FswikiToMd < Tefil::TextFilterBase
       end
 
       case
-      when type == :head1   then line.sub!(/^/, "#"           )
-      when type == :head2   then line.sub!(/^/, "##"          )
-      when type == :head3   then line.sub!(/^/, "###"         )
-      when type == :item4   then line.sub!(/^/, "      *"     )
-      when type == :item3   then line.sub!(/^/, "    *"       )
-      when type == :item2   then line.sub!(/^/, "  *"         )
-      when type == :item1   then line.sub!(/^/, "*"           )
-      when type == :enum4   then line.sub!(/^/, "      0."    )
-      when type == :enum3   then line.sub!(/^/, "    0."      )
-      when type == :enum2   then line.sub!(/^/, "  0."        )
-      when type == :enum1   then line.sub!(/^/, "0."          )
-      when type == :pre     then line.sub!(/^/, "    "        )
-      when type == :hline   then line.sub!(/^/, "---"         )
+      when type == :head1   then line.sub!(/^/, "#"              )
+      when type == :head2   then line.sub!(/^/, "##"             )
+      when type == :head3   then line.sub!(/^/, "###"            )
+      when type == :item4   then line.sub!(/^/, "            *"  )
+      when type == :item3   then line.sub!(/^/, "        *"      )
+      when type == :item2   then line.sub!(/^/, "    *"          )
+      when type == :item1   then line.sub!(/^/, "*"              )
+      when type == :enum4   then line.sub!(/^/, "            1.")
+      when type == :enum3   then line.sub!(/^/, "        1."    )
+      when type == :enum2   then line.sub!(/^/, "    1."        )
+      when type == :enum1   then line.sub!(/^/, "1."            )
+      when type == :pre     then line.sub!(/^/, "    "          )
+      when type == :hline   then line.sub!(/^/, "---"           )
       when type == :comment then line = "<!--#{line.chomp}-->"
       else # type == :pain  then  'do nothing'
       end

data/lib/tefil/mdtofswiki.rb

@@ -18,10 +18,10 @@ class Tefil::MdToFswiki < Tefil::TextFilterBase
       when line.sub!(/^    \*/  , '') then type = :item3
       when line.sub!(/^  \*/    , '') then type = :item2
       when line.sub!(/^\*/      , '') then type = :item1
-      when line.sub!(/^      0./, '') then type = :enum4
-      when line.sub!(/^    0./  , '') then type = :enum3
-      when line.sub!(/^  0./    , '') then type = :enum2
-      when line.sub!(/^0./      , '') then type = :enum1
+      when line.sub!(/^      \d+./, '') then type = :enum4
+      when line.sub!(/^    \d+./  , '') then type = :enum3
+      when line.sub!(/^  \d+./    , '') then type = :enum2
+      when line.sub!(/^\d+./      , '') then type = :enum1
       when line.sub!(/^    /    , '') then type = :pre
       when line.sub!(/^---/     , '') then type = :hline
       else                                 type = :plain

data/lib/tefil/statistics.rb

@@ -0,0 +1,38 @@
+class Tefil::Statistics < Tefil::TextFilterBase
+
+  HISTGRAM_LIMIT = 50
+
+  def initialize(options = {})
+    options[:smart_filename] = true
+    @minimum = options[:minimum]
+    super(options)
+  end
+
+  def process_stream(in_io, out_io)
+    data = in_io.readlines.map{|l| l.to_f}
+
+    sum = 0.0
+    data.each { |datum| sum += datum }
+
+    average = sum / data.size.to_f
+
+    #variance
+    tmp = 0.0
+    data.each { |datum| tmp += (datum - average)**2 }
+    variance = tmp / data.size.to_f
+
+    #standard deviation
+    standard_deviation = Math::sqrt(variance)
+
+    out_io.puts "sample:             #{data.size}\n"
+    out_io.puts "highest:            #{data.max}\n"
+    out_io.puts "lowest:             #{data.min}\n"
+    out_io.puts "sum:                #{sum}"
+    out_io.puts "average:            #{average}"
+    out_io.puts "variance:           #{variance}"
+    out_io.puts "standard_deviation: #{standard_deviation}"
+  end
+
+end
+
+

data/lib/tefil/textfilterbase.rb

@@ -53,7 +53,7 @@ class Tefil::TextFilterBase
   #end
 
   #line ごとのファイル名表示はここでは提供せず、したければ process_stream で作る。
-  #grep のように行数を表示するなど、複雑な表示をすることもありうる。
+  #grep の行数表示のように、複雑な表示をすることもありうる。
   def filter(filenames)
     @num_files = filenames.size
     input_io = $stdin

data/tefil.gemspec

@@ -2,19 +2,19 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: tefil 0.1.3 ruby lib
+# stub: tefil 0.1.4 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "tefil"
-  s.version = "0.1.3"
+  s.version = "0.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["ippei94da"]
-  s.date = "2016-05-31"
+  s.date = "2017-05-31"
   s.description = "This gem provides a framework of text filter.\n    Tefil eneable to make text filter commands which have overwrite option easily.\n  "
   s.email = "ippei94da@gmail.com"
-  s.executables = ["calc", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "zshescape"]
+  s.executables = ["calc", "columnanalyze", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "statistics", "zshescape"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/calc",
+    "bin/columnanalyze",
     "bin/columnform",
     "bin/eachsentence",
     "bin/fswiki2md",
@@ -36,8 +37,14 @@ Gem::Specification.new do |s|
     "bin/linesub",
     "bin/md2fswiki",
     "bin/percentpack",
+    "bin/statistics",
     "bin/zshescape",
     "doc/memo.txt",
+    "example/calc/run.zsh",
+    "example/columnanalyze/ps.out",
+    "example/columnanalyze/qstat.out",
+    "example/columnanalyze/run.zsh",
+    "example/columnformer/indent.txt",
     "example/columnformer/sample.txt",
     "example/eachsentence/sample.txt",
     "example/indentconv/sample0.txt",
@@ -49,6 +56,7 @@ Gem::Specification.new do |s|
     "example/zshescape/sample.txt",
     "lib/tefil.rb",
     "lib/tefil/calculator.rb",
+    "lib/tefil/columnanalyzer.rb",
     "lib/tefil/columnformer.rb",
     "lib/tefil/eachsentence.rb",
     "lib/tefil/fswikitomd.rb",
@@ -57,13 +65,16 @@ Gem::Specification.new do |s|
     "lib/tefil/linesubstituter.rb",
     "lib/tefil/mdtofswiki.rb",
     "lib/tefil/percentpacker.rb",
+    "lib/tefil/statistics.rb",
     "lib/tefil/textfilterbase.rb",
     "lib/tefil/zshescaper.rb",
     "tefil.gemspec",
     "test/calc/sample.dat",
     "test/formcolumn_space",
     "test/helper.rb",
+    "test/qstat.out",
     "test/test_calculator.rb",
+    "test/test_columnanalyzer.rb",
     "test/test_columnformer.rb",
     "test/test_eachsentence.rb",
     "test/test_fswikitomd.rb",
@@ -72,6 +83,7 @@ Gem::Specification.new do |s|
     "test/test_linesubstituter.rb",
     "test/test_mdtofswiki.rb",
     "test/test_percentpacker.rb",
+    "test/test_statistics.rb",
     "test/test_textfilterbase.rb",
     "test/test_zshescaper.rb"
   ]

data/test/qstat.out

@@ -0,0 +1,18 @@
+job-ID  prior   name       user         state submit/start at     queue                          slots ja-task-ID 
+-----------------------------------------------------------------------------------------------------------------
+ 292671 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd05.calc.atom                4        
+ 292672 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd12.calc.atom                4        
+ 292673 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd15.calc.atom                4        
+ 292674 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd07.calc.atom                4        
+ 292675 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd11.calc.atom                4        
+ 292676 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd13.calc.atom                4        
+ 292677 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd09.calc.atom                4        
+ 292678 0.25000 vaspdirall ippei        r     05/29/2017 13:55:27 Pd.q@Pd06.calc.atom                4        
+ 292679 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:51                                    4        
+ 292680 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:51                                    4        
+ 292681 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:52                                    4        
+ 292682 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:53                                    4        
+ 292683 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:53                                    4        
+ 292684 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:53                                    4        
+ 292685 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:54                                    4        
+ 292686 0.25000 vaspdirall ippei        qw    05/29/2017 13:54:55                                    4