tefil 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,13 @@
1
+ #! /usr/bin/zsh
2
+
3
+ cat qstat.out | columnanalyze
4
+ echo ------------------------------------------------------------
5
+ cat qstat.out | columnanalyze 4
6
+ echo ------------------------------------------------------------
7
+ cat qstat.out | columnanalyze 4 5
8
+ echo ------------------------------------------------------------
9
+ cat qstat.out | columnanalyze 4=bob
10
+ echo ------------------------------------------------------------
11
+ cat qstat.out | columnanalyze 4=bob 5=r
12
+ echo ------------------------------------------------------------
13
+ cat qstat.out | columnanalyze 4=bob 5=r 3
@@ -0,0 +1,2 @@
1
+ a ab
2
+ abc a
data/lib/tefil.rb CHANGED
@@ -8,8 +8,10 @@ require 'tefil/textfilterbase'
8
8
 
9
9
  require 'tefil/calculator.rb'
10
10
  require 'tefil/columnformer'
11
+ require 'tefil/columnanalyzer'
11
12
  require 'tefil/indentconverter'
12
13
  require 'tefil/indentstatistics'
14
+ require 'tefil/statistics'
13
15
  require 'tefil/linesubstituter'
14
16
  require 'tefil/percentpacker'
15
17
  require 'tefil/zshescaper'
@@ -0,0 +1,166 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ #INPUT_SEPARATOR = /\s+/
5
+
6
+ #class String
7
+ # #http://www.techscore.com/blog/2012/12/25/
8
+ # #def mb_ljust(width, padding=' ')
9
+ # def left_just(width, padding=' ')
10
+ # output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
11
+ # padding_size = [0, width - output_width].max
12
+ # self + padding * padding_size
13
+ # end
14
+ #
15
+ # #def mb_rjust(width, padding=' ')
16
+ # def right_just(width, padding=' ')
17
+ # output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
18
+ # padding_size = [0, width - output_width].max
19
+ # padding * padding_size + self
20
+ # end
21
+ #end
22
+
23
+
24
+ #
25
+ #
26
+ #
27
+ class Tefil::ColumnAnalyzer < Tefil::TextFilterBase
28
+
29
+ # keys should be array including keys or key=value items) as like:
30
+ # ['1', '2']
31
+ # ['1=str1', '2=str2']
32
+ # ['1=str1', '2']
33
+ def initialize(keys = [])
34
+ @nums_values = {}
35
+ @keys = []
36
+ keys.each do |str|
37
+ if str.include? '='
38
+ key, value = str.split('=')
39
+ @nums_values[key.to_i-1] = value
40
+ else
41
+ @keys << str
42
+ end
43
+ end
44
+ super({})
45
+ end
46
+
47
+ private
48
+
49
+ def process_stream(in_io, out_io)
50
+ lines = in_io.readlines
51
+
52
+ # delete line consist of one character
53
+ lines.delete_if {|line| line.split.uniq.size == 1}
54
+
55
+ ranges = get_ranges(projection_ary(lines))
56
+ items_list = lines.map do |line|
57
+ ranges.map { |range| line[range].strip }
58
+ end
59
+
60
+ # screen items
61
+ items_head = items_list[0]
62
+ items_list.select! do |items|
63
+ flag = true
64
+ @nums_values.each do |key, value|
65
+ if items[key] != value
66
+ flag = false
67
+ break
68
+ end
69
+ end
70
+ flag
71
+ end
72
+
73
+ # output head
74
+ results = []
75
+ results << (1..(items_head.size)).to_a.map{|v| v.to_s}
76
+ #pp items_list[0]
77
+ #pp items_head
78
+ #exit
79
+ results << items_head unless items_list[0] == items_head
80
+ results += items_list
81
+ Tefil::ColumnFormer.new.form(results, out_io)
82
+
83
+ out_io.puts
84
+ out_io.puts "All: #{lines.size}"
85
+ out_io.print "Extracted: #{items_list.size}"
86
+ conditions = []
87
+ @nums_values.each do |key, val|
88
+ conditions << "#{key}=#{val}"
89
+ end
90
+ out_io.puts " (#{conditions.join(' ')})"
91
+ out_io.puts
92
+
93
+ if items_list.size != 0
94
+ results = []
95
+ results << %w(key head types)
96
+
97
+ ranges.each_with_index do |range, i|
98
+ results << [(i+1).to_s, lines[0][range].strip,
99
+ items_list.map {|items| items[i]}.sort.uniq.size.to_s
100
+ ]
101
+ end
102
+
103
+ Tefil::ColumnFormer.new.form(results, out_io)
104
+ end
105
+
106
+ unless @keys.empty?
107
+ out_io.puts
108
+ out_io.puts "key analysis"
109
+ @keys.each do |key|
110
+ out_io.puts "(key=#{key})"
111
+ values = items_list.map{|items| items[key.to_i-1]}
112
+ names = values.sort.uniq
113
+ results = []
114
+ names.each do |name|
115
+ results << [name, values.count(name).to_s]
116
+ end
117
+ results.sort_by!{|v| v[1].to_i}
118
+ Tefil::ColumnFormer.new.form(results, out_io)
119
+ out_io.puts
120
+ end
121
+ end
122
+ end
123
+
124
+ # true の範囲を示す二重配列を返す。
125
+ # 各要素は 始点..終点 の各インデックスで出来た範囲。
126
+ ## 各要素は[始点, 終点] の各インデックス。
127
+ def get_ranges(ary)
128
+ results = []
129
+ start = nil
130
+ prev = false
131
+ ary << false # for true in final item
132
+ ary.each_with_index do |cur, i|
133
+ if prev == false && cur == true
134
+ start = i
135
+ prev = cur
136
+ elsif prev == true && cur == false
137
+ results << (start..(i - 1))
138
+ prev = cur
139
+ else
140
+ next
141
+ end
142
+ end
143
+ results
144
+ end
145
+
146
+ # 全ての文字列の最大長を要素数とする配列で、
147
+ # 空白文字以外があれば true, 全て空白文字ならば false にした配列。
148
+ def projection_ary(lines)
149
+ max_length = lines.max_by{|line| line.size}.size
150
+ results = Array.new(max_length).fill(false)
151
+ lines.each do |line|
152
+ line.chomp.size.times do |i|
153
+ c = line[i]
154
+ next if results[i] == true
155
+ if c == ' '
156
+ next
157
+ else
158
+ results[i] = true
159
+ end
160
+ end
161
+ end
162
+ results
163
+ end
164
+
165
+ end
166
+
@@ -5,13 +5,15 @@ INPUT_SEPARATOR = /\s+/
5
5
 
6
6
  class String
7
7
  #http://www.techscore.com/blog/2012/12/25/
8
- def mb_ljust(width, padding=' ')
8
+ #def mb_ljust(width, padding=' ')
9
+ def left_just(width, padding=' ')
9
10
  output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
10
11
  padding_size = [0, width - output_width].max
11
12
  self + padding * padding_size
12
13
  end
13
14
 
14
- def mb_rjust(width, padding=' ')
15
+ #def mb_rjust(width, padding=' ')
16
+ def right_just(width, padding=' ')
15
17
  output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
16
18
  padding_size = [0, width - output_width].max
17
19
  padding * padding_size + self
@@ -24,7 +26,14 @@ end
24
26
  #
25
27
  class Tefil::ColumnFormer < Tefil::TextFilterBase
26
28
 
27
- def form(matrix, io = $stdout, separator = " ", left = false)
29
+ def initialize(options = {})
30
+ @just = options[:just] || :left
31
+ @separator = options[:separator] || ' '
32
+ super(options)
33
+ end
34
+
35
+ #def form(matrix, io = $stdout, separator = " ", left = false)
36
+ def form(matrix, io = $stdout, indent = 0)
28
37
  #Obtain max length for each column.
29
38
  max_lengths = []
30
39
  matrix.each do |row|
@@ -40,11 +49,11 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
40
49
  matrix.each do |row|
41
50
  new_items = []
42
51
  row.each_with_index do |item, index|
43
- method = :mb_rjust
44
- method = :mb_ljust if left
52
+ method = (@just.to_s + "_just").to_sym
45
53
  new_items[index] = item.send(method, max_lengths[index])
46
54
  end
47
- io.puts new_items.join(separator).sub(/ +$/, "")
55
+ io.print(" " * indent)
56
+ io.puts new_items.join(@separator).sub(/ +$/, "")
48
57
  end
49
58
  end
50
59
 
@@ -52,10 +61,15 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
52
61
  private
53
62
 
54
63
  def process_stream(in_io, out_io)
64
+ space_width = []
55
65
  rows = in_io.readlines.map do |line|
66
+ #pp line
67
+ line =~ /^(\s*)/
68
+ space_width << $1.length
56
69
  line.strip.split(INPUT_SEPARATOR)
57
70
  end
58
- form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
71
+ #form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
72
+ form(rows, out_io, space_width.min)
59
73
  end
60
74
 
61
75
  def print_size(string)
@@ -39,19 +39,19 @@ class Tefil::FswikiToMd < Tefil::TextFilterBase
39
39
  end
40
40
 
41
41
  case
42
- when type == :head1 then line.sub!(/^/, "#" )
43
- when type == :head2 then line.sub!(/^/, "##" )
44
- when type == :head3 then line.sub!(/^/, "###" )
45
- when type == :item4 then line.sub!(/^/, " *" )
46
- when type == :item3 then line.sub!(/^/, " *" )
47
- when type == :item2 then line.sub!(/^/, " *" )
48
- when type == :item1 then line.sub!(/^/, "*" )
49
- when type == :enum4 then line.sub!(/^/, " 0." )
50
- when type == :enum3 then line.sub!(/^/, " 0." )
51
- when type == :enum2 then line.sub!(/^/, " 0." )
52
- when type == :enum1 then line.sub!(/^/, "0." )
53
- when type == :pre then line.sub!(/^/, " " )
54
- when type == :hline then line.sub!(/^/, "---" )
42
+ when type == :head1 then line.sub!(/^/, "#" )
43
+ when type == :head2 then line.sub!(/^/, "##" )
44
+ when type == :head3 then line.sub!(/^/, "###" )
45
+ when type == :item4 then line.sub!(/^/, " *" )
46
+ when type == :item3 then line.sub!(/^/, " *" )
47
+ when type == :item2 then line.sub!(/^/, " *" )
48
+ when type == :item1 then line.sub!(/^/, "*" )
49
+ when type == :enum4 then line.sub!(/^/, " 1.")
50
+ when type == :enum3 then line.sub!(/^/, " 1." )
51
+ when type == :enum2 then line.sub!(/^/, " 1." )
52
+ when type == :enum1 then line.sub!(/^/, "1." )
53
+ when type == :pre then line.sub!(/^/, " " )
54
+ when type == :hline then line.sub!(/^/, "---" )
55
55
  when type == :comment then line = "<!--#{line.chomp}-->"
56
56
  else # type == :pain then 'do nothing'
57
57
  end
@@ -18,10 +18,10 @@ class Tefil::MdToFswiki < Tefil::TextFilterBase
18
18
  when line.sub!(/^ \*/ , '') then type = :item3
19
19
  when line.sub!(/^ \*/ , '') then type = :item2
20
20
  when line.sub!(/^\*/ , '') then type = :item1
21
- when line.sub!(/^ 0./, '') then type = :enum4
22
- when line.sub!(/^ 0./ , '') then type = :enum3
23
- when line.sub!(/^ 0./ , '') then type = :enum2
24
- when line.sub!(/^0./ , '') then type = :enum1
21
+ when line.sub!(/^ \d+./, '') then type = :enum4
22
+ when line.sub!(/^ \d+./ , '') then type = :enum3
23
+ when line.sub!(/^ \d+./ , '') then type = :enum2
24
+ when line.sub!(/^\d+./ , '') then type = :enum1
25
25
  when line.sub!(/^ / , '') then type = :pre
26
26
  when line.sub!(/^---/ , '') then type = :hline
27
27
  else type = :plain
@@ -0,0 +1,38 @@
1
+ class Tefil::Statistics < Tefil::TextFilterBase
2
+
3
+ HISTGRAM_LIMIT = 50
4
+
5
+ def initialize(options = {})
6
+ options[:smart_filename] = true
7
+ @minimum = options[:minimum]
8
+ super(options)
9
+ end
10
+
11
+ def process_stream(in_io, out_io)
12
+ data = in_io.readlines.map{|l| l.to_f}
13
+
14
+ sum = 0.0
15
+ data.each { |datum| sum += datum }
16
+
17
+ average = sum / data.size.to_f
18
+
19
+ #variance
20
+ tmp = 0.0
21
+ data.each { |datum| tmp += (datum - average)**2 }
22
+ variance = tmp / data.size.to_f
23
+
24
+ #standard deviation
25
+ standard_deviation = Math::sqrt(variance)
26
+
27
+ out_io.puts "sample: #{data.size}\n"
28
+ out_io.puts "highest: #{data.max}\n"
29
+ out_io.puts "lowest: #{data.min}\n"
30
+ out_io.puts "sum: #{sum}"
31
+ out_io.puts "average: #{average}"
32
+ out_io.puts "variance: #{variance}"
33
+ out_io.puts "standard_deviation: #{standard_deviation}"
34
+ end
35
+
36
+ end
37
+
38
+
@@ -53,7 +53,7 @@ class Tefil::TextFilterBase
53
53
  #end
54
54
 
55
55
  #line ごとのファイル名表示はここでは提供せず、したければ process_stream で作る。
56
- #grep のように行数を表示するなど、複雑な表示をすることもありうる。
56
+ #grep の行数表示のように、複雑な表示をすることもありうる。
57
57
  def filter(filenames)
58
58
  @num_files = filenames.size
59
59
  input_io = $stdin
data/tefil.gemspec CHANGED
@@ -2,19 +2,19 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: tefil 0.1.3 ruby lib
5
+ # stub: tefil 0.1.4 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "tefil"
9
- s.version = "0.1.3"
9
+ s.version = "0.1.4"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib"]
13
13
  s.authors = ["ippei94da"]
14
- s.date = "2016-05-31"
14
+ s.date = "2017-05-31"
15
15
  s.description = "This gem provides a framework of text filter.\n Tefil eneable to make text filter commands which have overwrite option easily.\n "
16
16
  s.email = "ippei94da@gmail.com"
17
- s.executables = ["calc", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "zshescape"]
17
+ s.executables = ["calc", "columnanalyze", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "statistics", "zshescape"]
18
18
  s.extra_rdoc_files = [
19
19
  "LICENSE.txt",
20
20
  "README.rdoc"
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
28
28
  "Rakefile",
29
29
  "VERSION",
30
30
  "bin/calc",
31
+ "bin/columnanalyze",
31
32
  "bin/columnform",
32
33
  "bin/eachsentence",
33
34
  "bin/fswiki2md",
@@ -36,8 +37,14 @@ Gem::Specification.new do |s|
36
37
  "bin/linesub",
37
38
  "bin/md2fswiki",
38
39
  "bin/percentpack",
40
+ "bin/statistics",
39
41
  "bin/zshescape",
40
42
  "doc/memo.txt",
43
+ "example/calc/run.zsh",
44
+ "example/columnanalyze/ps.out",
45
+ "example/columnanalyze/qstat.out",
46
+ "example/columnanalyze/run.zsh",
47
+ "example/columnformer/indent.txt",
41
48
  "example/columnformer/sample.txt",
42
49
  "example/eachsentence/sample.txt",
43
50
  "example/indentconv/sample0.txt",
@@ -49,6 +56,7 @@ Gem::Specification.new do |s|
49
56
  "example/zshescape/sample.txt",
50
57
  "lib/tefil.rb",
51
58
  "lib/tefil/calculator.rb",
59
+ "lib/tefil/columnanalyzer.rb",
52
60
  "lib/tefil/columnformer.rb",
53
61
  "lib/tefil/eachsentence.rb",
54
62
  "lib/tefil/fswikitomd.rb",
@@ -57,13 +65,16 @@ Gem::Specification.new do |s|
57
65
  "lib/tefil/linesubstituter.rb",
58
66
  "lib/tefil/mdtofswiki.rb",
59
67
  "lib/tefil/percentpacker.rb",
68
+ "lib/tefil/statistics.rb",
60
69
  "lib/tefil/textfilterbase.rb",
61
70
  "lib/tefil/zshescaper.rb",
62
71
  "tefil.gemspec",
63
72
  "test/calc/sample.dat",
64
73
  "test/formcolumn_space",
65
74
  "test/helper.rb",
75
+ "test/qstat.out",
66
76
  "test/test_calculator.rb",
77
+ "test/test_columnanalyzer.rb",
67
78
  "test/test_columnformer.rb",
68
79
  "test/test_eachsentence.rb",
69
80
  "test/test_fswikitomd.rb",
@@ -72,6 +83,7 @@ Gem::Specification.new do |s|
72
83
  "test/test_linesubstituter.rb",
73
84
  "test/test_mdtofswiki.rb",
74
85
  "test/test_percentpacker.rb",
86
+ "test/test_statistics.rb",
75
87
  "test/test_textfilterbase.rb",
76
88
  "test/test_zshescaper.rb"
77
89
  ]
data/test/qstat.out ADDED
@@ -0,0 +1,18 @@
1
+ job-ID prior name user state submit/start at queue slots ja-task-ID
2
+ -----------------------------------------------------------------------------------------------------------------
3
+ 292671 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd05.calc.atom 4
4
+ 292672 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd12.calc.atom 4
5
+ 292673 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd15.calc.atom 4
6
+ 292674 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd07.calc.atom 4
7
+ 292675 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd11.calc.atom 4
8
+ 292676 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd13.calc.atom 4
9
+ 292677 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd09.calc.atom 4
10
+ 292678 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd06.calc.atom 4
11
+ 292679 0.25000 vaspdirall ippei qw 05/29/2017 13:54:51 4
12
+ 292680 0.25000 vaspdirall ippei qw 05/29/2017 13:54:51 4
13
+ 292681 0.25000 vaspdirall ippei qw 05/29/2017 13:54:52 4
14
+ 292682 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
15
+ 292683 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
16
+ 292684 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
17
+ 292685 0.25000 vaspdirall ippei qw 05/29/2017 13:54:54 4
18
+ 292686 0.25000 vaspdirall ippei qw 05/29/2017 13:54:55 4