tefil 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ #! /usr/bin/zsh
2
+
3
+ cat qstat.out | columnanalyze
4
+ echo ------------------------------------------------------------
5
+ cat qstat.out | columnanalyze 4
6
+ echo ------------------------------------------------------------
7
+ cat qstat.out | columnanalyze 4 5
8
+ echo ------------------------------------------------------------
9
+ cat qstat.out | columnanalyze 4=bob
10
+ echo ------------------------------------------------------------
11
+ cat qstat.out | columnanalyze 4=bob 5=r
12
+ echo ------------------------------------------------------------
13
+ cat qstat.out | columnanalyze 4=bob 5=r 3
@@ -0,0 +1,2 @@
1
+ a ab
2
+ abc a
data/lib/tefil.rb CHANGED
@@ -8,8 +8,10 @@ require 'tefil/textfilterbase'
8
8
 
9
9
  require 'tefil/calculator.rb'
10
10
  require 'tefil/columnformer'
11
+ require 'tefil/columnanalyzer'
11
12
  require 'tefil/indentconverter'
12
13
  require 'tefil/indentstatistics'
14
+ require 'tefil/statistics'
13
15
  require 'tefil/linesubstituter'
14
16
  require 'tefil/percentpacker'
15
17
  require 'tefil/zshescaper'
@@ -0,0 +1,166 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ #INPUT_SEPARATOR = /\s+/
5
+
6
+ #class String
7
+ # #http://www.techscore.com/blog/2012/12/25/
8
+ # #def mb_ljust(width, padding=' ')
9
+ # def left_just(width, padding=' ')
10
+ # output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
11
+ # padding_size = [0, width - output_width].max
12
+ # self + padding * padding_size
13
+ # end
14
+ #
15
+ # #def mb_rjust(width, padding=' ')
16
+ # def right_just(width, padding=' ')
17
+ # output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
18
+ # padding_size = [0, width - output_width].max
19
+ # padding * padding_size + self
20
+ # end
21
+ #end
22
+
23
+
24
+ #
25
+ #
26
+ #
27
+ class Tefil::ColumnAnalyzer < Tefil::TextFilterBase
28
+
29
+ # keys should be array including keys or key=value items) as like:
30
+ # ['1', '2']
31
+ # ['1=str1', '2=str2']
32
+ # ['1=str1', '2']
33
+ def initialize(keys = [])
34
+ @nums_values = {}
35
+ @keys = []
36
+ keys.each do |str|
37
+ if str.include? '='
38
+ key, value = str.split('=')
39
+ @nums_values[key.to_i-1] = value
40
+ else
41
+ @keys << str
42
+ end
43
+ end
44
+ super({})
45
+ end
46
+
47
+ private
48
+
49
+ def process_stream(in_io, out_io)
50
+ lines = in_io.readlines
51
+
52
+ # delete line consist of one character
53
+ lines.delete_if {|line| line.split.uniq.size == 1}
54
+
55
+ ranges = get_ranges(projection_ary(lines))
56
+ items_list = lines.map do |line|
57
+ ranges.map { |range| line[range].strip }
58
+ end
59
+
60
+ # screen items
61
+ items_head = items_list[0]
62
+ items_list.select! do |items|
63
+ flag = true
64
+ @nums_values.each do |key, value|
65
+ if items[key] != value
66
+ flag = false
67
+ break
68
+ end
69
+ end
70
+ flag
71
+ end
72
+
73
+ # output head
74
+ results = []
75
+ results << (1..(items_head.size)).to_a.map{|v| v.to_s}
76
+ #pp items_list[0]
77
+ #pp items_head
78
+ #exit
79
+ results << items_head unless items_list[0] == items_head
80
+ results += items_list
81
+ Tefil::ColumnFormer.new.form(results, out_io)
82
+
83
+ out_io.puts
84
+ out_io.puts "All: #{lines.size}"
85
+ out_io.print "Extracted: #{items_list.size}"
86
+ conditions = []
87
+ @nums_values.each do |key, val|
88
+ conditions << "#{key}=#{val}"
89
+ end
90
+ out_io.puts " (#{conditions.join(' ')})"
91
+ out_io.puts
92
+
93
+ if items_list.size != 0
94
+ results = []
95
+ results << %w(key head types)
96
+
97
+ ranges.each_with_index do |range, i|
98
+ results << [(i+1).to_s, lines[0][range].strip,
99
+ items_list.map {|items| items[i]}.sort.uniq.size.to_s
100
+ ]
101
+ end
102
+
103
+ Tefil::ColumnFormer.new.form(results, out_io)
104
+ end
105
+
106
+ unless @keys.empty?
107
+ out_io.puts
108
+ out_io.puts "key analysis"
109
+ @keys.each do |key|
110
+ out_io.puts "(key=#{key})"
111
+ values = items_list.map{|items| items[key.to_i-1]}
112
+ names = values.sort.uniq
113
+ results = []
114
+ names.each do |name|
115
+ results << [name, values.count(name).to_s]
116
+ end
117
+ results.sort_by!{|v| v[1].to_i}
118
+ Tefil::ColumnFormer.new.form(results, out_io)
119
+ out_io.puts
120
+ end
121
+ end
122
+ end
123
+
124
+ # true の範囲を示す二重配列を返す。
125
+ # 各要素は 始点..終点 の各インデックスで出来た範囲。
126
+ ## 各要素は[始点, 終点] の各インデックス。
127
+ def get_ranges(ary)
128
+ results = []
129
+ start = nil
130
+ prev = false
131
+ ary << false # for true in final item
132
+ ary.each_with_index do |cur, i|
133
+ if prev == false && cur == true
134
+ start = i
135
+ prev = cur
136
+ elsif prev == true && cur == false
137
+ results << (start..(i - 1))
138
+ prev = cur
139
+ else
140
+ next
141
+ end
142
+ end
143
+ results
144
+ end
145
+
146
+ # 全ての文字列の最大長を要素数とする配列で、
147
+ # 空白文字以外があれば true, 全て空白文字ならば false にした配列。
148
+ def projection_ary(lines)
149
+ max_length = lines.max_by{|line| line.size}.size
150
+ results = Array.new(max_length).fill(false)
151
+ lines.each do |line|
152
+ line.chomp.size.times do |i|
153
+ c = line[i]
154
+ next if results[i] == true
155
+ if c == ' '
156
+ next
157
+ else
158
+ results[i] = true
159
+ end
160
+ end
161
+ end
162
+ results
163
+ end
164
+
165
+ end
166
+
@@ -5,13 +5,15 @@ INPUT_SEPARATOR = /\s+/
5
5
 
6
6
  class String
7
7
  #http://www.techscore.com/blog/2012/12/25/
8
- def mb_ljust(width, padding=' ')
8
+ #def mb_ljust(width, padding=' ')
9
+ def left_just(width, padding=' ')
9
10
  output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
10
11
  padding_size = [0, width - output_width].max
11
12
  self + padding * padding_size
12
13
  end
13
14
 
14
- def mb_rjust(width, padding=' ')
15
+ #def mb_rjust(width, padding=' ')
16
+ def right_just(width, padding=' ')
15
17
  output_width = each_char.map{|c| c.bytesize == 1 ? 1 : 2}.reduce(0, &:+)
16
18
  padding_size = [0, width - output_width].max
17
19
  padding * padding_size + self
@@ -24,7 +26,14 @@ end
24
26
  #
25
27
  class Tefil::ColumnFormer < Tefil::TextFilterBase
26
28
 
27
- def form(matrix, io = $stdout, separator = " ", left = false)
29
+ def initialize(options = {})
30
+ @just = options[:just] || :left
31
+ @separator = options[:separator] || ' '
32
+ super(options)
33
+ end
34
+
35
+ #def form(matrix, io = $stdout, separator = " ", left = false)
36
+ def form(matrix, io = $stdout, indent = 0)
28
37
  #Obtain max length for each column.
29
38
  max_lengths = []
30
39
  matrix.each do |row|
@@ -40,11 +49,11 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
40
49
  matrix.each do |row|
41
50
  new_items = []
42
51
  row.each_with_index do |item, index|
43
- method = :mb_rjust
44
- method = :mb_ljust if left
52
+ method = (@just.to_s + "_just").to_sym
45
53
  new_items[index] = item.send(method, max_lengths[index])
46
54
  end
47
- io.puts new_items.join(separator).sub(/ +$/, "")
55
+ io.print(" " * indent)
56
+ io.puts new_items.join(@separator).sub(/ +$/, "")
48
57
  end
49
58
  end
50
59
 
@@ -52,10 +61,15 @@ class Tefil::ColumnFormer < Tefil::TextFilterBase
52
61
  private
53
62
 
54
63
  def process_stream(in_io, out_io)
64
+ space_width = []
55
65
  rows = in_io.readlines.map do |line|
66
+ #pp line
67
+ line =~ /^(\s*)/
68
+ space_width << $1.length
56
69
  line.strip.split(INPUT_SEPARATOR)
57
70
  end
58
- form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
71
+ #form(rows, out_io, OPTIONS[:separator], OPTIONS[:left])
72
+ form(rows, out_io, space_width.min)
59
73
  end
60
74
 
61
75
  def print_size(string)
@@ -39,19 +39,19 @@ class Tefil::FswikiToMd < Tefil::TextFilterBase
39
39
  end
40
40
 
41
41
  case
42
- when type == :head1 then line.sub!(/^/, "#" )
43
- when type == :head2 then line.sub!(/^/, "##" )
44
- when type == :head3 then line.sub!(/^/, "###" )
45
- when type == :item4 then line.sub!(/^/, " *" )
46
- when type == :item3 then line.sub!(/^/, " *" )
47
- when type == :item2 then line.sub!(/^/, " *" )
48
- when type == :item1 then line.sub!(/^/, "*" )
49
- when type == :enum4 then line.sub!(/^/, " 0." )
50
- when type == :enum3 then line.sub!(/^/, " 0." )
51
- when type == :enum2 then line.sub!(/^/, " 0." )
52
- when type == :enum1 then line.sub!(/^/, "0." )
53
- when type == :pre then line.sub!(/^/, " " )
54
- when type == :hline then line.sub!(/^/, "---" )
42
+ when type == :head1 then line.sub!(/^/, "#" )
43
+ when type == :head2 then line.sub!(/^/, "##" )
44
+ when type == :head3 then line.sub!(/^/, "###" )
45
+ when type == :item4 then line.sub!(/^/, " *" )
46
+ when type == :item3 then line.sub!(/^/, " *" )
47
+ when type == :item2 then line.sub!(/^/, " *" )
48
+ when type == :item1 then line.sub!(/^/, "*" )
49
+ when type == :enum4 then line.sub!(/^/, " 1.")
50
+ when type == :enum3 then line.sub!(/^/, " 1." )
51
+ when type == :enum2 then line.sub!(/^/, " 1." )
52
+ when type == :enum1 then line.sub!(/^/, "1." )
53
+ when type == :pre then line.sub!(/^/, " " )
54
+ when type == :hline then line.sub!(/^/, "---" )
55
55
  when type == :comment then line = "<!--#{line.chomp}-->"
56
56
  else # type == :pain then 'do nothing'
57
57
  end
@@ -18,10 +18,10 @@ class Tefil::MdToFswiki < Tefil::TextFilterBase
18
18
  when line.sub!(/^ \*/ , '') then type = :item3
19
19
  when line.sub!(/^ \*/ , '') then type = :item2
20
20
  when line.sub!(/^\*/ , '') then type = :item1
21
- when line.sub!(/^ 0./, '') then type = :enum4
22
- when line.sub!(/^ 0./ , '') then type = :enum3
23
- when line.sub!(/^ 0./ , '') then type = :enum2
24
- when line.sub!(/^0./ , '') then type = :enum1
21
+ when line.sub!(/^ \d+./, '') then type = :enum4
22
+ when line.sub!(/^ \d+./ , '') then type = :enum3
23
+ when line.sub!(/^ \d+./ , '') then type = :enum2
24
+ when line.sub!(/^\d+./ , '') then type = :enum1
25
25
  when line.sub!(/^ / , '') then type = :pre
26
26
  when line.sub!(/^---/ , '') then type = :hline
27
27
  else type = :plain
@@ -0,0 +1,38 @@
1
+ class Tefil::Statistics < Tefil::TextFilterBase
2
+
3
+ HISTGRAM_LIMIT = 50
4
+
5
+ def initialize(options = {})
6
+ options[:smart_filename] = true
7
+ @minimum = options[:minimum]
8
+ super(options)
9
+ end
10
+
11
+ def process_stream(in_io, out_io)
12
+ data = in_io.readlines.map{|l| l.to_f}
13
+
14
+ sum = 0.0
15
+ data.each { |datum| sum += datum }
16
+
17
+ average = sum / data.size.to_f
18
+
19
+ #variance
20
+ tmp = 0.0
21
+ data.each { |datum| tmp += (datum - average)**2 }
22
+ variance = tmp / data.size.to_f
23
+
24
+ #standard deviation
25
+ standard_deviation = Math::sqrt(variance)
26
+
27
+ out_io.puts "sample: #{data.size}\n"
28
+ out_io.puts "highest: #{data.max}\n"
29
+ out_io.puts "lowest: #{data.min}\n"
30
+ out_io.puts "sum: #{sum}"
31
+ out_io.puts "average: #{average}"
32
+ out_io.puts "variance: #{variance}"
33
+ out_io.puts "standard_deviation: #{standard_deviation}"
34
+ end
35
+
36
+ end
37
+
38
+
@@ -53,7 +53,7 @@ class Tefil::TextFilterBase
53
53
  #end
54
54
 
55
55
  #line ごとのファイル名表示はここでは提供せず、したければ process_stream で作る。
56
- #grep のように行数を表示するなど、複雑な表示をすることもありうる。
56
+ #grep の行数表示のように、複雑な表示をすることもありうる。
57
57
  def filter(filenames)
58
58
  @num_files = filenames.size
59
59
  input_io = $stdin
data/tefil.gemspec CHANGED
@@ -2,19 +2,19 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: tefil 0.1.3 ruby lib
5
+ # stub: tefil 0.1.4 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "tefil"
9
- s.version = "0.1.3"
9
+ s.version = "0.1.4"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib"]
13
13
  s.authors = ["ippei94da"]
14
- s.date = "2016-05-31"
14
+ s.date = "2017-05-31"
15
15
  s.description = "This gem provides a framework of text filter.\n Tefil eneable to make text filter commands which have overwrite option easily.\n "
16
16
  s.email = "ippei94da@gmail.com"
17
- s.executables = ["calc", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "zshescape"]
17
+ s.executables = ["calc", "columnanalyze", "columnform", "eachsentence", "fswiki2md", "indentconv", "indentstat", "linesub", "md2fswiki", "percentpack", "statistics", "zshescape"]
18
18
  s.extra_rdoc_files = [
19
19
  "LICENSE.txt",
20
20
  "README.rdoc"
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
28
28
  "Rakefile",
29
29
  "VERSION",
30
30
  "bin/calc",
31
+ "bin/columnanalyze",
31
32
  "bin/columnform",
32
33
  "bin/eachsentence",
33
34
  "bin/fswiki2md",
@@ -36,8 +37,14 @@ Gem::Specification.new do |s|
36
37
  "bin/linesub",
37
38
  "bin/md2fswiki",
38
39
  "bin/percentpack",
40
+ "bin/statistics",
39
41
  "bin/zshescape",
40
42
  "doc/memo.txt",
43
+ "example/calc/run.zsh",
44
+ "example/columnanalyze/ps.out",
45
+ "example/columnanalyze/qstat.out",
46
+ "example/columnanalyze/run.zsh",
47
+ "example/columnformer/indent.txt",
41
48
  "example/columnformer/sample.txt",
42
49
  "example/eachsentence/sample.txt",
43
50
  "example/indentconv/sample0.txt",
@@ -49,6 +56,7 @@ Gem::Specification.new do |s|
49
56
  "example/zshescape/sample.txt",
50
57
  "lib/tefil.rb",
51
58
  "lib/tefil/calculator.rb",
59
+ "lib/tefil/columnanalyzer.rb",
52
60
  "lib/tefil/columnformer.rb",
53
61
  "lib/tefil/eachsentence.rb",
54
62
  "lib/tefil/fswikitomd.rb",
@@ -57,13 +65,16 @@ Gem::Specification.new do |s|
57
65
  "lib/tefil/linesubstituter.rb",
58
66
  "lib/tefil/mdtofswiki.rb",
59
67
  "lib/tefil/percentpacker.rb",
68
+ "lib/tefil/statistics.rb",
60
69
  "lib/tefil/textfilterbase.rb",
61
70
  "lib/tefil/zshescaper.rb",
62
71
  "tefil.gemspec",
63
72
  "test/calc/sample.dat",
64
73
  "test/formcolumn_space",
65
74
  "test/helper.rb",
75
+ "test/qstat.out",
66
76
  "test/test_calculator.rb",
77
+ "test/test_columnanalyzer.rb",
67
78
  "test/test_columnformer.rb",
68
79
  "test/test_eachsentence.rb",
69
80
  "test/test_fswikitomd.rb",
@@ -72,6 +83,7 @@ Gem::Specification.new do |s|
72
83
  "test/test_linesubstituter.rb",
73
84
  "test/test_mdtofswiki.rb",
74
85
  "test/test_percentpacker.rb",
86
+ "test/test_statistics.rb",
75
87
  "test/test_textfilterbase.rb",
76
88
  "test/test_zshescaper.rb"
77
89
  ]
data/test/qstat.out ADDED
@@ -0,0 +1,18 @@
1
+ job-ID prior name user state submit/start at queue slots ja-task-ID
2
+ -----------------------------------------------------------------------------------------------------------------
3
+ 292671 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd05.calc.atom 4
4
+ 292672 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd12.calc.atom 4
5
+ 292673 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd15.calc.atom 4
6
+ 292674 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd07.calc.atom 4
7
+ 292675 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd11.calc.atom 4
8
+ 292676 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd13.calc.atom 4
9
+ 292677 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd09.calc.atom 4
10
+ 292678 0.25000 vaspdirall ippei r 05/29/2017 13:55:27 Pd.q@Pd06.calc.atom 4
11
+ 292679 0.25000 vaspdirall ippei qw 05/29/2017 13:54:51 4
12
+ 292680 0.25000 vaspdirall ippei qw 05/29/2017 13:54:51 4
13
+ 292681 0.25000 vaspdirall ippei qw 05/29/2017 13:54:52 4
14
+ 292682 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
15
+ 292683 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
16
+ 292684 0.25000 vaspdirall ippei qw 05/29/2017 13:54:53 4
17
+ 292685 0.25000 vaspdirall ippei qw 05/29/2017 13:54:54 4
18
+ 292686 0.25000 vaspdirall ippei qw 05/29/2017 13:54:55 4