trace_visualization 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5779b63579f93699f091c22ada7121df9469fc7
-  data.tar.gz: 336cfdc696d598e2247d09b089ec2b426df18e58
+  metadata.gz: 12c27504ea6bbbabd78f47e5bee3df251a5a7050
+  data.tar.gz: 75a0bf55e52a108f449def1998de4fbf319d1f94
 SHA512:
-  metadata.gz: 583564ac99432f9a0b05e1a51550497ae2f18d30fad51a7443fd8f9e6930567a66cac481a48c9cd594483a35eccdb4eff852a644bc8ce896fe7bffa14c3562fa
-  data.tar.gz: 721cf7cdf88fe2ac0380f4eb9517ffda3e6846cba7d317e6c5f1cdfac453c288b40b2b7d674de2039a0a2e88aab0a2b33b89dcde0d379a3b0c674b59fb5bb688
+  metadata.gz: f6fff098106b2ba908595a05345bebafc5686348454061983671c937c21a26ecb0cee59a8d6209203707949e7283daba5d620b7d386cd1ac230e021668f4a7f5
+  data.tar.gz: 6e39264b5f0141746d7fda75d4c2c2be624a0829c12ed548160e6bd9f74afac162b06a6e9e670155ef042f7b7baf65eb0bff43a42236051f0aec81b15842495c

data/Gemfile

@@ -2,6 +2,8 @@ source 'https://rubygems.org'
 
 ruby '2.0.0'
 
+gem 'trace_preprocessor', "~> 0.0.2"
+
 # Specify your gem's dependencies in trace_visualization.gemspec
 gemspec
 

data/Rakefile

@@ -1,9 +1,9 @@
 require 'bundler/gem_tasks'
 
-desc 'Compile preprocessor'
-task :install do
-  system('cd lib/trace_visualization/preprocessor; make install; make clean')
-end
+#desc 'Compile preprocessor'
+#task :install do
+#  system('cd ext/preprocessor; make install')
+#end
 
 desc 'Spec all functionality of gem'
 task :spec_all do

data/bin/trace_visualization

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'trace_visualization'
+require 'tempfile'
+require 'benchmark'
+require 'optparse'
+
+include TraceVisualization
+
+##
+# Parse command line options
+def parse_options
+  options = {}
+
+  optionParser = OptionParser.new do|opts|
+    opts.banner = "Usage: trace_visualization [options] file"
+
+    options[:offset] = 0
+    opts.on('-x', '--offset N', 'Number of lines for offset. Default 0') do |n|
+      options[:offset] = n.to_i
+    end
+
+    options[:limit] = 2**32
+    opts.on('-l', '--limit N', "Number of lines. Default #{options[:limit]}") do |n|
+      options[:limit] = n.to_i
+    end
+
+    options[:out] = 3
+    opts.on('-o', '--out N', 'Number of output results') do |n|
+      options[:out] = n.to_i
+    end
+
+    options[:positions_min_size] = 3
+    opts.on('-m', '--positions_min_size N', "The minimum number of repetition positions. Default #{options[:positions_min_size]}") do |s|
+      options[:positions_min_size] = s.to_i
+    end
+
+    options[:test_preprocessor_performance] = false
+    opts.on('-t', '--test_preprocessor_performance', 'Test preprocessor performance') do
+      options[:test_preprocessor_performance] = true
+    end
+
+    opts.on('-h', '--help', 'Display this screen') do
+      puts opts
+      exit
+    end
+  end
+
+  optionParser.parse!
+
+  options[:source_file_name] = ARGV[0]
+
+  options
+end
+
+#------------------------------------------------------------------------------
+def foo(options)
+  
+  preprocessorConfig = TracePreprocessor.init do
+    use_default_lexemes
+    
+    output_token "printf(\"{TOKEN;%s;%s;%ld;%d}\", name, source, value, value_kind);"
+  end
+
+  preprocessor = nil
+  bm('Preprocessor Generate') do  
+    preprocessor = TracePreprocessor.generate(preprocessorConfig, :c)
+  end
+  
+  tmp_file = Tempfile.new('trace_visualization')
+  
+  bm('Preprocessor work') do  
+    preprocessor.run options[:source_file_name], tmp_file.path
+  end
+  
+  mapping = nil
+  bm('Mapping') do
+    mapping = TraceVisualization::Mapping.new
+    mapping.process_without_reorder { from_preprocessed_file(tmp_file.path, options[:offset], options[:limit], !options[:test_preprocessor_performance]) }
+  end
+
+  source_file_length = `wc -c #{options[:source_file_name]}`.to_i
+  tmp_file_length = tmp_file.length
+
+  puts "source_file_length = #{source_file_length}"
+  puts "tmp_file_length    = #{tmp_file_length} (+#{((tmp_file_length - source_file_length) * 100.0 / source_file_length).round(2)}%)"
+  puts "mapping_length     = #{mapping.length} (-#{((source_file_length - mapping.length) * 100.0 / source_file_length).round(2)}%)"
+
+  tmp_file.close
+  tmp_file.unlink
+
+  if options[:test_preprocessor_performance]
+    puts "Finish preprocessor"
+    exit 0
+  end
+
+  rs = nil
+  bm('PSY1') do
+    rs = Repetitions.psy1(mapping, options[:positions_min_size], true)
+  end
+
+  puts "rs.size = #{rs.size}"
+
+  #context = nil
+  #bm('Context') do
+  #  context = TraceVisualization::Repetitions::Context.new(mapping, rs)
+  #end
+
+  RepetitionsScore.fill_score(rs, :sort => true, :order => 'desc', :version => 'relative')
+
+  rs[0 ... options[:out]].each do |r|
+    puts '---------------------'
+    puts r
+    puts Visualization::ConsoleColorPrint.hl(mapping, r)
+  end
+
+  rs
+end
+
+
+def bm(name, &block)
+  Benchmark.bm(17) do |x|
+    x.report(name, &block)
+  end
+end
+
+
+options = parse_options
+rs = foo(options)
+
+
+
+

data/lib/trace_visualization.rb

@@ -2,6 +2,12 @@
   mapping
   utils
   version
+  reorder
+  repetitions
+  repetitions_psy
+  repetitions_context
+  repetitions_score
+  visualization/console_color_print
 ).each { |file| require File.join(File.dirname(__FILE__), 'trace_visualization', file) }
 
 require 'logger'

data/lib/trace_visualization/mapping.rb

@@ -58,61 +58,73 @@ module TraceVisualization
       @max_value = TraceVisualization::Reorder.process(@mapped_str)
     end
 
+    def process_without_reorder(&block)
+      instance_eval(&block)
+    end
+
+    ##
     # Load data from source file. File is read line by line
+
     def from_file(path)
-      raise ArgumentError, 'Argument must be a string' unless path.instance_of? String 
-      raise ArgumentError, 'File path is not defined'  unless path.empty?
-      raise RuntimeError,  'File doesn\'t exists'      unless File.exists?(path)
-      
-      fd = open(path)
-      process_line(line) while line = fd.gets
-      fd.close
+      validate_file_name_argument(path)
+
+      open(path) do |fd|
+        while (line = fd.gets)
+          process_line(line)
+        end
+      end
     end
 
+    ##
     # Load data from preprocessed file. File is read line by line
-    def from_preprocessed_file(path)
-      raise ArgumentError, 'Argument must be a string' unless path.instance_of? String
-      raise ArgumentError, 'File path is not defined'  unless path.empty?
-      raise RuntimeError,  'File doesn\'t exists'      unless File.exists?(path)
 
-      fd = open(path)
-      process_preprocessed_line line while line = fd.gets
-      fd.close
+    def from_preprocessed_file(path, offset, limit, use_lexeme_table = true)
+      validate_file_name_argument(path)
+
+      idx = 0
+      open(path) do |fd|
+        while (line = fd.gets)
+          process_preprocessed_line(line, use_lexeme_table) if (idx >= offset && idx < offset + limit)
+
+          idx += 1
+        end
+      end
     end
 
     # new
     def from_string(str)
-      raise ArgumentError, 'Argument must be a string' unless str.instance_of? String 
-      raise ArgumentError, 'String is not defined'     if str.empty?
+      validate_string_argument(str)
       
       @str = str
 
       str.split("\n").each do |line|
         process_line(line)
       end
+
+      @mapped_str.pop if @mapped_str[-1].value == "\n" && str[-1] != "\n"
     end
-    
-    def from_preprocessed_string(str)
-      raise ArgumentError, 'Argument must be a string' if not str.instance_of? String 
-      raise ArgumentError, 'String is not defined'     if str.empty?
+
+    def from_preprocessed_string(str, use_lexeme_table = true)
+      validate_string_argument(str)
 
       str.split("\n").each do |line|
         process_preprocessed_line(line)
       end
     end
     
-    def process_preprocessed_line(line)
+    def process_preprocessed_line(line, use_lexeme_table = true)
       lexeme_positions = []
       pos = 0
       while (m = LEXEME_REGEXP.match(line, pos))
         pos = m.begin(0)
-        
-        lexeme = install_lexeme_m(m)
+
+        lexeme = install_lexeme_m(m, use_lexeme_table)
         lexeme_positions << TraceVisualization::Data::LexemePos.new(lexeme, pos)
 
         pos += lexeme.lexeme_length
       end
-      
+
+
       pos, idx = 0, 0
       while pos < line.length
         lexeme = nil
@@ -120,13 +132,13 @@ module TraceVisualization
           lexeme = lexeme_positions[idx].lexeme
           idx += 1
         else
-          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1)
+          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1, use_lexeme_table)
         end
         pos += lexeme.lexeme_length
         @mapped_str << lexeme
       end
-      
-      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1)
+
+      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1, use_lexeme_table)
     end
     
     # new
@@ -164,28 +176,21 @@ module TraceVisualization
       @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1)
     end
     
-    def install_lexeme(name, lexeme_string, int_value, lexeme_length)
-      lexeme = @lexeme_table[lexeme_string]
+    def install_lexeme(name, lexeme_string, int_value, lexeme_length, use_lexeme_table = true)
+      lexeme = use_lexeme_table ? @lexeme_table[lexeme_string] : nil
       
-      if lexeme == nil
+      if lexeme.nil?
         lexeme = TraceVisualization::Data::Lexeme.new(name, lexeme_string, int_value)
         lexeme.lexeme_length = lexeme_length
-        @lexeme_table[lexeme_string] = lexeme
+
+        @lexeme_table[lexeme_string] = lexeme if use_lexeme_table
       end
       
       lexeme
     end
 
-    def install_lexeme_m(m)
-      lexeme = @lexeme_table[m[:source]]
-
-      if lexeme == nil
-        lexeme = TraceVisualization::Data::Lexeme.new(m[:name], m[:source], m[:value].to_i)
-        lexeme.lexeme_length = m.to_s.length
-        @lexeme_table[m[:source]] = lexeme
-      end
-
-      lexeme
+    def install_lexeme_m(m, use_lexeme_table = true)
+      install_lexeme(m[:name], m[:source], m[:value].to_i, m.to_s.length, use_lexeme_table)
     end
     
     def [](index)
@@ -201,7 +206,9 @@ module TraceVisualization
     end
     
     def <<(object)
-      @mapped_str << Item.new(object, "unknown")
+      lexeme = install_lexeme('UNKNOWN', object, object.to_i, object.to_s.length)
+      lexeme.ord = 0 # fake ord because Reorder already processed
+      @mapped_str << lexeme
     end
     
     def pop
@@ -279,5 +286,17 @@ module TraceVisualization
       end
       
     end
+
+    def validate_file_name_argument(path)
+      raise ArgumentError, 'Argument must be a string' if not path.instance_of? String
+      raise ArgumentError, 'File path is not defined' if path.empty?
+      raise RuntimeError, 'File doesn\'t exists' if not File.exists?(path)
+    end
+
+    def validate_string_argument(str)
+      raise ArgumentError, 'Argument must be a string' unless str.instance_of? String
+      raise ArgumentError, 'String is not defined' if str.empty?
+    end
+
   end
 end

data/lib/trace_visualization/reorder.rb

@@ -1,3 +1,5 @@
+require 'trace_visualization'
+
 module TraceVisualization
   module Reorder
 

data/lib/trace_visualization/repetitions_score.rb

@@ -0,0 +1,87 @@
+module TraceVisualization
+  module RepetitionsScore
+
+    # Priority length when repetition score is calculated
+    ALPHA_SCORE = 0.5
+
+    # Priority positions.size when repetition score is calculated
+    BETA_SCORE = 1 - ALPHA_SCORE
+
+    # Options:
+    # *sort* true / false
+    # *order* order for sort (default 'asc')
+    # *version* version of importance function
+    def self.fill_score(rs, options = {})
+
+      opts = {
+          :sort => false,
+          :order => 'asc'
+      }.merge options
+
+      case opts[:version]
+        when 'relative'
+          function_relative(rs)
+        when 'linear'
+          function_linear(rs)
+        else
+          throw Exception.new("Unknown version")
+      end
+
+      if opts[:sort]
+        rs.sort! do |a, b|
+          opts[:order] == 'desc' ? b.score <=> a.score : a.score <=> b.score
+        end
+      end
+    end
+
+    # f(len, size) = alpha * len' + beta * size'
+    # len', size' - relative to max values
+    def self.function_relative(rs)
+      len_max, size_max = 0, 0
+
+      rs.each do |r|
+        len = r.length
+        len_max = len if len > len_max
+
+        size = r.positions_size
+        size_max = size if size > size_max
+      end
+
+      rs.each do |r|
+        r.score = ALPHA_SCORE * r.strict_length / len_max +
+            BETA_SCORE * r.positions_size / size_max
+      end
+    end
+
+    # f(len, size, k) = alpha * len + beta * size + gamma * k
+    def self.function_linear(rs)
+      len_max, len_min, size_max, size_min, k_max = 0, 2**32, 0, 2**32, 0
+
+      rs.each do |r|
+        len = r.length
+        len_max = len if len > len_max
+        len_min = len if len < len_min
+
+        size = r.positions_size
+        size_max = size if size > size_max
+        size_min = size if size < size_min
+
+        k_max = r.k if r.k > k_max
+      end
+
+      d = (len_max * size_min + size_max * k_max - size_max * len_min - len_max * k_max).to_f
+
+      alpha =  (size_min + k_max * (size_max - 1)) / d
+      beta  = - (len_min + k_max * (len_max - 1)) / d
+      gamma = (len_max * size_min - size_max * len_min - size_min + len_min) / d
+
+      puts "len_max = #{len_max}, len_min = #{len_min}, size_max = #{size_max}, size_min = #{size_min}, k_max = #{k_max}"
+      puts "alpha = #{alpha}, beta = #{beta}, gamma = #{gamma}"
+
+      rs.each do |r|
+        r.score = alpha * r.length + beta * r.positions_size + gamma * r.k
+      end
+    end
+
+  end
+end