trace_visualization 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12c27504ea6bbbabd78f47e5bee3df251a5a7050
-  data.tar.gz: 75a0bf55e52a108f449def1998de4fbf319d1f94
+  metadata.gz: 2fee2d1c5c6920d3966ae5190b05bcb40ce2b359
+  data.tar.gz: c8cb276ef0bfef1d293b996bd236c4d1cdc6f1b7
 SHA512:
-  metadata.gz: f6fff098106b2ba908595a05345bebafc5686348454061983671c937c21a26ecb0cee59a8d6209203707949e7283daba5d620b7d386cd1ac230e021668f4a7f5
-  data.tar.gz: 6e39264b5f0141746d7fda75d4c2c2be624a0829c12ed548160e6bd9f74afac162b06a6e9e670155ef042f7b7baf65eb0bff43a42236051f0aec81b15842495c
+  metadata.gz: 9dc0447760f9d5b95f27ec88e95bde2beb73037f36cb3af6d6aa4a324d17afa348e7462735e9a14e78aefdf9c779bb446beaaca5a249af32e467d4b14695f32f
+  data.tar.gz: d9b3c4ce306cd563ab394104fec2e85654e8bc55e8592a64ff7367a8269b7b33635f16412eba65af404c1eb2b790f9986c820a6fbb0aa483bbfbf155774e590b

data/Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 ruby '2.0.0'
 
-gem 'trace_preprocessor', "~> 0.0.2"
+gem 'trace_preprocessor'
 
 # Specify your gem's dependencies in trace_visualization.gemspec
 gemspec

data/bin/trace_visualization

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'trace_visualization'
+require 'trace_preprocessor'
 require 'tempfile'
 require 'benchmark'
 require 'optparse'
@@ -39,6 +40,11 @@ def parse_options
     opts.on('-t', '--test_preprocessor_performance', 'Test preprocessor performance') do
       options[:test_preprocessor_performance] = true
     end
+    
+    opts.on('-v', '--version', 'Display version') do
+      puts "Version #{VERSION}"
+      exit
+    end
 
     opts.on('-h', '--help', 'Display this screen') do
       puts opts
@@ -75,8 +81,10 @@ def foo(options)
   
   mapping = nil
   bm('Mapping') do
-    mapping = TraceVisualization::Mapping.new
-    mapping.process_without_reorder { from_preprocessed_file(tmp_file.path, options[:offset], options[:limit], !options[:test_preprocessor_performance]) }
+    mapping = Mapping.new
+    mapping.process do
+      from_preprocessed_file(tmp_file.path, options[:offset], options[:limit], !options[:test_preprocessor_performance])
+    end
   end
 
   source_file_length = `wc -c #{options[:source_file_name]}`.to_i
@@ -94,40 +102,27 @@ def foo(options)
     exit 0
   end
 
-  rs = nil
-  bm('PSY1') do
-    rs = Repetitions.psy1(mapping, options[:positions_min_size], true)
-  end
-
-  puts "rs.size = #{rs.size}"
+  repetitions = Algorithm.process(mapping, {})
 
-  #context = nil
-  #bm('Context') do
-  #  context = TraceVisualization::Repetitions::Context.new(mapping, rs)
-  #end
+  RepetitionsScore.fill_score(repetitions, :sort => true, :order => 'desc', :version => 'relative')
 
-  RepetitionsScore.fill_score(rs, :sort => true, :order => 'desc', :version => 'relative')
+  # 
+  # rs[0 ... options[:out]].each do |r|
+  #   puts '---------------------'
+  #   puts r
+  #   puts Visualization::ConsoleColorPrint.hl(mapping, r)
+  # end
 
-  rs[0 ... options[:out]].each do |r|
-    puts '---------------------'
-    puts r
-    puts Visualization::ConsoleColorPrint.hl(mapping, r)
-  end
+  Visualization::ConsoleColorPrint.hl_stdout(mapping, repetitions[0])
 
-  rs
+  repetitions
 end
 
-
 def bm(name, &block)
   Benchmark.bm(17) do |x|
     x.report(name, &block)
   end
 end
 
-
 options = parse_options
-rs = foo(options)
-
-
-
-
+rs = foo(options)

data/lib/trace_visualization.rb

@@ -5,7 +5,7 @@
   reorder
   repetitions
   repetitions_psy
-  repetitions_context
+  repetitions/context
   repetitions_score
   visualization/console_color_print
 ).each { |file| require File.join(File.dirname(__FILE__), 'trace_visualization', file) }
@@ -34,7 +34,7 @@ module TraceVisualization
     # Read & mapping file
     mapping = TraceVisualization::Mapping.new
     mapping.process do
-      from_preprocessed_file options[:file_name]
+      from_file options[:file_name]
     end
 
 =begin

data/lib/trace_visualization/algorithm.rb

@@ -0,0 +1,42 @@
+require 'trace_visualization/utils'
+require 'trace_visualization/assert'
+require 'trace_visualization/repetitions/filter'
+require 'trace_visualization/repetitions/concatenation'
+
+module TraceVisualization
+  
+  # Main algorithm
+  module Algorithm
+    include TraceVisualization
+    include TraceVisualization::Repetitions    
+    
+    # Run main algorithm of approximate repeats processing
+    # @param mapping [TraceVisualization::Mapping] 
+    # @param options [Hash]
+    def self.process(mapping, options = {})
+      TraceVisualization.assert_instance_of(mapping, Mapping)
+      
+      Utils.set_default_options(options, { :positions_min_size => 3, :k => 3 })
+
+      context = Context.new(mapping)
+
+      # First step - calculate strict repetitions
+      context.repetitions = Repetitions.psy1(mapping, options[:positions_min_size])
+      
+      # Second step - iterative process concatenation & incrementation algorithms for approximate repeats processing
+      Filter.strict_repetitions_filter(mapping, context, options)
+      
+      for kk in 1 .. options[:k]
+        Concatenation.process(context, { 
+          :k => kk,
+          :positions_min_size => options[:positions_min_size]
+        })
+        
+        Filter.repetitions_filter(mapping, context, options)
+      end
+      
+      context.repetitions
+    end
+    
+  end # Algorithm
+end # TraceVisualization

data/lib/trace_visualization/data/irepetition.rb

@@ -1,3 +1,7 @@
+require 'trace_visualization/data/sorted_array'
+
+include TraceVisualization::Data
+
 # Interface for Repetition 
 module IRepetition
   def positions_size; end
@@ -31,7 +35,7 @@ class RepetitionBase
   attr_accessor :lines
 
   def initialize
-    @lines = []
+    @lines = SortedArray.new
   end
   
   def left_length

data/lib/trace_visualization/data/repetition.rb

@@ -5,11 +5,13 @@ module TraceVisualization
   module Data
     class Repetition < RepetitionBase
       @@sequence = 0
-  
+
+      attr_reader :id  
+      attr_reader :left_positions, :right_positions
+      
       attr_accessor :score
       attr_accessor :pcount
-      attr_reader :left_positions, :right_positions
-      attr_reader :id
+
       attr_accessor :strict_ids
   
       def to_s
@@ -23,7 +25,7 @@ module TraceVisualization
         "left = #{@left != nil ? @left.id : 'nil'}, " +
         "right = #{@right != nil ? @right.id : 'nil'}"
       end
-  
+
       def initialize(length, left_positions, right_positions = nil)
         super()
     
@@ -35,10 +37,16 @@ module TraceVisualization
         @right_positions = right_positions 
     
         @id = (@@sequence += 1)
-        @strict_ids = [@id]
-        @lines = []
+        @strict_ids = [ @id ]
       end
   
+      def ==(other_repetition)
+        other_repetition != nil && 
+          @length == other_repetition.length && 
+          @left_positions == other_repetition.left_positions && 
+          @right_positions == other_repetition.right_positions
+      end
+
       def positions_size
         @left_positions.size
       end

data/lib/trace_visualization/data/sorted_array.rb

@@ -0,0 +1,44 @@
+module TraceVisualization
+  module Data
+    class SortedArray < Array
+      def self.[] *array
+        SortedArray.new(array)
+      end
+
+      def initialize(array = nil)
+        super( array.sort ) if array
+      end
+
+      def << value
+        insert index_of_last_LE(value), value
+      end
+
+      alias push <<
+      alias shift <<
+
+      def index_of_last_LE value
+        l, r = 0, length - 1
+        while l <= r
+          m = (r + l) / 2
+          
+          if value < self[m]
+            r = m - 1
+          else
+            l = m + 1
+          end
+        end
+        
+        l
+      end
+      
+      def index value
+        idx = index_of_last_LE(value)
+        
+        if idx != 0 && self[idx - 1] == value
+          idx -= 1 while idx != 0 && self[idx - 1] == value
+          idx
+        end
+      end
+    end
+  end
+end

data/lib/trace_visualization/data/{lexeme.rb → token.rb}

@@ -1,12 +1,12 @@
 module TraceVisualization
   module Data
-    class Lexeme
+    class Token
       attr_accessor :name
       attr_accessor :value
       attr_accessor :int_value
 
-      # Length of preprocessed string (see LEXEME_REGEXP)
-      attr_accessor :lexeme_length
+      # Length of preprocessed string (see TOKEN_REGEXP)
+      attr_accessor :token_length
       
       attr_accessor :ord
   
@@ -47,12 +47,11 @@ module TraceVisualization
       end
     end
     
-    class LexemePos
-      attr_accessor :lexeme
-      attr_accessor :pos
+    class TokenPosition
+      attr_accessor :token, :pos
       
-      def initialize(lexeme, pos)
-        @lexeme, @pos = lexeme, pos
+      def initialize(token, pos)
+        @token, @pos = token, pos
       end
     end
   end

data/lib/trace_visualization/lexeme_overlap_filter.rb

@@ -1,10 +1,10 @@
 module TraceVisualization
   module LexemeOverlapFilter
     
-    def self.process(lexeme_poss)
-      lexeme_poss.sort! { |a, b| a.pos <=> b.pos }
-      left_bound = lexeme_poss.inject(0) do |left_bound, lexeme_pos| 
-        [lexeme_pos.pos + lexeme_pos.lexeme.length, left_bound].max 
+    def self.process(token_positions)
+      token_positions.sort! { |a, b| a.pos <=> b.pos }
+      left_bound = token_positions.inject(0) do |left_bound, token_pos| 
+        [token_pos.pos + token_pos.token.length, left_bound].max 
       end
     
       idx, current, result = 0, [], []
@@ -13,24 +13,24 @@ module TraceVisualization
   
         i = 0
         while i < current.size
-          lexeme_pos = current[i]
+          token_pos = current[i]
     
-          if lexeme_pos.pos + lexeme_pos.lexeme.length == pos
-            fl_delete_lexeme = false
+          if token_pos.pos + token_pos.token.length == pos
+            fl_delete_token = false
           
             if current.size == 1
-              result << lexeme_pos
-              fl_delete_lexeme = true
+              result << token_pos
+              fl_delete_token = true
             else
-              if is_longest_lexeme(current, lexeme_pos)
-                result << lexeme_pos
+              if is_longest_token(current, token_pos)
+                result << token_pos
                 current = []
               else
-                fl_delete_lexeme = true
+                fl_delete_token = true
               end
             end
       
-            if fl_delete_lexeme
+            if fl_delete_token
               current.delete_at(i)
               i -= 1
             end      
@@ -39,8 +39,8 @@ module TraceVisualization
           i += 1
         end
 
-        while idx < lexeme_poss.size && lexeme_poss[idx].pos == pos
-          current << lexeme_poss[idx] 
+        while idx < token_positions.size && token_positions[idx].pos == pos
+          current << token_positions[idx] 
           idx += 1
         end
       end
@@ -48,11 +48,11 @@ module TraceVisualization
       result
     end
   
-    def self.is_longest_lexeme(lexeme_poss, lexeme_pos)
+    def self.is_longest_token(token_positions, token_pos)
       result = true
 
-      lexeme_poss.each do |other_lexeme_pos|
-        if lexeme_pos != other_lexeme_pos && lexeme_pos.lexeme.length < other_lexeme_pos.lexeme.length
+      token_positions.each do |other_token_pos|
+        if token_pos != other_token_pos && token_pos.token.length < other_token_pos.token.length
           result = false
           break
         end

data/lib/trace_visualization/mapping.rb

@@ -3,38 +3,18 @@ require 'ipaddr'
 
 require 'trace_visualization/reorder'
 require 'trace_visualization/lexeme_overlap_filter'
-require 'trace_visualization/data/lexeme'
+require 'trace_visualization/data/token'
+
+include TraceVisualization
 
 module TraceVisualization
   class Mapping
-    
-    attr_accessor :tokens
-  
-    LEXEME_REGEXP = /\{LEXEME;(?<name>[a-zA-Z0-9]+);(?<source>[^;]+);(?<value>[0-9]+)\}/
-    
-    DEFAULT_TOKENS = {
-      :ID => [ 
-        /(?<lexeme>\[\d{3,}\])/, 
-        lambda { |source| source[1 ... -1].to_i } 
-      ],
-      
-      :IP => [
-        /(?<lexeme>(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))/,
-        lambda { |source| IPAddr.new(source).to_i }
-      ],
-        
-      :TIME => [
-        /(?<lexeme>\[\d{2} [a-zA-Z]{3} \d{4} \d{2}\:\d{2}\:\d{2}\])/,
-        lambda { |source| Time.parse(source[1 ... -1]).to_i }
-      ]      
-    }
+    TOKEN_REGEXP = /\{TOKEN;(?<name>[a-zA-Z0-9]+);(?<source>[^;]+);(?<value>[0-9]+);(?<kind>[0-9]+)\}/
     
     def initialize
-      # hash map: lexeme-object by lexeme-string
-      @lexeme_table = {}
-      
-      @tokens = {}
-      @mapped_str = []
+      @tokens_map = {}
+      @data       = []
+      @lines      = []
     end
     
     def self.init(&block)
@@ -42,163 +22,112 @@ module TraceVisualization
       mapping.instance_eval(&block)
       mapping
     end
-
-    # new    
-    def token(name, pattern, converter_func)
-      @tokens[name] = [pattern, converter_func]
-    end
-    
-    def default_tokens
-      @tokens.merge!(DEFAULT_TOKENS)
-    end
     
-    # new
+    # Process input data with reorder
     def process(&block)
       instance_eval(&block)
-      @max_value = TraceVisualization::Reorder.process(@mapped_str)
+      @max_value = Reorder.process(@data)
     end
 
+    # Process input data without reorder
     def process_without_reorder(&block)
       instance_eval(&block)
     end
 
-    ##
-    # Load data from source file. File is read line by line
-
-    def from_file(path)
-      validate_file_name_argument(path)
-
-      open(path) do |fd|
-        while (line = fd.gets)
-          process_line(line)
-        end
-      end
-    end
-
-    ##
     # Load data from preprocessed file. File is read line by line
-
-    def from_preprocessed_file(path, offset, limit, use_lexeme_table = true)
+    def from_file(path, offset = 0, limit = 2**32, use_lexeme_table = true)
       validate_file_name_argument(path)
 
       idx = 0
       open(path) do |fd|
         while (line = fd.gets)
-          process_preprocessed_line(line, use_lexeme_table) if (idx >= offset && idx < offset + limit)
+          line.chomp!
+          if idx >= offset && idx < offset + limit
+            process_line(line, use_lexeme_table) 
+          elsif idx >= offset + limit
+            break
+          end
 
           idx += 1
         end
       end
-    end
-
-    # new
-    def from_string(str)
-      validate_string_argument(str)
       
-      @str = str
-
-      str.split("\n").each do |line|
-        process_line(line)
-      end
-
-      @mapped_str.pop if @mapped_str[-1].value == "\n" && str[-1] != "\n"
+      @data.pop
     end
 
-    def from_preprocessed_string(str, use_lexeme_table = true)
+    # Load data from string
+    def from_string(str, use_lexeme_table = true)
       validate_string_argument(str)
 
       str.split("\n").each do |line|
-        process_preprocessed_line(line)
+        process_line(line)
       end
+      
+      @data.pop
     end
     
-    def process_preprocessed_line(line, use_lexeme_table = true)
-      lexeme_positions = []
+    # Process line
+    def process_line(line, use_lexeme_table = true)
+      @lines << @data.length
+      
+      token_positions = []
       pos = 0
-      while (m = LEXEME_REGEXP.match(line, pos))
+      while (m = TOKEN_REGEXP.match(line, pos))
         pos = m.begin(0)
 
-        lexeme = install_lexeme_m(m, use_lexeme_table)
-        lexeme_positions << TraceVisualization::Data::LexemePos.new(lexeme, pos)
-
-        pos += lexeme.lexeme_length
-      end
-
+        token = install_token_m(m, use_lexeme_table)
+        token_positions << Data::TokenPosition.new(token, pos)
 
-      pos, idx = 0, 0
-      while pos < line.length
-        lexeme = nil
-        if idx < lexeme_positions.size && lexeme_positions[idx].pos == pos
-          lexeme = lexeme_positions[idx].lexeme
-          idx += 1
-        else
-          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1, use_lexeme_table)
-        end
-        pos += lexeme.lexeme_length
-        @mapped_str << lexeme
+        pos += token.token_length
       end
 
-      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1, use_lexeme_table)
-    end
-    
-    # new
-    def process_line(line)
-      lexeme_poss = []
 
-      @tokens.each do |name, value|
-        pattern, converter_func = value
-        pos = 0
-        while (m = pattern.match(line, pos))
-          lexeme_string, pos = m[:lexeme], m.begin(0)
-          
-          lexeme = install_lexeme(name, lexeme_string, converter_func.call(lexeme_string), lexeme_string.length)
-          lexeme_poss << TraceVisualization::Data::LexemePos.new(lexeme, pos)
-          
-          pos += lexeme_string.length
-        end
-      end
-      
-      lexeme_poss = TraceVisualization::LexemeOverlapFilter.process(lexeme_poss)
-      
       pos, idx = 0, 0
       while pos < line.length
-        lexeme = nil
-        if idx < lexeme_poss.size && lexeme_poss[idx].pos == pos
-          lexeme = lexeme_poss[idx].lexeme
+        token = nil
+        if idx < token_positions.size && token_positions[idx].pos == pos
+          token = token_positions[idx].token
           idx += 1
         else
-          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1)
+          token = install_token('CHAR', line[pos], line[pos].ord, 1, use_lexeme_table)
         end
-        pos += lexeme.length
-        @mapped_str << lexeme
+        pos += token.token_length
+        @data << token
       end
       
-      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1)
+      @data << install_lf_token
     end
     
-    def install_lexeme(name, lexeme_string, int_value, lexeme_length, use_lexeme_table = true)
-      lexeme = use_lexeme_table ? @lexeme_table[lexeme_string] : nil
+    def install_token(name, token_string, int_value, token_length, use_token_table = true)
+      name = name.intern
       
-      if lexeme.nil?
-        lexeme = TraceVisualization::Data::Lexeme.new(name, lexeme_string, int_value)
-        lexeme.lexeme_length = lexeme_length
+      token = use_token_table ? @tokens_map[token_string] : nil
+      
+      if token.nil?
+        token = Data::Token.new(name, token_string, int_value)
+        token.token_length = token_length
 
-        @lexeme_table[lexeme_string] = lexeme if use_lexeme_table
+        @tokens_map[token_string] = token if use_token_table
       end
       
-      lexeme
+      token
     end
 
-    def install_lexeme_m(m, use_lexeme_table = true)
-      install_lexeme(m[:name], m[:source], m[:value].to_i, m.to_s.length, use_lexeme_table)
+    def install_token_m(m, use_lexeme_table = true)
+      install_token(m[:name], m[:source], m[:value].to_i, m.to_s.length, use_lexeme_table)
+    end
+    
+    # Install token with LF (line feed or end of line)
+    def install_lf_token
+      install_token('CHAR', "\n", 0x0A, 1)
     end
     
     def [](index)
-      @mapped_str[index]
+      @data[index]
     end
     
     def length
-      @mapped_str.length
+      @data.length
     end
     
     def size
@@ -206,13 +135,13 @@ module TraceVisualization
     end
     
     def <<(object)
-      lexeme = install_lexeme('UNKNOWN', object, object.to_i, object.to_s.length)
+      lexeme = install_token('UNKNOWN', object, object.to_i, object.to_s.length)
       lexeme.ord = 0 # fake ord because Reorder already processed
-      @mapped_str << lexeme
+      @data << lexeme
     end
     
     def pop
-      @mapped_str.pop
+      @data.pop
     end
     
     def max
@@ -220,15 +149,23 @@ module TraceVisualization
     end
     
     def find_all
-      @mapped_str.find_all { |item| yield(item) }
+      @data.find_all { |item| yield(item) }
     end
 
-    def restore(pos = 0, length = @mapped_str.length)
-      @mapped_str[pos ... pos + length].inject("") { |res, c| res += c.value }
+    # Restore source data
+    # @param pos [Integer] begin position
+    # @param length [Integer] the length of restore part 
+    def restore(pos = 0, length = @data.length)
+      @data[pos ... pos + length].inject("") { |res, c| res += c.value }
     end
     
     def to_ary
-      @mapped_str.collect { |lexeme| lexeme.value }
+      @data.collect { |lexeme| lexeme.value }
+    end
+    
+    # @return array of lines positions
+    def lines
+      @lines
     end
     
     def method_missing(name, *args, &blk)
@@ -236,57 +173,7 @@ module TraceVisualization
     end
 
     private
-
-    def parse(str)
-      map = {}
-      ppos = []
-      itemByPos = {}
-    
-      DEFAULT_TOKENS.each do |type, patterns|
-        patterns.each do |pattern|
-          match(str, type, pattern, map, ppos, itemByPos)
-        end
-      end
-    
-      i, j = 0, 0
-      result = []
-    
-      ppos.sort!
-    
-      while i < str.size
-        if i == ppos[j]
-          item = itemByPos[ppos[j]]
-          result << item
-          i, j = i + item.length, j + 1
-        else
-          result << Item.new(str[i], "char")
-          i += 1
-        end        
-      end
-      
-      @max_value = TraceVisualization::Reorder.process(result)
-      
-      result
-    end
      
-    def match(str, type, pattern, map, ppos, itemByPos)
-      pos = 0
-      
-      limit = 1000
-      
-      while (m = pattern.match(str, pos))
-        value = m[:value]
-        pos = m.begin(0)
-        ppos << pos
-      
-        map[value] = Item.new(value, type) unless map[value]
-        itemByPos[pos] = map[value]
-        
-        pos += value.size
-      end
-      
-    end
-
     def validate_file_name_argument(path)
       raise ArgumentError, 'Argument must be a string' if not path.instance_of? String
       raise ArgumentError, 'File path is not defined' if path.empty?