trace_visualization 0.0.1

data/lib/trace_visualization/longest_common_prefix.rb

@@ -0,0 +1,34 @@
+module TraceVisualization
+  module LongestCommonPrefix
+    
+    # A linear-time algorithm to compute the longest common prefix information
+    # in suffix arrays from article: "Linear-Time Longest-Common-Prefix 
+    # Computation in Suffix Arrays and Its Applications" Toru Kasai et al.
+    # 
+    # The method signature and variable names are stored under the specified 
+    # work without changes.
+    def self.effective(a, pos, n)
+      rank = Array.new(n, 0)
+      height = Array.new(n, 0)
+    
+      for i in 0 ... n
+        rank[pos[i]] = i
+      end
+    
+      h = 0
+      for i in 0 ... n
+        if (rank[i] > 0)
+          j = pos[rank[i] - 1]
+          while a[i + h] == a[j + h]
+            h += 1
+          end
+          height[rank[i]] = h
+          h -= 1 if h > 0
+        end
+      end
+    
+      height
+    end
+    
+  end
+end

data/lib/trace_visualization/mapping.rb

@@ -0,0 +1,120 @@
+module TraceVisualization
+  module Mapping
+    require 'time'
+    require 'ipaddr'
+
+    require 'trace_visualization/reorder'
+    
+    PATTERNS = {
+    
+      "id" => [
+        /(?<value>\[\d{3,}\])/
+      ],
+
+      "ip" => [
+        /(?<value>(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))/
+      ],
+    
+      "time" => [
+        /(?<value>\[\d{2} [a-zA-Z]{3} \d{4} \d{2}\:\d{2}\:\d{2}\])/
+      ]
+
+    }
+    
+    class Item
+      include Comparable
+  
+      attr_reader :value # integer value for comparison
+      attr_reader :src   # source value 
+      attr_reader :type  # source type
+
+      attr_accessor :ord # re-order value
+  
+      def initialize(src, type)
+        @src  = src
+        @type = type
+    
+        case type
+        when "id"
+          @value = @src[1 ... -1].to_i
+        when "ip"
+          @value = IPAddr.new(src).to_i
+        when "time"
+          @value = Time.parse(@src[1 ... -1]).to_i
+        when "char"
+          @value = src.getbyte(0)
+        else
+          raise Exception.new("unknown type")
+        end
+      end
+  
+      def length
+        @src.length
+      end
+  
+      def <=>(anOther)
+        @ord <=> anOther.ord
+      end
+      
+      def to_str
+        @src
+      end
+    end
+
+    def self.parse(str)
+      map = {}
+      ppos = []
+      itemByPos = {}
+    
+      PATTERNS.each do |type, patterns|
+        patterns.each do |pattern|
+          match(str, type, pattern, map, ppos, itemByPos)
+        end
+      end
+    
+      i, j = 0, 0
+      result = []
+    
+      ppos.sort!
+    
+      while i < str.size
+        if i == ppos[j]
+          item = itemByPos[ppos[j]]
+          result << item
+          i += item.length
+          j += 1
+        else
+          result << Item.new(str[i], "char")
+          i += 1
+        end        
+      end
+      
+      TraceVisualization::Reorder.process(result)
+      
+      result
+    end
+     
+    def self.match(str, type, pattern, map, ppos, itemByPos)
+      pos = 0
+      
+      limit = 1000
+      
+      while (m = pattern.match(str, pos))
+        value = m[:value]
+        pos = m.begin(0)
+        ppos << pos
+      
+        map[value] = Item.new(value, type) unless map[value]
+        itemByPos[pos] = map[value]
+        
+        pos += value.size
+      end
+      
+    end
+
+    def self.restore(array)
+      array.inject("") { |res, c| res += c.to_str }
+    end
+    
+  end
+end

data/lib/trace_visualization/reorder.rb

@@ -0,0 +1,25 @@
+module TraceVisualization
+  module Reorder
+
+    # Assign new values (ord field) in order to reduce the distance between min
+    # and max values. It's necessary to reduce the size of the alphabet.
+    def self.process(data)
+      sorted = data.sort do |a, b|
+        c = a.value - b.value
+        c == 0 ? 0 : (c < 0 ? -1 : 1) 
+      end
+      
+      idx = 0
+      prev = nil
+      
+      sorted.each do |item|
+        if prev != item.value
+          prev = item.value
+          idx += 1
+        end
+
+        item.ord = idx
+      end
+    end
+  end
+end

data/lib/trace_visualization/repetitions.rb

@@ -0,0 +1,66 @@
+module TraceVisualization
+  module Repetitions
+
+    # The naive approach to finding repetitions in the string.
+    # Time complexity: О(n^2)
+    def self.naive(str, l)
+      result = Array.new(l + 1) { [] }
+      result[0] << Array.new(str.length) {|idx| idx + 1 }
+    
+      for i in 1 .. l
+        result[i - 1].each do |item|
+          counter = char_counter_inc(str, item, i - 1)
+          counter.each do |positions|
+            result[i] << positions if positions != nil && positions.size > 1
+          end
+        end
+      end
+    
+      result[l].sort
+    end
+    
+    # The naive approach to finding repetitions in the string.
+    # Time complexity: О(n^2)
+    def self.naive_all(str, p_min)
+      result = []
+    
+      result << []
+      result[0] << Array.new(str.length) {|idx| idx + 1 }
+    
+      idx = 1
+      while true
+        result << []
+      
+        result[idx - 1].each do |item|
+          counter = char_counter_inc(str, item, idx - 1)
+          counter.each do |positions|
+            if positions != nil && positions.size > 1
+              result[idx] << positions
+              result[idx - 1].delete_if { |item| item == positions }
+            end
+          end
+        end
+      
+        break if result.last.size == 0
+      
+        idx += 1
+      end
+    
+      result[p_min ... -1] || []
+    end    
+    
+    def self.char_counter_inc(str, pos, offset)
+      counter = Array.new(256)
+    
+      pos.each do |pos|
+        if pos + offset - 1 < str.length
+          c = str[pos + offset - 1].ord
+          counter[c] = [] if counter[c] == nil
+          counter[c] << pos        
+        end
+      end
+    
+      counter
+    end
+  end
+end

data/lib/trace_visualization/repetitions_concatenation.rb

@@ -0,0 +1,134 @@
+require 'trace_visualization'
+
+module TraceVisualization
+  module RepetitionsConcatenation
+
+    def self.process(rs, k, context, options = {})
+      opts = {
+        :positions_min_size => 3
+      }.merge options
+  
+      result = []
+  
+      useful_cnt = 0
+
+      pairs_cnt = {}
+      context.rs_by_line.each do |item|
+        for i in 0 ... item.size
+          for j in i + 1 ... item.size
+            left, right = item[i][0], item[j][0]
+            delta = k - left.k - right.k
+
+            next if not concat_condition(left, right, delta, opts[:positions_min_size])
+
+            key = (left.id << 32) + right.id
+            val = (pairs_cnt[key] || 0) + 1
+            pairs_cnt[key] = val
+            next if val != opts[:positions_min_size]
+
+            lps, rps = process_common_positions(left, right, delta, context)
+      
+            if lps.size >= opts[:positions_min_size]
+              result << create_repetition(left, right, delta, lps, rps)
+            end
+      
+            useful_cnt += 1        
+          end
+        end
+      end
+  
+      options[:counter] << [k, useful_cnt] if options[:counter]
+  
+      puts "Total: #{rs.size ** 2} #{useful_cnt} #{result.size}"
+  
+      process_new_repetitions(result)
+  
+      rs.concat(result)
+    end
+
+    def self.process_new_repetitions(rs, context)
+      TraceVisualization::Utils.rs_by_line(rs, context.lines_pos, context.rs_by_line)
+    end
+
+    def self.process_full_search(rs, k, context, options = {})
+      opts = {
+        :positions_min_size => 3
+      }.merge options
+  
+      result = []
+  
+      useful_cnt = 0
+
+      for left in rs
+        for right in rs
+          delta = k - left.k - right.k
+          next if not concat_condition(left, right, delta, opts[:positions_min_size])
+      
+          # @@processed_path.add(key(left, right, delta))  
+      
+          lps, rps = process_common_positions(left, right, delta, context)
+      
+          if lps.size >= opts[:positions_min_size]
+            result << create_repetition(left, right, delta, lps, rps)
+          end
+      
+          useful_cnt += 1
+        end
+      end
+
+      puts "Total: #{rs.size ** 2} #{useful_cnt} #{result.size}"
+  
+      rs.concat(result)
+    end
+
+    def self.delete_repetition(r, context)
+      r.lines.each do |line|
+        context.rs_by_line[line].delete_if { |item| item[0] == r }
+      end
+    end
+
+    def self.concat_condition(left, right, delta, positions_min_size)
+      delta >= 0 && left.id != right.id && 
+        left.positions_size  >= positions_min_size && 
+        right.positions_size >= positions_min_size
+    end
+
+    # *Attention* Position arrays are modified in place which can lead to side 
+    # effects. Don't send left == right!
+    def self.process_common_positions(left, right, delta, context)
+      lr_pos = left.left_positions
+      lr_pos.collect! { |pos| pos + left.length + delta }
+
+      rr_pos = right.left_positions
+  
+      cpr = lr_pos & rr_pos
+      cpl = cpr.collect { |pos| pos - left.length - delta }
+  
+      idx = 0
+      while idx < cpr.size
+        if context.str[cpl[idx] + left.length ... cpr[idx]].scan(TraceVisualization::FORBIDDEN_CHARS).size != 0
+          cpr.delete_at(idx)
+          cpl.delete_at(idx)
+        else
+          idx += 1
+        end
+      end
+
+      lr_pos.collect! { |lpos| lpos - left.length - delta }
+      
+      [cpl, cpr]
+    end
+
+    def self.create_repetition(left, right, delta, lps, rps)
+      r = left.class.new(left.length + right.length + delta, lps, rps)
+
+      r.k          = left.k + right.k + delta
+      r.pcount     = left.pcount + right.pcount
+      r.left       = left
+      r.right      = right
+      r.strict_ids = left.strict_ids + right.strict_ids
+
+      r
+    end
+  end
+end

data/lib/trace_visualization/repetitions_context.rb

@@ -0,0 +1,18 @@
+require 'trace_visualization/utils'
+
+module TraceVisualization
+  module Repetitions
+    class Context
+      attr_accessor :str, :lines_pos, :rs_by_line
+      
+      def initialize(str, rs)
+        @str        = str
+        @lines_pos  = TraceVisualization::Utils.lines_pos(str)
+        @rs_by_line = Array.new(@lines_pos.size) { [] }
+    
+        TraceVisualization::Utils.rs_by_line(rs, @lines_pos, @rs_by_line)
+      end
+      
+    end
+  end
+end

data/lib/trace_visualization/repetitions_incrementation.rb

@@ -0,0 +1,81 @@
+require 'trace_visualization/utils'
+
+module TraceVisualization
+  module RepetitionsIncrementation
+  
+    #--------------------------------------------------------------------------
+    def self.incrementation(str, repetitions, hashes, k)
+      result = []
+
+      for core in repetitions
+        next if core.k != 0
+      
+        # Positions for Right Incrementation
+        positions_ri = []
+      
+        # Positions for Left Incrementation
+        positions_li = []
+      
+        for r in repetitions
+          next if r == core || core.length + k != r.length
+        
+          clps = core.left_positions
+          rlps = r.left_positions
+        
+          common_positions_ri = clps & rlps
+          common_positions_li = clps.collect { |p| p - k } & rlps
+                              
+          positions_ri |= common_positions_ri if common_positions_ri.size > 2
+          positions_li |= common_positions_li if common_positions_li.size > 2        
+        end
+      
+        if positions_ri.size > 1
+          left_positions  = positions_ri.sort
+          right_positions = left_positions.collect { |pos| pos + core.length + k}
+        
+          hash = TraceVisualization::Utils.rhash(left_positions, right_positions)
+        
+          if (not hashes.include? hash)
+            hashes << hash
+            result << create_repetition(core, k, left_positions, right_positions, "right")
+          end
+        end
+      
+        if positions_li.size > 1
+          left_positions  = positions_li.sort
+          right_positions = left_positions.collect { |pos| pos + k}
+        
+          hash = TraceVisualization::Utils.rhash(left_positions, right_positions)
+        
+          if (not hashes.include? hash)
+            hashes << hash
+            result << create_repetition(core, k, left_positions, right_positions, "left")
+          end
+        end
+      end
+
+      repetitions.concat(result)
+    end
+
+    #--------------------------------------------------------------------------
+    def self.create_repetition(core, k, left_positions, right_positions, type)
+      repetition = core.class.new(core.length + k, left_positions, right_positions)
+      repetition.k = k
+
+      fake = fake_repetition(core.class, left_positions, right_positions, type)
+
+      if type == "left"
+        repetition.left, repetition.right = fake, core
+      else
+        repetition.left, repetition.right = core, fake
+      end
+
+      repetition
+    end
+
+    #---------------------------------------------------------------------------
+    def self.fake_repetition(cls, left_positions, right_positions, type)
+      cls.new(0, type == "left" ? left_positions : right_positions)
+    end
+  end # module RepetitionsIncrementation
+end # module TraceVisualization

data/lib/trace_visualization/repetitions_psy.rb

@@ -0,0 +1,83 @@
+require 'trace_visualization/bwt'
+require 'trace_visualization/suffix_array'
+require 'trace_visualization/longest_common_prefix'
+require 'trace_visualization/data/repetition'
+
+module TraceVisualization
+  module Repetitions
+    def self.psy1(str, p_min, decode_result = true)
+      sa  = TraceVisualization::SuffixArray.effective(str)
+      lcp = TraceVisualization::LongestCommonPrefix.effective(str, sa, str.size)
+      bwt = TraceVisualization::BurrowsWheelerTransform.bwt(str, sa, str.length)
+      
+      result = psy1_original(lcp, bwt, 3, str.length)
+      result = decode_psy1_result(result, sa) if decode_result
+      
+      result
+    end
+
+    ##
+    # PSY1 computes all the complete nonextendible repeats in 'str' of length
+    # p >= p_min. Complexity: \Theta(n)
+    #
+    # Article: Fast Optimal Algorithms for Computing All the Repeats is a String
+    # by Simon J. Puglisi, William F. Smyth, Munina Yusufu
+    def self.psy1_original(_LCP, _BWT, p_min, n)
+      result = []
+    
+      lcp = -1
+      lb = 0
+      bwt1 = _BWT[0]
+            
+      _LB = []
+      _LB.push(:lcp => lcp, :lb => lb, :bwt => bwt1)
+            
+      for j in 0 ... n
+        lb = j
+              
+        lcp  = j + 1 < n ? _LCP[j + 1] : -1
+        bwt2 = j + 1 < n ? _BWT[j + 1] : TraceVisualization::TERMINATION_CHAR
+              
+        bwt = le_letter(bwt1, bwt2)
+        bwt1 = bwt2
+              
+        while _LB.last()[:lcp] > lcp
+          prev = _LB.pop()
+                      
+          if prev[:bwt] == TraceVisualization::TERMINATION_CHAR && prev[:lcp] >= p_min
+            result.push(:lcp => prev[:lcp], :i => prev[:lb], :j => j)
+          end
+                      
+          lb = prev[:lb]
+          _LB.last()[:bwt] = le_letter(prev[:bwt], _LB.last()[:bwt])
+          bwt = le_letter(prev[:bwt], bwt)
+        end
+              
+        if _LB.last()[:lcp] == lcp
+          _LB.last()[:bwt] = le_letter(_LB.last()[:bwt], bwt)
+        else
+          _LB.push(:lcp => lcp, :lb => lb, :bwt => bwt)
+        end
+      end
+    
+      result    
+    end
+    
+    def self.le_letter(l1, l2)
+      (l1 == TraceVisualization::TERMINATION_CHAR || l1 != l2) ? TraceVisualization::TERMINATION_CHAR : l1
+    end
+    
+    def self.decode_psy1_result(result, sa)
+      repetitions = []
+
+      result.each do |item|
+        positions = (item[:i] .. item[:j]).collect { |idx| sa[idx] }
+        repetitions << TraceVisualization::Data::Repetition.new(item[:lcp], positions.sort)
+      
+      end
+      
+      repetitions
+    end
+    
+  end # module Repetitions
+end # module TraceVisualization