trace_visualization 0.0.1 → 0.0.2

data/spec/bwt_spec.rb

@@ -34,14 +34,36 @@ describe TraceVisualization::BurrowsWheelerTransform do
   end
   
   it "test with mapping" do
-    str = "127.0.0.1 a 127.0.0.1 b"
-    arr = TraceVisualization::Mapping.parse(str)
-    ip, ws, a, b = arr[0], arr[1], arr[2], arr[6]
+    str = "127.0.0.1 a 127.0.0.1 b" + TraceVisualization::TERMINATION_CHAR
     
-    bwt     = TraceVisualization::BurrowsWheelerTransform.bwt(arr, TraceVisualization::SuffixArray.effective(arr), arr.length)
-    bwt_str = TraceVisualization::Mapping.restore(bwt)
+    mapped_str = TraceVisualization::Mapping.init do
+      default_tokens
+    end
+    
+    mapped_str.process { from_string str }
+    
+    ip, ws, a, b = mapped_str[0], mapped_str[1], mapped_str[2], mapped_str[6]
+
+    ip.to_str.should eq "127.0.0.1"
+    ws.to_str.should eq " "
+    a.to_str.should eq "a"
+    b.to_str.should eq "b" 
+    
+    puts "ip.ord = #{ip.ord}, ws.ord = #{ws.ord}, a = #{a.ord}, b = #{b.ord}"
+    
+    sa = TraceVisualization::SuffixArray.effective(mapped_str)
+    
+    (sa.length == mapped_str.length).should be_true
+
+    bwt     = TraceVisualization::BurrowsWheelerTransform.bwt(mapped_str, sa, mapped_str.length)
+    bwt_str = bwt.inject("") { |res, c| res += c.to_str }
 
-    bwt.should     eq [ip, ip, a, ws, ws, b, ws]
-    bwt_str.should eq "127.0.0.1127.0.0.1a  b "
+    if ip < ws
+      bwt.should     eq [ip, ip, a, ws, ws, b, ws]
+      bwt_str.should eq "127.0.0.1127.0.0.1a  b "
+    else
+      sadasfa 
+    end
   end
+  
 end

data/spec/lexeme_overlap_filter_spec.rb

@@ -0,0 +1,59 @@
+require 'trace_visualization/lexeme_overlap_filter'
+require 'trace_visualization/data/lexeme'
+
+include TraceVisualization::Data
+
+describe TraceVisualization::LexemeOverlapFilter do
+
+  it 'test 1' do
+    lexeme2 = Lexeme.new(:name, "aa")
+    lexeme3 = Lexeme.new(:name, "aaa")
+    lexeme6 = Lexeme.new(:name, "aaaaaa")    
+    
+    i1 = LexemePos.new(lexeme2, 2)
+    i2 = LexemePos.new(lexeme2, 6)
+    i3 = LexemePos.new(lexeme2, 10)
+    i4 = LexemePos.new(lexeme6, 8)
+    i5 = LexemePos.new(lexeme3, 3)
+    
+    lexeme_positions = [i1, i2, i3, i4, i5]
+    
+    result = TraceVisualization::LexemeOverlapFilter.process(lexeme_positions)
+    
+    result.should eq [i5, i2, i4]
+  end
+
+  it 'test 2' do
+    lexeme2 = Lexeme.new(:name, "aa")
+    lexeme4 = Lexeme.new(:name, "aaaa")
+    
+    i1 = LexemePos.new(lexeme2, 0)
+    i2 = LexemePos.new(lexeme4, 2)
+    i3 = LexemePos.new(lexeme2, 4)
+    i4 = LexemePos.new(lexeme4, 5)
+    i5 = LexemePos.new(lexeme2, 7)
+    
+    lexemes = [i1, i2, i3, i4, i5]
+    
+    result = TraceVisualization::LexemeOverlapFilter.process(lexemes)
+    
+    result.should eq [i1, i2, i5]
+  end
+
+  it 'test 3' do
+    lexeme1 = Lexeme.new(:name, "a")
+    lexeme3 = Lexeme.new(:name, "aaa")
+    
+    i1 = LexemePos.new(lexeme1, 1)
+    i2 = LexemePos.new(lexeme1, 3)
+    i3 = LexemePos.new(lexeme3, 0)
+    i4 = LexemePos.new(lexeme3, 3)
+    
+    lexemes = [i1, i2, i3, i4]
+    
+    result = TraceVisualization::LexemeOverlapFilter.process(lexemes)
+    
+    result.should eq [i3, i4]
+  end
+
+end

data/spec/longest_common_prefix_spec.rb

@@ -16,10 +16,10 @@ describe TraceVisualization::LongestCommonPrefix do
     
     it 'should return correct result for mapped string', :current => true do
       str = "127.0.0.1 foo\r\n127.0.0.1 bar"
-      arr = TraceVisualization::Mapping.parse(str)
+      mapped_str = TraceVisualization::Mapping.new(str)
       
-      sa = TraceVisualization::SuffixArray.effective(arr)
-      lcp = TraceVisualization::LongestCommonPrefix.effective(arr, sa, arr.size)
+      sa = TraceVisualization::SuffixArray.effective(mapped_str)
+      lcp = TraceVisualization::LongestCommonPrefix.effective(mapped_str, sa, mapped_str.length)
 
       sa.should eq([6, 5, 8, 1, 10, 9, 2, 4, 3, 11, 7, 0])
       lcp.should eq([0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 2])

data/spec/mapping_spec.rb

@@ -1,67 +1,113 @@
+require 'trace_visualization'
 require 'trace_visualization/mapping'
 
+include TraceVisualization
+include TraceVisualization::Data
+
 describe TraceVisualization::Mapping do
-  it "simple id values" do
-    str = "foo[1234]bar[1235]far[1234]"
+  it 'simple id values' do
+    str = "foo[1234]bar[1235]far[1234]\n"
     
-    arr = TraceVisualization::Mapping.parse(str)
-
-    arr.size.should eq(12)
-
+    mapping = TraceVisualization::Mapping.init do
+      default_tokens
+    end 
+    
+    mapping.process do
+      from_string(str)
+    end
+    
+    mapping.length.should eq 13
     
-    ids = arr.find_all { |item| item.type == "id" }
+    ids = mapping.find_all { |lexeme| lexeme.name == :ID }
     ids.size.should eq(3)
-    ids[0].src.should eq("[1234]")
-    ids[1].src.should eq("[1235]")
-    ids[2].src.should eq("[1234]")
+    ids[0].value.should eq("[1234]")
+    ids[1].value.should eq("[1235]")
+    ids[2].value.should eq("[1234]")
     ids[0].should eq(ids[2])
     
-    str2 = TraceVisualization::Mapping.restore(arr)
-    str2.should eq(str)    
-    
+    mapping.restore.should eq str
   end
   
-  it "ip values" do
-    str = "user1 ip : 127.0.0.1 \r\nuser2 ip : 127.0.0.2"
+  it 'ip values' do
+    str = "user1 ip : 127.0.0.1 \r\nuser2 ip : 127.0.0.2\r\n"
+    
+    mapping = TraceVisualization::Mapping.init do
+      default_tokens
+    end 
+    
+    mapping.process do
+      from_string(str)
+    end
     
-    arr = TraceVisualization::Mapping.parse(str)
-    arr.size.should eq(27)
+    mapping.length.should eq 29
     
-    ips = arr.find_all { |item| item.type == "ip" }
+    ips = mapping.find_all { |lexeme| lexeme.name == :IP }
     ips.size.should eq(2)
-    ips[0].src.should eq("127.0.0.1")
-    ips[1].src.should eq("127.0.0.2")
+    ips[0].value.should eq("127.0.0.1")
+    ips[1].value.should eq("127.0.0.2")
     
-    str2 = TraceVisualization::Mapping.restore(arr)
-    str2.should eq(str)
+    mapping.restore.should eq str
   end
   
-  it "compare different types" do
+  it 'compare different types' do
+    mapping = TraceVisualization::Mapping.init do
+      default_tokens
+    end
+    
+    mapping.tokens.should_not be_nil
+    mapping.tokens[:ID].should_not be_nil
+    mapping.tokens[:IP].should_not be_nil
+    mapping.tokens[:TIME].should_not be_nil
+    
     # Ids
-    id_1 = TraceVisualization::Mapping::Item.new("[12345678]", "id")
-    id_2 = TraceVisualization::Mapping::Item.new("[12345679]", "id")    
-    TraceVisualization::Reorder.process([id_1, id_2])
+    id_1 = Lexeme.new(:ID, "[12345678]", mapping.tokens[:ID][1].call("[12345678]"))
+    id_2 = Lexeme.new(:ID, "[12345679]", mapping.tokens[:ID][1].call("[12345679]"))
+    Reorder.process([id_1, id_2])
     
     id_1.should be < id_2
     
     # IPs
-    ip_1 = TraceVisualization::Mapping::Item.new("127.0.0.1", "ip")
-    ip_2 = TraceVisualization::Mapping::Item.new("127.0.0.2", "ip")
-    TraceVisualization::Reorder.process([ip_1, ip_2])
+    ip_1 = Lexeme.new(:IP, "127.0.0.1", mapping.tokens[:IP][1].call("127.0.0.1"))
+    ip_2 = Lexeme.new(:IP, "127.0.0.2", mapping.tokens[:IP][1].call("127.0.0.2"))
+    Reorder.process([ip_1, ip_2])
     
     ip_1.should be < ip_2
 
     # Time
-    time_1 = TraceVisualization::Mapping::Item.new("[16 Jan 2013 00:10:00]", "time")
-    time_2 = TraceVisualization::Mapping::Item.new("[16 Jan 2013 00:10:01]", "time")    
-    TraceVisualization::Reorder.process([time_1, time_2])
+    time_1 = Lexeme.new(:TIME, '[16 Jan 2013 00:10:00]', mapping.tokens[:TIME][1].call('[16 Jan 2013 00:10:00]'))
+    time_2 = Lexeme.new(:TIME, '[16 Jan 2013 00:10:01]', mapping.tokens[:TIME][1].call('[16 Jan 2013 00:10:01]'))
+    Reorder.process([time_1, time_2])
 
     # Different
-    TraceVisualization::Reorder.process([time_1, time_2, id_1, ip_1])        
+    Reorder.process([time_1, time_2, id_1, ip_1])        
 
     time_1.should be < time_2
     id_1.should be < ip_1
     id_1.should be < time_1
   end
-end
+  
+  it 'Lexeme to_i conversion' do
+    lexeme = TraceVisualization::Data::Lexeme.new('unknown', 0, 0)
+    lexeme.ord = 0
+    lexeme.to_i.should eq 0
+  end
+  
+  it 'item as array index' do
+    lexeme = TraceVisualization::Data::Lexeme.new('unknown', 0, 0)
+    lexeme.ord = 1
+    array = [0, 1, 2]
+    
+    array[lexeme].should eq 1
+  end
 
+  it 'preprocessed string' do
+    str = 'Text {LEXEME;ID;[1234];1234} text {LEXEME;IP;127.0.0.127;1} text'
+    
+    mapping = TraceVisualization::Mapping.new
+    mapping.process do
+      from_preprocessed_string str
+    end
+    
+    mapping.size.should eq 19
+  end
+end

data/spec/reorder_spec.rb

@@ -1,3 +1,4 @@
+require 'trace_visualization'
 require 'trace_visualization/reorder'
 
 describe TraceVisualization::Reorder do
@@ -6,37 +7,54 @@ describe TraceVisualization::Reorder do
     include Comparable
     
     attr_accessor :ord
-    attr_accessor :value
+    attr_accessor :int_value
     
     def initialize(theValue)
-      @value = theValue
+      @int_value = theValue
     end
     
     def <=>(anOther)
-      @value <=> anOther.value
+      @ord <=> anOther.ord
     end
+    
   end
 
   it "reorder values and set correct order" do
     data = Array.new(100) { |index| A.new(index * 100) }
     data.shuffle!
     
-    TraceVisualization::Reorder.process(data)
+    max_value = TraceVisualization::Reorder.process(data)
     
-    data.each { |item| item.ord.should eq(item.value / 100 + 1) }
+    data.each { |item| item.ord.should eq(item.int_value / 100 + 1) }
+    max_value.should eq 100
   end
   
-  it "duplicate values with the same order", :current => true do
+  it "duplicate values with the same order" do
     x1, x2 = A.new(1), A.new(2)
     x3, x4 = A.new(123456789), A.new(123456789)
 
     data = [x1, x2, x3, x4].shuffle
     
-    TraceVisualization::Reorder.process(data)
+    max_value = TraceVisualization::Reorder.process(data)
     
     x1.ord.should eq(1)
     x2.ord.should eq(2)
     x3.ord.should eq(3)    
     x4.ord.should eq(3)
+    
+    max_value.should eq 3
+  end
+  
+  it "test TERMINATION_CHAR" do
+    x1 = A.new(2**7)
+    x2 = A.new(2**9)
+    x3 = A.new(TraceVisualization::TERMINATION_CHAR.ord)
+    
+    data = [x1, x2, x3].shuffle
+    max_value = TraceVisualization::Reorder.process(data)
+   
+    (x1 < x2).should be_true
+    (x2 < x3).should be_true
+    max_value.should eq x3.ord
   end
 end

data/spec/repetitions_psy_spec.rb

@@ -16,16 +16,16 @@ describe TraceVisualization::Repetitions do
   
   it 'simple test PSY1 with mapping' do
     str = "aaa[123]aaa"
-    arr = TraceVisualization::Mapping.parse(str)
+    mapped_str = TraceVisualization::Mapping.new(str)
     
-    TraceVisualization::Repetitions.psy1(arr, 3, false).should eq [{:lcp=>3, :i=>2, :j=>3}]
+    TraceVisualization::Repetitions.psy1(mapped_str, 3, false).should eq [{:lcp=>3, :i=>2, :j=>3}]
   end
   
   it 'test decode PSY1' do
     str = "aaa[123]xyz[654]aaa[123]" # <---> "aaaXxyzYaaaX"
-    arr = TraceVisualization::Mapping.parse(str)
+    mapped_str = TraceVisualization::Mapping.new(str)
     
-    rs = TraceVisualization::Repetitions.psy1(arr, 3)
+    rs = TraceVisualization::Repetitions.psy1(mapped_str, 3)
     
     rs.size.should eq 1
     rs[0].length.should eq 4
@@ -34,6 +34,6 @@ describe TraceVisualization::Repetitions do
     length   = rs[0].length
     position = rs[0].left_positions[0]
     
-    TraceVisualization::Mapping.restore(arr[position ... position + length]).should eq "aaa[123]"    
+    mapped_str.restore(position, length).should eq "aaa[123]"    
   end
 end

data/spec/suffix_array_spec.rb

@@ -11,9 +11,9 @@ describe TraceVisualization::SuffixArray do
     
     it "should correct process mapped string" do
       str = "abc[123]def[456]ghi"
-      arr = TraceVisualization::Mapping.parse(str)
+      mapped_str = TraceVisualization::Mapping.new(str)
       
-      TraceVisualization::SuffixArray.naive(arr).should eq([0, 1, 2, 4, 5, 6, 8, 9, 10, 3, 7])
+      TraceVisualization::SuffixArray.naive(mapped_str).should eq([0, 1, 2, 4, 5, 6, 8, 9, 10, 3, 7])
     end
   end
   
@@ -51,18 +51,40 @@ describe TraceVisualization::SuffixArray do
     
     it "should correct process mapped string" do
       str = "abc[123]def[456]ghi"
-      arr = TraceVisualization::Mapping.parse(str)
+      mapped_str = TraceVisualization::Mapping.new(str)
       
-      TraceVisualization::SuffixArray.effective(arr).should eq([0, 1, 2, 4, 5, 6, 8, 9, 10, 3, 7])
+      TraceVisualization::SuffixArray.effective(mapped_str).should eq([0, 1, 2, 4, 5, 6, 8, 9, 10, 3, 7])
     end
     
-    it "bug with endless loop" do
-      str = "127.0.0.1 user login\r\n127.0.0.1 user logout"
-      arr = TraceVisualization::Mapping.parse(str)
+    it "another string for mapped processing" do
+      str = "127.0.0.1 a 127.0.0.1 b"
+      
+      # 'X a X b'
+      mapped_str = TraceVisualization::Mapping.new(str)
+      
+      mapped_str[0].to_int.should eq 4
+      mapped_str[1].to_int.should eq 1
+      mapped_str[2].to_int.should eq 2
+      mapped_str[3].to_int.should eq 1
+      mapped_str[4].to_int.should eq 4
+      mapped_str[5].to_int.should eq 1
+      mapped_str[6].to_int.should eq 3      
+
+      length_before = mapped_str.length
       
-      sa = TraceVisualization::SuffixArray.effective(arr)      
+      sa = TraceVisualization::SuffixArray.effective(mapped_str)
+      mapped_str.length.should be length_before
+
+      sa.should eq [1, 5, 3, 2, 6, 0, 4]
+      sa.length.should be mapped_str.length
     end
     
+    it "bug with endless loop" do
+      str = "127.0.0.1 user login\r\n127.0.0.1 user logout"
+      mapped_str = TraceVisualization::Mapping.new(str)
+      
+      sa = TraceVisualization::SuffixArray.effective(mapped_str)      
+    end    
   end
   
 end

data/spec/utils_spec.rb

@@ -0,0 +1,30 @@
+require 'trace_visualization/utils'
+
+require 'tempfile'
+
+describe TraceVisualization::Utils do
+  it "lines pos" do
+    str = "qwerty\r\nqwerty\r\nqwerty"
+    lines_pos = TraceVisualization::Utils.lines_pos(str)
+    
+    lines_pos.should eq [0, 8, 16]    
+  end
+  
+  it "read file test" do
+    file = Tempfile.new('trace_visualization_test')
+    begin
+      file.write("first line\r\nsecond line\r\nthird line")
+      file.close
+      
+      str = TraceVisualization::Utils.read_file({
+        :file_name => file.path,
+        :n_lines => 2
+      })
+      
+      expect(str).to eq "first line\r\nsecond line\r\n"
+    ensure 
+      file.close
+      file.unlink
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trace_visualization
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Sergey
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-04 00:00:00.000000000 Z
+date: 2013-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -132,17 +132,32 @@ files:
 - .gitignore
 - Gemfile
 - LICENSE
-- LICENSE.txt
 - README.md
 - Rakefile
 - lib/trace_visualization.rb
+- lib/trace_visualization/assert.rb
 - lib/trace_visualization/bwt.rb
 - lib/trace_visualization/bwt.rbold
 - lib/trace_visualization/data/irepetition.rb
+- lib/trace_visualization/data/lexeme.rb
 - lib/trace_visualization/data/repetition.rb
 - lib/trace_visualization/generators.rb
+- lib/trace_visualization/lexeme_overlap_filter.rb
 - lib/trace_visualization/longest_common_prefix.rb
 - lib/trace_visualization/mapping.rb
+- lib/trace_visualization/preprocessor/Makefile
+- lib/trace_visualization/preprocessor/hashmap.c
+- lib/trace_visualization/preprocessor/hashmap.h
+- lib/trace_visualization/preprocessor/hashmap_test.cpp
+- lib/trace_visualization/preprocessor/lexeme.h
+- lib/trace_visualization/preprocessor/lexeme_table.c
+- lib/trace_visualization/preprocessor/lexeme_table.h
+- lib/trace_visualization/preprocessor/lexeme_table_cpp.h
+- lib/trace_visualization/preprocessor/parser_functions.c
+- lib/trace_visualization/preprocessor/parser_test.cpp
+- lib/trace_visualization/preprocessor/preprocessor.l
+- lib/trace_visualization/preprocessor/test_main.cpp
+- lib/trace_visualization/profile.rb
 - lib/trace_visualization/reorder.rb
 - lib/trace_visualization/repetitions.rb
 - lib/trace_visualization/repetitions_concatenation.rb
@@ -155,6 +170,7 @@ files:
 - lib/trace_visualization/visualization/console_color_print.rb
 - spec/bwt_spec.rb
 - spec/generators_spec.rb
+- spec/lexeme_overlap_filter_spec.rb
 - spec/longest_common_prefix_spec.rb
 - spec/mapping_spec.rb
 - spec/reorder_spec.rb
@@ -164,6 +180,7 @@ files:
 - spec/repetitions_spec.rb
 - spec/spec_helper.rb
 - spec/suffix_array_spec.rb
+- spec/utils_spec.rb
 - trace_visualization.gemspec
 homepage: ''
 licenses:
@@ -192,6 +209,7 @@ summary: Software for smart trace visualization
 test_files:
 - spec/bwt_spec.rb
 - spec/generators_spec.rb
+- spec/lexeme_overlap_filter_spec.rb
 - spec/longest_common_prefix_spec.rb
 - spec/mapping_spec.rb
 - spec/reorder_spec.rb
@@ -201,4 +219,5 @@ test_files:
 - spec/repetitions_spec.rb
 - spec/spec_helper.rb
 - spec/suffix_array_spec.rb
+- spec/utils_spec.rb
 has_rdoc: 

data/LICENSE.txt

@@ -1,22 +0,0 @@
-Copyright (c) 2013 TODO: Write your name
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.