trace_visualization 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f106c8e07d09c745ae16a15ed0e3b96b932e2424
-  data.tar.gz: 1c0d4dcd86297c71fe3028c0db5db7a959cd7e38
+  metadata.gz: a5779b63579f93699f091c22ada7121df9469fc7
+  data.tar.gz: 336cfdc696d598e2247d09b089ec2b426df18e58
 SHA512:
-  metadata.gz: 4e15434bcc8cfb6a9546d039ecd5d5b5e2030d5b315eadeec2a8e39e27c832f3362c540bb7e1c9b9d3e9df48400bc9021560cc96353389de45d64041d2bdfc29
-  data.tar.gz: c322d0e30da40e0dec6d987d3dddef2ba95aa7497a2e3cbaa252197b07bdc7a6721a074888d18d9822579a582cb97cc662b4bdb2fbd6015af9c273ad223ca3e2
+  metadata.gz: 583564ac99432f9a0b05e1a51550497ae2f18d30fad51a7443fd8f9e6930567a66cac481a48c9cd594483a35eccdb4eff852a644bc8ce896fe7bffa14c3562fa
+  data.tar.gz: 721cf7cdf88fe2ac0380f4eb9517ffda3e6846cba7d317e6c5f1cdfac453c288b40b2b7d674de2039a0a2e88aab0a2b33b89dcde0d379a3b0c674b59fb5bb688

data/Gemfile

@@ -1,6 +1,7 @@
 source 'https://rubygems.org'
 
-ruby "2.0.0"
+ruby '2.0.0'
 
 # Specify your gem's dependencies in trace_visualization.gemspec
 gemspec
+

data/Rakefile

@@ -1 +1,11 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+
+desc 'Compile preprocessor'
+task :install do
+  system('cd lib/trace_visualization/preprocessor; make install; make clean')
+end
+
+desc 'Spec all functionality of gem'
+task :spec_all do
+  system('rspec spec/*')
+end

data/lib/trace_visualization/assert.rb

@@ -0,0 +1,5 @@
+module TraceVisualization
+  def self.assert_instance_of(object, expected_class)
+    raise "Illegal parameter type: expected #{expected_class}, actual #{object.class}. Object: #{object}" if not object.instance_of? expected_class
+  end
+end

data/lib/trace_visualization/data/lexeme.rb

@@ -0,0 +1,59 @@
+module TraceVisualization
+  module Data
+    class Lexeme
+      attr_accessor :name
+      attr_accessor :value
+      attr_accessor :int_value
+
+      # Length of preprocessed string (see LEXEME_REGEXP)
+      attr_accessor :lexeme_length
+      
+      attr_accessor :ord
+  
+      def initialize(name, value, int_value = -1)
+        @name, @value, @int_value = name, value, int_value
+      end
+  
+      def length
+        @value.length
+      end
+      
+      def <=>(anOther)
+        @ord <=> anOther.ord
+      end
+      
+      def <(other)
+        @ord < other.ord
+      end
+      
+      def to_int
+        @ord
+      end
+
+      def to_i
+        to_int
+      end
+
+      def to_str
+        @value
+      end
+      
+      def to_s
+        to_str
+      end
+      
+      def method_missing(name, *args, &blk)
+        raise "Missing method #{name}" 
+      end
+    end
+    
+    class LexemePos
+      attr_accessor :lexeme
+      attr_accessor :pos
+      
+      def initialize(lexeme, pos)
+        @lexeme, @pos = lexeme, pos
+      end
+    end
+  end
+end

data/lib/trace_visualization/lexeme_overlap_filter.rb

@@ -0,0 +1,64 @@
+module TraceVisualization
+  module LexemeOverlapFilter
+    
+    def self.process(lexeme_poss)
+      lexeme_poss.sort! { |a, b| a.pos <=> b.pos }
+      left_bound = lexeme_poss.inject(0) do |left_bound, lexeme_pos| 
+        [lexeme_pos.pos + lexeme_pos.lexeme.length, left_bound].max 
+      end
+    
+      idx, current, result = 0, [], []
+
+      for pos in 0 .. left_bound
+  
+        i = 0
+        while i < current.size
+          lexeme_pos = current[i]
+    
+          if lexeme_pos.pos + lexeme_pos.lexeme.length == pos
+            fl_delete_lexeme = false
+          
+            if current.size == 1
+              result << lexeme_pos
+              fl_delete_lexeme = true
+            else
+              if is_longest_lexeme(current, lexeme_pos)
+                result << lexeme_pos
+                current = []
+              else
+                fl_delete_lexeme = true
+              end
+            end
+      
+            if fl_delete_lexeme
+              current.delete_at(i)
+              i -= 1
+            end      
+          end
+    
+          i += 1
+        end
+
+        while idx < lexeme_poss.size && lexeme_poss[idx].pos == pos
+          current << lexeme_poss[idx] 
+          idx += 1
+        end
+      end
+    
+      result
+    end
+  
+    def self.is_longest_lexeme(lexeme_poss, lexeme_pos)
+      result = true
+
+      lexeme_poss.each do |other_lexeme_pos|
+        if lexeme_pos != other_lexeme_pos && lexeme_pos.lexeme.length < other_lexeme_pos.lexeme.length
+          result = false
+          break
+        end
+      end
+    
+      result
+    end
+  end
+end

data/lib/trace_visualization/mapping.rb

@@ -1,72 +1,241 @@
-module TraceVisualization
-  module Mapping
-    require 'time'
-    require 'ipaddr'
+require 'time'
+require 'ipaddr'
+
+require 'trace_visualization/reorder'
+require 'trace_visualization/lexeme_overlap_filter'
+require 'trace_visualization/data/lexeme'
 
-    require 'trace_visualization/reorder'
+module TraceVisualization
+  class Mapping
     
-    PATTERNS = {
+    attr_accessor :tokens
+  
+    LEXEME_REGEXP = /\{LEXEME;(?<name>[a-zA-Z0-9]+);(?<source>[^;]+);(?<value>[0-9]+)\}/
     
-      "id" => [
-        /(?<value>\[\d{3,}\])/
+    DEFAULT_TOKENS = {
+      :ID => [ 
+        /(?<lexeme>\[\d{3,}\])/, 
+        lambda { |source| source[1 ... -1].to_i } 
       ],
-
-      "ip" => [
-        /(?<value>(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))/
+      
+      :IP => [
+        /(?<lexeme>(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))/,
+        lambda { |source| IPAddr.new(source).to_i }
       ],
+        
+      :TIME => [
+        /(?<lexeme>\[\d{2} [a-zA-Z]{3} \d{4} \d{2}\:\d{2}\:\d{2}\])/,
+        lambda { |source| Time.parse(source[1 ... -1]).to_i }
+      ]      
+    }
+    
+    def initialize
+      # hash map: lexeme-object by lexeme-string
+      @lexeme_table = {}
+      
+      @tokens = {}
+      @mapped_str = []
+    end
     
-      "time" => [
-        /(?<value>\[\d{2} [a-zA-Z]{3} \d{4} \d{2}\:\d{2}\:\d{2}\])/
-      ]
+    def self.init(&block)
+      mapping = Mapping.new
+      mapping.instance_eval(&block)
+      mapping
+    end
 
-    }
+    # new    
+    def token(name, pattern, converter_func)
+      @tokens[name] = [pattern, converter_func]
+    end
     
-    class Item
-      include Comparable
-  
-      attr_reader :value # integer value for comparison
-      attr_reader :src   # source value 
-      attr_reader :type  # source type
+    def default_tokens
+      @tokens.merge!(DEFAULT_TOKENS)
+    end
+    
+    # new
+    def process(&block)
+      instance_eval(&block)
+      @max_value = TraceVisualization::Reorder.process(@mapped_str)
+    end
 
-      attr_accessor :ord # re-order value
-  
-      def initialize(src, type)
-        @src  = src
-        @type = type
-    
-        case type
-        when "id"
-          @value = @src[1 ... -1].to_i
-        when "ip"
-          @value = IPAddr.new(src).to_i
-        when "time"
-          @value = Time.parse(@src[1 ... -1]).to_i
-        when "char"
-          @value = src.getbyte(0)
+    # Load data from source file. File is read line by line
+    def from_file(path)
+      raise ArgumentError, 'Argument must be a string' unless path.instance_of? String 
+      raise ArgumentError, 'File path is not defined'  unless path.empty?
+      raise RuntimeError,  'File doesn\'t exists'      unless File.exists?(path)
+      
+      fd = open(path)
+      process_line(line) while line = fd.gets
+      fd.close
+    end
+
+    # Load data from preprocessed file. File is read line by line
+    def from_preprocessed_file(path)
+      raise ArgumentError, 'Argument must be a string' unless path.instance_of? String
+      raise ArgumentError, 'File path is not defined'  unless path.empty?
+      raise RuntimeError,  'File doesn\'t exists'      unless File.exists?(path)
+
+      fd = open(path)
+      process_preprocessed_line line while line = fd.gets
+      fd.close
+    end
+
+    # new
+    def from_string(str)
+      raise ArgumentError, 'Argument must be a string' unless str.instance_of? String 
+      raise ArgumentError, 'String is not defined'     if str.empty?
+      
+      @str = str
+
+      str.split("\n").each do |line|
+        process_line(line)
+      end
+    end
+    
+    def from_preprocessed_string(str)
+      raise ArgumentError, 'Argument must be a string' if not str.instance_of? String 
+      raise ArgumentError, 'String is not defined'     if str.empty?
+
+      str.split("\n").each do |line|
+        process_preprocessed_line(line)
+      end
+    end
+    
+    def process_preprocessed_line(line)
+      lexeme_positions = []
+      pos = 0
+      while (m = LEXEME_REGEXP.match(line, pos))
+        pos = m.begin(0)
+        
+        lexeme = install_lexeme_m(m)
+        lexeme_positions << TraceVisualization::Data::LexemePos.new(lexeme, pos)
+
+        pos += lexeme.lexeme_length
+      end
+      
+      pos, idx = 0, 0
+      while pos < line.length
+        lexeme = nil
+        if idx < lexeme_positions.size && lexeme_positions[idx].pos == pos
+          lexeme = lexeme_positions[idx].lexeme
+          idx += 1
         else
-          raise Exception.new("unknown type")
+          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1)
         end
+        pos += lexeme.lexeme_length
+        @mapped_str << lexeme
       end
-  
-      def length
-        @src.length
+      
+      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1)
+    end
+    
+    # new
+    def process_line(line)
+      lexeme_poss = []
+
+      @tokens.each do |name, value|
+        pattern, converter_func = value
+        pos = 0
+        while (m = pattern.match(line, pos))
+          lexeme_string, pos = m[:lexeme], m.begin(0)
+          
+          lexeme = install_lexeme(name, lexeme_string, converter_func.call(lexeme_string), lexeme_string.length)
+          lexeme_poss << TraceVisualization::Data::LexemePos.new(lexeme, pos)
+          
+          pos += lexeme_string.length
+        end
       end
-  
-      def <=>(anOther)
-        @ord <=> anOther.ord
+      
+      lexeme_poss = TraceVisualization::LexemeOverlapFilter.process(lexeme_poss)
+      
+      pos, idx = 0, 0
+      while pos < line.length
+        lexeme = nil
+        if idx < lexeme_poss.size && lexeme_poss[idx].pos == pos
+          lexeme = lexeme_poss[idx].lexeme
+          idx += 1
+        else
+          lexeme = install_lexeme('CHAR', line[pos], line[pos].ord, 1)
+        end
+        pos += lexeme.length
+        @mapped_str << lexeme
       end
       
-      def to_str
-        @src
+      @mapped_str << install_lexeme('CHAR', "\n", "\n".ord, 1)
+    end
+    
+    def install_lexeme(name, lexeme_string, int_value, lexeme_length)
+      lexeme = @lexeme_table[lexeme_string]
+      
+      if lexeme == nil
+        lexeme = TraceVisualization::Data::Lexeme.new(name, lexeme_string, int_value)
+        lexeme.lexeme_length = lexeme_length
+        @lexeme_table[lexeme_string] = lexeme
+      end
+      
+      lexeme
+    end
+
+    def install_lexeme_m(m)
+      lexeme = @lexeme_table[m[:source]]
+
+      if lexeme == nil
+        lexeme = TraceVisualization::Data::Lexeme.new(m[:name], m[:source], m[:value].to_i)
+        lexeme.lexeme_length = m.to_s.length
+        @lexeme_table[m[:source]] = lexeme
       end
+
+      lexeme
+    end
+    
+    def [](index)
+      @mapped_str[index]
+    end
+    
+    def length
+      @mapped_str.length
+    end
+    
+    def size
+      length
+    end
+    
+    def <<(object)
+      @mapped_str << Item.new(object, "unknown")
+    end
+    
+    def pop
+      @mapped_str.pop
+    end
+    
+    def max
+      @max_value
+    end
+    
+    def find_all
+      @mapped_str.find_all { |item| yield(item) }
     end
 
-    def self.parse(str)
+    def restore(pos = 0, length = @mapped_str.length)
+      @mapped_str[pos ... pos + length].inject("") { |res, c| res += c.value }
+    end
+    
+    def to_ary
+      @mapped_str.collect { |lexeme| lexeme.value }
+    end
+    
+    def method_missing(name, *args, &blk)
+      raise "Missing method #{name}" 
+    end
+
+    private
+
+    def parse(str)
       map = {}
       ppos = []
       itemByPos = {}
     
-      PATTERNS.each do |type, patterns|
+      DEFAULT_TOKENS.each do |type, patterns|
         patterns.each do |pattern|
           match(str, type, pattern, map, ppos, itemByPos)
         end
@@ -81,20 +250,19 @@ module TraceVisualization
         if i == ppos[j]
           item = itemByPos[ppos[j]]
           result << item
-          i += item.length
-          j += 1
+          i, j = i + item.length, j + 1
         else
           result << Item.new(str[i], "char")
           i += 1
         end        
       end
       
-      TraceVisualization::Reorder.process(result)
+      @max_value = TraceVisualization::Reorder.process(result)
       
       result
     end
      
-    def self.match(str, type, pattern, map, ppos, itemByPos)
+    def match(str, type, pattern, map, ppos, itemByPos)
       pos = 0
       
       limit = 1000
@@ -111,10 +279,5 @@ module TraceVisualization
       end
       
     end
-
-    def self.restore(array)
-      array.inject("") { |res, c| res += c.to_str }
-    end
-    
   end
 end

data/lib/trace_visualization/preprocessor/Makefile

@@ -0,0 +1,43 @@
+LEX = flex
+CC  = gcc
+CPP = g++
+
+LDFLAGS =-lfl
+
+CPPUNIT_BIN_CFG = cppunit-config
+CPPUNIT_CFLAGS  = `$(CPPUNIT_BIN_CFG) --cflags`
+CPPUNIT_LIBS    = `$(CPPUNIT_BIN_CFG) --libs`
+
+CFLAGS   =-O0 -Wall -g
+CPPFLAGS =-O0 -Wall $(CPPUNIT_CFLAGS) -g
+# CPPFLAGS =-O0 -Wall -std=c++11 $(CPPUNIT_CFLAGS)
+
+.cpp.o :
+	$(CPP) $(CPPFLAGS) -o $@ -c $< 
+
+.c.o :
+	$(CC) $(CFLAGS) -o $@ -c $< 
+
+%.cppo : %.c
+	$(CPP) $(CPPFLAGS) -o $@ -c $< 
+
+preprocessor: preprocessor.o parser_functions.o
+	$(CC) -o $@ $(LDFLAGS) $^
+
+preprocessor.c: preprocessor.l
+	$(LEX) -o $@ $^
+
+unit_tests: test_main.o hashmap.cppo hashmap_test.o \
+            parser_functions.cppo parser_test.o \
+            lexeme_table.cppo
+	$(CPP) -o $@ $(CPPUNIT_LIBS) $^
+	
+unit_tests_run: unit_tests
+	./unit_tests	
+
+install: preprocessor
+	cp preprocessor ../../../bin/
+	make clean
+
+clean: 
+	rm -f *.o *.cppo preprocessor.c unit_tests preprocessor

data/lib/trace_visualization/preprocessor/hashmap.c

@@ -0,0 +1,93 @@
+#include "hashmap.h"
+#include <stdio.h>
+
+hashmap_t* hashmap_new() {
+	hashmap_t* hashmap = (hashmap_t*)malloc(sizeof(hashmap_t));
+	
+	hashmap->size = 0;
+	hashmap->table = (hashmap_element**)calloc(HASHMAP_SIZE, sizeof(hashmap_element*));
+
+	return hashmap;
+}
+
+void* hashmap_get(hashmap_t* map, long key) {
+	hashmap_element* ptr;
+	int idx = (int)(key % HASHMAP_SIZE);
+	
+	for (ptr = map->table[idx]; ptr != NULL; ptr = ptr->next) {
+		if (ptr->key == key) {
+			return ptr->value;
+		}
+	}
+	
+	return NULL;	
+}
+
+int hashmap_put(hashmap_t* map, long key, void* value) {
+	int result = 1;
+	
+	if (hashmap_get(map, key) == NULL) {
+		hashmap_element* ptr;
+		hashmap_element* element = (hashmap_element*) malloc(sizeof(hashmap_element));
+
+		element->key   = key;
+		element->value = value;
+		element->next  = NULL;
+		
+		int idx = (int)(key % HASHMAP_SIZE);
+		ptr = map->table[idx];
+	
+		if (ptr) {
+			while (ptr->next) ptr = ptr->next;
+			ptr->next = element;
+		} else {
+			map->table[idx] = element;
+		}
+
+		map->size += 1;
+
+		result = 0;
+	} 
+	
+	return result;	
+}
+
+void** hashmap_values(hashmap_t* map) {
+	int i = 0;
+	int j = 0;
+	hashmap_element* ptr;	
+	void** values = (void**)malloc(map->size * sizeof(void*));
+	
+	for (; i < HASHMAP_SIZE; i += 1) {
+		ptr = map->table[i];
+		
+		while (ptr) {
+			values[j] = ptr->value;
+			ptr = ptr->next;
+			j += 1;
+		}
+	}
+	
+	return values;
+}
+
+void hashmap_values_free(void** values) {
+	free(values);
+}
+
+void hashmap_free(hashmap_t* map) {
+	hashmap_element* ptr;
+	hashmap_element* tmp;
+	int i;
+	for (i = 0; i < HASHMAP_SIZE; i++) {
+		for (ptr = map->table[i]; ptr != NULL; ) {
+			tmp = ptr;
+			ptr = (hashmap_element*)ptr->next;
+
+			free(tmp);
+		}
+	}
+	free(map);
+}
+
+

data/lib/trace_visualization/preprocessor/hashmap.h

@@ -0,0 +1,27 @@
+#ifndef __HASHMAP_H__
+#define __HASHMAP_H__
+
+#include <stdlib.h>
+
+#define HASHMAP_SIZE (256)
+
+// typedef struct hashmap_element hashmap_element; 
+struct hashmap_element {
+	long  key;
+	void* value;
+	hashmap_element * next;
+};
+
+struct hashmap_t {
+	hashmap_element** table;
+	int size;
+};
+
+extern hashmap_t* hashmap_new();
+extern int  hashmap_put(hashmap_t* map, long key, void* value);
+extern void* hashmap_get(hashmap_t* map, long key);
+extern void** hashmap_values(hashmap_t* map); 
+extern void hashmap_values_free(void** values);
+extern void  hashmap_free(hashmap_t* map);
+
+#endif