crosslanguagespotter 0.0.2-java

data/lib/crosslanguagespotter/methods/context.rb

@@ -0,0 +1,127 @@
+require 'set'
+require 'crosslanguagespotter/figures_evaluator'
+
+module CrossLanguageSpotter
+
+class Pair
+
+    def initialize(a,b)
+        raise "error" unless a.source.position(:absolute)
+        raise "error" unless b.source.position(:absolute)
+        if b.source.artifact(:absolute).filename < a.source.artifact(:absolute).filename
+            @nodes = [b,a]
+        else
+            @nodes = [a,b]
+        end
+    end
+
+    def first
+        nodes[0]
+    end
+
+    def second
+        nodes[1]
+    end
+
+    def nodes
+        @nodes
+    end
+
+    def eql?(other)
+        return false unless other.is_a?(Pair)
+        self.nodes[0]==other.nodes[0] && self.nodes[1]==other.nodes[1]
+    end
+
+    def ==(other)
+        self.eql?(other)
+    end
+
+    def to_s
+        "[#{nodes[0]} <-> #{nodes[1]}]"
+    end
+
+    def hash
+        nodes[0].hash*3+nodes[1].hash
+    end
+
+end
+
+class PointsMap
+
+    def initialize(alpha)
+        @alpha = alpha
+        @points = Hash.new {|h,k| h[k]=0.0}
+    end
+
+    def points(pair)
+        @points[pair]
+    end
+
+    def register_context_contribute(pair,value)
+        @points[pair] += 0.2*@alpha*value.to_f
+    end
+
+    def register_child_contribute(pair)
+        @points[pair] += 0.1
+    end
+
+    def each(threshold, &block)
+        @points.select{|k,v| v>=threshold}.each(&block)
+    end
+
+end
+
+class ContextReferencesProducer
+
+    attr_accessor :verbose
+
+    def initialize(parameters)
+        @threshold = parameters[:threshold]
+        @verbose = parameters[:verbose]
+        @alpha = parameters[:alpha]
+    end
+
+    def points_map(project)
+        # fill points map
+        points_map = PointsMap.new(@alpha)
+        block1 = Proc.new do |ni,nj|
+            context_ni = context(ni)
+            context_nj = context(nj)
+            shared_ctx = context_nj.intersection(context_ni).to_a
+            shared_ctx.each do |shared_ctx_entry|
+                v = shared_ctx_entry[:value]
+                context_ni.declarators_per_value(v).each do |di|
+                    context_nj.declarators_per_value(v).each do |dj|
+                        points_map.register_child_contribute(Pair.new(di,dj))
+                    end
+                end
+            end
+            points_map.register_context_contribute(Pair.new(ni,nj),shared_ctx.count)
+        end     
+        project.iter_over_shared_ids_instances {|ni,nj| block1.call(ni,nj) }
+        points_map
+    end
+
+    # It should produce a set of node ids
+    def produce_set(project)
+        set = Set.new
+        puts "Context method:" if @verbose
+                
+        points_map = points_map(project)
+        
+        # look into points map
+        points_map.each(@threshold).each do |pair,value|
+            f = pair.first
+            s = pair.second
+            id_i = NodeId.from_node(f)
+            id_j = NodeId.from_node(s)
+            set << CrossLanguageRelation.new([id_i,id_j])
+        end
+
+        puts "Context method, set produced: #{set.count} elements" if @verbose
+        set
+    end
+
+end
+
+end

data/lib/crosslanguagespotter/methods/jaro.rb

@@ -0,0 +1,118 @@
+require 'set'
+require 'crosslanguagespotter/figures_evaluator'
+
+class Array
+    # select array items with index
+    # give a block both the item with index of array
+    # filtered by a select statement
+    def select_with_index
+        index = -1
+        select { |x| index += 1; yield(x, index) }
+    end
+
+    # return indices array of array item
+    # example all indices of a in string "aaabaaabba"
+    def aindices(o)
+        out = Array.new
+        select_with_index { |x, i|
+        out << i if x == o }
+        out
+    end
+end
+
+module CrossLanguageSpotter
+
+class JaroReferencesProducer
+
+    attr_accessor :verbose
+
+    def initialize(parameters)
+        @threshold = parameters[:threshold]
+        @verbose = parameters[:verbose]
+        @winkleradjust = parameters[:winkleradjust]
+    end
+
+    # It should produce a set of node ids
+    def produce_set(project)
+        set = Set.new
+        puts "Jaro method:" if @verbose
+
+        block1 = Proc.new do |ni,nj|
+            context_ni = context(ni).sequence_of_values.map{|v| v.to_s}
+            context_nj = context(nj).sequence_of_values.map{|v| v.to_s}
+            if jaro_coefficient(context_ni,context_nj)>@threshold
+                id_i = NodeId.from_node(ni)
+                id_j = NodeId.from_node(nj)
+                set << CrossLanguageRelation.new([id_i,id_j])
+            end
+        end     
+        project.iter_over_shared_ids_instances {|ni,nj| block1.call(ni,nj) }                
+        puts "Jaro method, set produced: #{set.count} elements" if @verbose
+        set
+    end
+
+    def jaro_coefficient_from_nodes(ni,nj)
+        jaro_coefficient_from_context(context(ni),context(nj))
+    end
+
+    def jaro_coefficient_from_context(context_ni,context_nj)
+        s1 = context_ni.sequence_of_values.map{|v| v.to_s}
+        s2 = context_nj.sequence_of_values.map{|v| v.to_s}  
+        jaro_coefficient(s1,s2) 
+    end
+
+    def jaro_coefficient(s1,s2)
+        # if strings (without trailing & leadning spaces) are equal - return 1
+        #return 1 if str1.strip==str2.strip
+        # either string blank - return 0
+        #return 0 if str1.size==0 or str2.size==0
+        m = 0 # number of matching chars
+        tr = 0 # number of transpositions
+
+        # get character array length
+        s1l = s1.length
+        s2l = s2.length
+        # str2 should be the longer string
+        if s1l > s2l
+            s1, s2 = s2, s1
+        end
+        # hash from all unique str2 chars + occurances
+        # example 'aba': hash={ a => 0, b => 0 } a: first occurance, b first occurance
+        # if the first a was visited: { a => 1, b => 0} a: second occuance, b second occurance
+        found = Hash[*s2.uniq.sort.collect {|v| [v,0]}.flatten]
+        # matching distance definition
+        md = (([s1l,s2l].max / 2) - 1).to_i
+        s1.each_with_index do |c,i|
+            # find number of matching chars
+            if !found[c].nil? # character exists in str2
+                # calculates distance between 2 matching characters compare with md
+                if !s2.aindices(c)[found[c]].nil?
+                    x = (s2.aindices(c)[found[c]] - i).abs
+                    if x <= md
+                        found[c] += 1 # increase occurance of character
+                        m += 1 # increase number of matching characters
+                        # transpositions?
+                        if (x != 0)
+                            tr += 1
+                        end
+                    end
+                end
+            end
+        end
+        tr = (tr/2).to_i
+        # calc jaro-distance
+        third = 1.0/3
+        jd = (third * m / s1l) + (third * m / s2l) + (third * (m - tr) / m)
+        out = jd
+        # winkleradjust? if first l characters are the same
+        if @winkleradjust
+            l = 0
+            (0..s1l-1).each { |i| s1[i]==s2[i] ? l+=1 : break }
+            out = jd + (l * 0.1 * (1 - jd))
+        end
+        out
+    end
+
+end
+
+end

data/lib/crosslanguagespotter/methods/tversky.rb

@@ -0,0 +1,44 @@
+require 'set'
+require 'crosslanguagespotter/jaccard'
+require 'crosslanguagespotter/figures_evaluator'
+
+module CrossLanguageSpotter
+
+class TverskyReferencesProducer
+
+    attr_accessor :verbose
+
+    def initialize(parameters)
+        @threshold = parameters[:threshold]
+        @alpha     = parameters[:alpha]
+    end
+
+    # It should produce a set of node ids
+    def produce_set(project)
+        set = Set.new
+        puts "Tversky method:" if @verbose
+        block = Proc.new do |ni,nj|
+            context_ni = context(ni).values & project.shared_ids
+            context_nj = context(nj).values & project.shared_ids 
+            j = tversky_coefficient(context_ni,context_nj)
+            if j>=@threshold
+                id_i = NodeId.from_node(ni)
+                id_j = NodeId.from_node(nj)
+                puts " * '#{id_i.file}':#{id_i.index} -> '#{id_j.file}':#{id_j.index}" if @verbose
+                set << CrossLanguageRelation.new([id_i,id_j])
+            end
+        end     
+        project.iter_over_shared_ids_instances {|ni,nj| block.call(ni,nj) }     
+        puts "Tversky method, set produced: #{set.count} elements" if @verbose
+        set
+    end
+
+    def tversky_coefficient(context_ni,context_nj)
+        shared = context_ni & context_nj
+        others = (context_ni.count-shared.count)+(context_nj.count-shared.count)
+        shared.count.to_f/(shared.count.to_f+@alpha*others.to_f)
+    end
+
+end
+
+end

data/lib/crosslanguagespotter/model_loading.rb

@@ -0,0 +1,333 @@
+require "codemodels"
+require "codemodels/html"
+require "codemodels/js"
+require "crosslanguagespotter/context"
+
+module CrossLanguageSpotter
+
+AngularParser = CodeModels::Html::AngularJs.parser_considering_angular_embedded_code
+
+
+def traverse_index(node)
+    root = node.root(:also_foreign)
+    i = 1
+    root.traverse(:also_foreign) do |n|
+        return i if (n==node) && (n.source.position(:absolute)==node.source.position(:absolute))
+        i+=1
+    end
+    raise "Error..."
+end
+
+def node_at_traverse_index(root,index)
+    i = 1
+    root.traverse(:also_foreign) do |n|
+        return n if (i==index)
+        i+=1
+    end
+    raise "Error... traverse_index: #{index}. Reached #{i}"
+end
+
+
+def offset_referred_to_host(node)
+    base = node.eContainer ? offset_referred_to_host(node.eContainer) : 0
+    if node.eContainingFeature && node.eContainingFeature==:foreign_asts
+        base+node.eContainer.source.begin_pos.line-1
+    else
+        base
+    end
+end
+
+def line_referred_to_host(node,line)
+    offset_referred_to_host(node)+line
+end
+
+def host_lines(node)
+    [line_referred_to_host(node,node.source.begin_line),
+        line_referred_to_host(node,node.source.end_line)]
+end
+
+def is_in_line?(node,line)
+    line>=line_referred_to_host(node,node.source.begin_pos.line) && line<=line_referred_to_host(node,node.source.end_pos.line)  
+end
+
+class ModelLoader
+
+    def initialize
+        @models = Hash.new do |h,k|
+            h[k] = load_model(k)
+        end
+    end
+
+    def model(path)
+        @models[path]
+    end
+
+private
+
+    def load_model(relpath)
+        complete_path =relpath
+        raise "Unexisting file #{complete_path}" unless File.exist?(complete_path)
+        if relpath.end_with?'.html'
+            model = AngularParser.parse_file(complete_path)
+        elsif relpath.end_with?'.js'
+            model = CodeModels.parse_file(complete_path)
+        else
+            raise "I don't know what to do with: #{complete_path}"
+        end
+    end
+
+end
+
+class SequentialAst
+
+    attr_reader :node
+    attr_reader :value
+    attr_reader :reference_labels
+
+    def initialize(node,value,reference_labels)
+        @node = node
+        @value = value
+        @reference_labels = reference_labels
+    end
+
+end
+
+class Project
+
+    def initialize(src,verbose=false)
+        @models = {}
+        @verbose = verbose
+        load_all_models(src)    
+        @values_map_per_file = {}
+        @value_frequencies = Hash.new {|h,k| h[k]={} }  
+        @idf = {}
+    end
+
+    def models
+        @models
+    end
+
+    def sequential_asts
+        sequential_asts = []
+        @nodes_per_file.each do |f,nodes|
+            nodes.each do |n|
+                attribute_values_of_n = n.collect_values_with_count.keys
+                context = context(n) if attribute_values_of_n.count > 0
+                attribute_values_of_n.each do |v|
+                    sequential_asts << SequentialAst.new(n,v,context.values)
+                end
+            end
+        end
+        sequential_asts
+    end
+    
+    def shared_ids
+        @shared_ids = calc_shared_ids unless @shared_ids
+        @shared_ids
+    end
+
+    def files
+        files = Set.new
+        @files_per_values.values.each do |fs|
+            fs.each {|f| files<<f}
+        end
+        files
+    end
+
+    def iter_over_shared_ids_instances(&block)
+        shared_ids.each do |v|
+            iter_value_for_all_extensions(v,&block)
+        end
+    end
+
+    def iter_over_shared_ids_instances_case_insensitive(&block)
+        shared_ids.each do |v|
+            iter_value_for_all_extensions_case_insensitive(v,&block)
+        end
+    end
+
+    def iter_value_for_all_extensions(v,&block)
+        extensions = []
+        @files_per_values[v].each do |el|
+            ext = File.extname(el)  
+            extensions << ext unless extensions.include?(ext)
+        end
+        for i in 0...extensions.count
+            ext_i = extensions[i]
+            for j in (i+1)...extensions.count
+                ext_j = extensions[j]
+                 iter_value_in_extensions(v,ext_i,ext_j,&block)
+            end
+        end     
+    end
+
+    def iter_value_for_all_extensions_case_insensitive(v,&block)
+        extensions = []
+        @files_per_values[v].each do |el|
+            ext = File.extname(el)  
+            extensions << ext unless extensions.include?(ext)
+        end
+        for i in 0...extensions.count
+            ext_i = extensions[i]
+            for j in (i+1)...extensions.count
+                ext_j = extensions[j]
+                iter_value_in_extensions_case_insensitive(v,ext_i,ext_j,&block)
+            end
+        end     
+    end
+
+    def iter_value_in_extensions(v,ext_i,ext_j,&block)
+        raise "Error" if ext_i==ext_j
+        files_i = []
+        files_j = []
+        vs = [v]
+        vs.each do |v_el|
+            @files_per_values[v_el].each do |el|
+                files_i << el if File.extname(el)==ext_i
+                files_j << el if File.extname(el)==ext_j
+            end
+        end
+        nodes_i = []
+        nodes_j = []
+        files_i.each do |f|
+            vs.each do |v_el|
+                @nodes_per_value_and_file_map[v_el][f].each {|n| nodes_i << n}
+            end
+        end
+        files_j.each do |f|
+            vs.each do |v_el|
+                @nodes_per_value_and_file_map[v_el][f].each {|n| nodes_j << n}
+            end
+        end
+        for ni in nodes_i
+            for nj in nodes_j
+                block.call(ni,nj)
+            end
+        end
+    end
+
+    def iter_value_in_extensions_case_insensitive(v,ext_i,ext_j,&block)
+        raise "Error" if ext_i==ext_j
+        files_i = []
+        files_j = []
+        vs = values_case_insensitve(v)
+        vs.each do |v_el|
+            @files_per_values[v_el].each do |el|
+                files_i << el if File.extname(el)==ext_i
+                files_j << el if File.extname(el)==ext_j
+            end
+        end
+        nodes_i = []
+        nodes_j = []
+        files_i.each do |f|
+            vs.each do |v_el|
+                @nodes_per_value_and_file_map[v_el][f].each {|n| nodes_i << n}
+            end
+        end
+        files_j.each do |f|
+            vs.each do |v_el|
+                @nodes_per_value_and_file_map[v_el][f].each {|n| nodes_j << n}
+            end
+        end
+        for ni in nodes_i
+            for nj in nodes_j
+                block.call(ni,nj)
+            end
+        end
+    end 
+
+    def tf_idf(file,value)
+        value_frequency(file,value)*idf(value)
+    end
+
+    def itf_idf(file,value)
+        itf(file,value)*idf(value)
+    end
+
+    private
+
+    def idf(value)
+        unless @idf[value]
+            pos = 0
+            neg = 0
+            files.each do |f|
+                values_per_file(f).has_key?(value) ? pos+=1 : neg+=1
+            end
+            @idf[value] = Math.log((pos+neg).to_f/pos.to_f)
+        end
+        @idf[value]
+    end
+
+    def itf(file,value)
+        Math.log(1.0/value_frequency(file,value))
+    end
+
+    def value_frequency(file,value)
+        unless @value_frequencies[file][value]
+            values_map = values_per_file(file)
+            total = values_map.values.inject(:+)
+            @value_frequencies[file][value] = values_map[value].to_f/total.to_f
+        end
+        @value_frequencies[file][value]
+    end
+
+    def values_per_file(file)
+        unless @values_map_per_file[file]
+            @values_map_per_file[file] = @ml.model(file).collect_values_with_count_subtree(:also_foreign)
+        end
+        @values_map_per_file[file]
+    end
+
+    def values_case_insensitve(v)
+        @files_per_values.keys.select {|el| el.to_s.downcase==v.to_s.downcase}
+    end
+
+    def calc_shared_ids
+        shared = []
+        @files_per_values.each do |v,s|
+            extensions = []
+            s.each do |el|
+                ext = File.extname(el)  
+                extensions << ext unless extensions.include?(ext)
+            end
+            if extensions.count>1
+                shared << v
+            end
+        end
+        shared
+    end
+
+    def load_all_models(src)
+        @ml = ModelLoader.new
+        @nodes_per_file = Hash.new {|h,k| h[k] = []}
+        @files_per_values = Hash.new {|h,k| h[k] = Set.new}
+
+        # nodes per value, file
+        @nodes_per_value_and_file_map = Hash.new {|h,k| h[k] = Hash.new {|h,k| h[k] = [] }} 
+
+        Dir["#{src}/**/*.html"].each do |f|
+            puts "Loading model from #{f}" if @verbose
+            load_model_from_file(f)
+        end
+        Dir["#{src}/**/*.js"].each do |f|           
+            puts "Loading model from #{f}" if @verbose
+            load_model_from_file(f) 
+        end     
+    end
+
+    def load_model_from_file(f)
+        m = @ml.model(f)
+        @models[f]=m
+        m.traverse_also_foreign do |n|
+            @nodes_per_file[f] << n
+            values = n.collect_values_with_count.keys
+            values.each do |v|
+                @files_per_values[v] << f
+                @nodes_per_value_and_file_map[v][f] << n
+            end     
+        end
+    end
+
+end
+
+end