crosslanguagespotter 0.0.2-java

data/lib/crosslanguagespotter/oracle.rb

@@ -0,0 +1,261 @@
+require 'codemodels'
+require 'codemodels/js'
+require 'codemodels/html'
+require 'csv'
+require 'crosslanguagespotter/model_loading'
+#require 'console'
+#require 'code_processing'
+
+include CodeModels
+
+module CrossLanguageSpotter
+
+OracleRelationEnd = Struct.new :file, :line, :col, :surface_form
+MetaOracleRelationEnd = Struct.new :file, :index
+
+class OracleLoader
+
+    def build_weka_classifier(srcpath,oraclepath)        
+        features_data = to_train_data(srcpath,oraclepath)
+        data = []
+        features_data.each do |rel,row|
+            data.push(row)
+        end
+        keys = {    
+            shared_length: :numeric,
+            tfidf_shared: :numeric,
+            itfidf_shared: :numeric,
+            perc_shared_length_min: :numeric,
+            perc_shared_length_max: :numeric,
+            diff_min: :numeric,
+            diff_max: :numeric,
+            perc_diff_min: :numeric,
+            perc_diff_max: :numeric,
+            context: :numeric,
+            jaccard: :numeric,
+            jaro: :numeric,
+            tversky: :numeric,
+            result: :boolean
+        }
+        train_instances = hash2weka_instances("oracle",data,keys,:result)
+        WekaClassifier.new(train_instances)
+    end
+
+    def to_train_data(srcpath,oraclepath)
+        project = Project.new(srcpath)
+
+        spotter = Spotter.new
+        features = spotter.features_for_project(project)
+
+        @file_lines = Hash.new do |h,k|
+            h[k] = File.readlines(k)
+        end
+
+        ok_a = ok_b = ko_a = ko_b = 0
+         
+        train_data = {}
+        File.open(oraclepath,'r').each_with_index do |input_line,l|  
+            input_line.strip!
+            unless input_line.start_with?('#')
+                values = input_line.split ":"
+                if values.count!=8
+                    raise "Line #{l+1}, error: #{input_line}. Values: #{values}"
+                end
+                # we order them to facilitate searching for duplicates
+                end_a = OracleRelationEnd.new values[0], values[1].to_i, values[2].to_i, values[3]
+                end_b = OracleRelationEnd.new values[4], values[5].to_i, values[6].to_i, values[7]
+                if end_b.file < end_a.file
+                    end_a, end_b = end_b, end_a
+                end
+
+                file_a         = values[0]
+                line_a         = values[1].to_i
+                col_a          = values[2].to_i
+                surface_form_a = values[3]
+                file_b         = values[4]
+                line_b         = values[5].to_i
+                col_b          = values[6].to_i         
+                surface_form_b = values[7]
+                #if values[8]=='t'
+                #    result = true
+                #elsif values[8]=='f'
+                #    result = false
+                #else
+                #    raise "Exptected true or false"
+                #end
+
+                #if oracle_values.values.include?([end_a,end_b])
+                #    raise "Line #{l+1} is a duplicate of line #{oracle_values.find {|k,v| v==[end_a,end_b]}}"
+                #else
+                #    oracle_values[l] = [end_a,end_b]
+                #end             
+
+                file_a = "#{srcpath}/#{file_a}"
+                file_b = "#{srcpath}/#{file_b}"
+
+                model_a = project.models[file_a]
+                model_b = project.models[file_b]
+
+                raise "Model not found for #{file_a}. Available: #{project.models.keys}" unless model_a                    
+                raise "Model not found for #{file_b}. Available: #{project.models.keys}" unless model_b
+
+                plain_col_a = convert_from_tabcolumn_to_plaincolumn(file_a,line_a,col_a)
+                plain_col_b = convert_from_tabcolumn_to_plaincolumn(file_b,line_b,col_b)
+
+                pos_a = SourcePosition.new(SourcePoint.new(line_a,plain_col_a),SourcePoint.new(line_a,plain_col_a+surface_form_a.length-1))
+                pos_b = SourcePosition.new(SourcePoint.new(line_b,plain_col_b),SourcePoint.new(line_b,plain_col_b+surface_form_b.length-1))
+                begin
+                    node_a = find_node(model_a,surface_form_a,pos_a)
+                    ok_a+=1
+                rescue Exception => e
+                    ko_a+=1
+                    puts "Line #{l+1}) problem with '#{surface_form_a}', file: #{file_a}, pos #{pos_a}: #{e}"
+                end
+                begin
+                    node_b = find_node(model_b,surface_form_b,pos_b)
+                    ok_b+=1
+                rescue Exception => e
+                    ko_b+=1
+                    puts "Line #{l+1}) problem with '#{surface_form_b}', file: #{file_b}, pos #{pos_b}: #{e}"
+                end
+
+                if node_a and node_b
+                    trindex_a = traverse_index(node_a)
+                    trindex_b = traverse_index(node_b)
+
+                    metaoracle_end_a = MetaOracleRelationEnd.new file_a,trindex_a
+                    metaoracle_end_b = MetaOracleRelationEnd.new file_b,trindex_b
+                    if metaoracle_end_b.file < metaoracle_end_a.file
+                        metaoracle_end_a, metaoracle_end_b = metaoracle_end_b, metaoracle_end_a
+                    end
+                    #if metaoracle_values.values.include?([metaoracle_end_a,metaoracle_end_b])
+                    #    raise "Line #{l+1} (#{[metaoracle_end_a,metaoracle_end_b]}) is a duplicate of line #{metaoracle_values.find {|k,v| v==[metaoracle_end_a,metaoracle_end_b]}}"
+                    #else
+                    #    metaoracle_values[l+1] = [metaoracle_end_a,metaoracle_end_b]
+                    #end 
+
+                    id_a = NodeId.from_node(node_a)
+                    id_b = NodeId.from_node(node_b)
+                    rel = CrossLanguageRelation.new([id_a,id_b])
+                    f = features[rel]
+                    raise "Unknown features for #{rel} (a:#{node_a.source.artifact(:absolute).filename} L#{node_a.source.position(:absolute).begin_line},b:#{node_b.source.artifact(:absolute).filename} L#{node_b.source.position(:absolute).begin_line})" unless f
+                    entry = { result: true }
+                    f.each do |k,v|
+                        entry[k] = v
+                    end
+                    train_data[rel] = entry
+                end
+            end        
+        end
+
+        # all the others are implicitly negative examples
+        project.iter_over_shared_ids_instances do |node_a,node_b| 
+            id_a = NodeId.from_node(node_a)
+            id_b = NodeId.from_node(node_b)
+            rel = CrossLanguageRelation.new([id_a,id_b])
+            unless train_data.has_key?(rel)
+                f = features[rel]
+                entry = { result: false }
+                f.each do |k,v|
+                    entry[k] = v
+                end
+                train_data[rel] = entry
+            end
+        end
+
+        pos = 0
+        neg = 0
+        train_data.each do |k,v|
+            if v[:result]
+                pos+=1
+                #puts v
+            else
+                neg+=1
+            end
+        end
+        return train_data
+    end
+
+    private
+
+    def candidates_included_in_all_the_others(candidates_in_correct_position)
+        candidates_in_correct_position.each do |small|
+            ok = true
+            candidates_in_correct_position.each do |big|
+                if small!=big
+                    unless big.source.position.include?(small.source.position)
+                        ok = false
+                    end
+                end
+            end
+            return small if ok
+        end
+        nil
+    end
+
+    def verbose_msg(msg)
+    end
+
+    def find_node(model,surface_form,position)
+        verbose_msg "Looking for '#{surface_form}'"
+        candidates_in_correct_position = []
+        candidates_in_other_positions = []
+        max_embedding_level = -1
+        model.traverse(:also_foreign) do |n|
+            if n.collect_values_with_count.has_key?(surface_form)
+                if n.source.position(:absolute).include?(position)
+                    if n.source.embedding_level>=max_embedding_level
+                        if n.source.embedding_level>max_embedding_level
+                            candidates_in_correct_position.clear
+                        end
+                        max_embedding_level = n.source.embedding_level
+                        candidates_in_correct_position << n
+                    end
+                else
+                    candidates_in_other_positions << n
+                end
+            end
+        end
+        if candidates_in_correct_position.count!=1
+            smallest_candidate = candidates_included_in_all_the_others(candidates_in_correct_position)
+            unless smallest_candidate
+                puts "I did not find exactly once '#{surface_form}' at #{position}. I found it there #{candidates_in_correct_position.count} times (found elsewhere #{candidates_in_other_positions.count} times)"
+
+
+                candidates_in_other_positions.each do |wp|
+                    puts " * #{wp.source.position(:absolute)}"
+                end
+                puts "Candidate in corresponding position:"
+                candidates_in_correct_position.each do |c|
+                    puts " * #{c} (embedded? #{c.source.embedded?})"
+                end
+                raise "Candidates found in #{position} are #{candidates_in_correct_position.count}"
+            else
+                puts "More than one candidate, I pick up the smallest"
+                return smallest_candidate
+            end
+        end
+        candidates_in_correct_position[0]
+    end
+
+    # the given column is calculated counting 4 for each tab,
+    # while the output count just 1 also per tab
+    def convert_from_tabcolumn_to_plaincolumn(file,line_index,tabcol)
+        line = @file_lines[file][line_index-1]
+        tabcol_to_plaincol(line,tabcol)
+    end
+
+    def tabcol_to_plaincol(line,tabcol)
+        c   = 0
+        i   = 0
+        while c<tabcol
+            c+=((line[i]=="\t") ? 4 : 1)
+            i+=1
+        end
+        raise "error" unless c==tabcol
+        i
+    end
+
+end
+
+end

data/lib/crosslanguagespotter/report.rb

@@ -0,0 +1,88 @@
+# encoding: utf-8
+
+require "codemodels"
+require "codemodels/html"
+require "codemodels/js"
+require 'htmlentities'
+require 'liquid'
+
+module CrossLanguageSpotter
+
+def _language_from_filename(filename)
+    if filename.end_with?('.html')
+        'html'
+    else
+        'javascript'
+    end
+end
+
+def generate_report_file(relations,output)
+    files_content = Hash.new{|h,k| h[k]=File.readlines(k)}
+    template = Liquid::Template.parse(File.read('./resources/template.html'))
+
+    data = []
+    relations.each do |rel|
+        entry = {}
+        entry['filenameA'] = rel[:node_a_file]
+        entry['languageA'] = _language_from_filename(entry['filenameA'])
+        entry['srcfileA']  = _code(files_content,rel[:node_a_file],
+                rel[:node_a_begin_line]-1,rel[:node_a_end_line]-1,
+                rel[:node_a_begin_column],rel[:node_a_end_column])
+        entry['filenameB'] = rel[:node_b_file]
+        entry['languageB'] = _language_from_filename(entry['filenameB'])
+        entry['srcfileB']  = _code(files_content,rel[:node_b_file],
+                rel[:node_b_begin_line]-1,rel[:node_b_end_line]-1,
+                rel[:node_b_begin_column],rel[:node_b_end_column])        
+        data << entry
+    end
+
+    File.open(output, 'w') {|f| f.write(template.render({"relations"=>data})) }
+end
+
+def _code(files_content,filename,begin_line,end_line,begin_col,end_col)
+    code = ""
+    snippet_lines = _get_snippet_lines(files_content[filename],begin_line)
+    snippet_lines[:before].each do |l|
+        code += HTMLEntities.new.encode(l,:decimal)
+    end
+    snippet_lines[:lines].each do |l|
+        #l = l.gsub("\t",'    ')
+        code += HTMLEntities.new.encode(l[0...(begin_col-1)],:decimal)
+        puts "<<<#{l[(begin_col-1)...end_col]}>>>"
+        code += '<span style="background-color:yellow;padding:2px">'+HTMLEntities.new.encode(l[(begin_col-1)...end_col],:decimal)+"</span>"
+        code += HTMLEntities.new.encode(l[end_col..-1],:decimal)
+    end
+    snippet_lines[:after].each do |l|
+        code += HTMLEntities.new.encode(l,:decimal)
+    end
+    code = _remove_extra_spaces(code)
+    code
+end
+
+def _get_snippet_lines(lines,line_index)   
+    around = 5
+    start_line = [0,line_index-5].max
+    end_line   = [lines.count-1,line_index+5].min
+    before = lines[start_line...line_index]
+    sel_lines  = [lines[line_index]]
+    after  = lines[(line_index+1)..(end_line)]
+    {before:before,lines:sel_lines,after:after}
+end
+
+def _number_of_spaces(s)
+    return 0 unless s.start_with?(' ')
+    1+_number_of_spaces(s[1..-1])
+end
+
+def _remove_extra_spaces(code,newline="&#10;")
+    lines = code.split(newline)
+    spaces = []
+    lines.each do |l|
+        spaces << _number_of_spaces(l)
+    end
+    extra_spaces = spaces.min
+    lines.each_with_index {|l,i| lines[i] = l[extra_spaces..-1]}    
+    lines.join(newline)
+end
+
+end

data/lib/crosslanguagespotter/version.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+module CrossLanguageSpotter
+  VERSION = "0.0.2"
+end

data/lib/crosslanguagespotter/wekaintegration.rb

@@ -0,0 +1,83 @@
+require 'java'
+
+module CrossLanguageSpotter
+
+def build_classifier(training_instances)
+    c = Java::weka::classifiers::trees::RandomTree.new
+    c.build_classifier(training_instances)
+    c
+end
+
+class WekaClassifier
+
+    def initialize(training_instances)
+        @weka_classifier = build_classifier(training_instances)
+    end
+
+    def classify(data_instances)
+        results = []
+        data_instances.enumerate_instances.each do |instance|
+            #puts "Classifying #{instance}"
+            r = @weka_classifier.classify_instance(instance)
+            #puts "Result: #{r} #{instance}"
+            results.push({result: r==0.0, instance: instance})
+        end
+        return results
+    end
+
+end
+
+def hash2weka_instances(name,data,keys,class_value)
+    boolean_values = Java::weka::core::FastVector.new
+    boolean_values.add_element("true")
+    boolean_values.add_element("false")
+
+    # fill attributes
+    attributes = Java::weka::core::FastVector.new
+    attributes_map = {}
+    attributes_indexes = {}
+    i = 0    
+    keys.each do |k,v|
+        raise "Null key in keys: #{keys}" unless k
+        raise "Null value for key #{k} in keys: #{keys}" unless v!=nil
+        if v==:numeric
+            # creates a numeric attribute
+            a = Java::weka::core::Attribute.new(k.to_s)
+        elsif v==:boolean
+            a = Java::weka::core::Attribute.new(k.to_s,boolean_values)
+        else
+            raise "Unknown attribute type: #{v}"
+        end
+        attributes.add_element(a)
+        attributes_map[k] = a
+        attributes_indexes[k] = i
+        i+=1
+    end
+    instances = Java::weka::core::Instances.new name, attributes, data.count
+
+    # fill instances
+    data.each do |row|
+        instance = Java::weka::core::Instance.new keys.count
+        keys.each do |k,v|
+            a = attributes_map[k]
+            if v==:numeric
+                instance.setValue(a,row[k])
+            elsif v==:boolean
+                instance.setValue(a,row[k].to_s)
+            else
+                raise "Unknown attribute type: #{v}"
+            end
+        end
+        instances.add(instance)
+    end
+
+    if class_value
+        instances.setClassIndex(attributes_indexes[class_value])
+    end
+
+    #puts instances.to_s
+
+    return instances
+end
+
+end

data/lib/crosslanguagespotter.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+require 'jars/weka.jar'
+
+curr_dir = File.dirname(__FILE__)
+
+Dir["#{curr_dir}/crosslanguagespotter/*.rb"].each { |rb| require rb }