codemodels 0.2.2-java

data/.gitignore

@@ -0,0 +1,19 @@
+Gemfile.lock
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+ 
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 ftomassetti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+codemodels
+===========
+
+codemodels is a library to create and manipulate models. A serialization format (based on JSON) is defined.
+
+It is based on [RGen](http://github.com/mthiede/rgen) and it supportes the conversion of EMF models through [emf_jruby](http://github.com/ftomassetti/emf_jruby).
+
+There are different gems which transform source code in models of the code. Currently they are:
+* [html-lightmodels](http://github.com/ftomassetti/html-lightmodels)
+* [java-lightmodels](http://github.com/ftomassetti/java-lightmodels)
+* [js-lightmodels](http://github.com/ftomassetti/js-lightmodels)
+* [properties-lightmodels](http://github.com/ftomassetti/properties-lightmodels)
+* [ruby-lightmodels](http://github.com/ftomassetti/ruby-lightmodels)
+* [xml-lightmodels](http://github.com/ftomassetti/xml-lightmodels)
+
+Therefore it can be used to perform analysis on different languages.
+
+Features
+========
+
+It handles also the case in which different languages are hosted in the same file (for example a Javascript script inside an HTML page).
+
+Most of the parsers permit also to associate each node with a precise point in the source code, so thath they could be used to implement layout-preserving refactoring.

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/codemodels.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'codemodels/version'
+
+Gem::Specification.new do |s|
+  s.platform    = 'java'
+  s.name        = 'codemodels'
+  s.version     = CodeModels::VERSION
+  s.date        = '2013-08-27'
+  s.summary     = "Library to build models of code"
+  s.description = "Library to build models of code of different languages in a uniform way."
+  s.authors     = ["Federico Tomassetti"]
+  s.email       = 'f.tomassetti@gmail.com'
+  s.homepage    = 'https://github.com/ftomassetti/codemodels'
+  s.license     = "APACHE 2"
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_dependency('json')
+  s.add_dependency('rgen')
+
+  s.add_development_dependency "bundler", "1.3.5"
+  s.add_development_dependency "rake"  
+  s.add_development_dependency "simplecov"
+end

data/lib/codemodels/foreign_ast.rb

@@ -0,0 +1,11 @@
+require 'rgen/metamodel_builder'
+
+module CodeModels
+
+def self.enable_foreign_asts(clazz)
+	clazz.class_eval do
+		contains_many_uni 'foreign_asts', CodeModelsAstNode		
+	end
+end
+
+end

data/lib/codemodels/info_extraction.rb

@@ -0,0 +1,139 @@
+require 'rgen/metamodel_builder'
+
+module CodeModels
+module InfoExtraction
+
+class TermsBreaker
+
+	attr_accessor :sequences, :inv_sequences
+
+	def initialize(language_specific_logic)
+		@language_specific_logic = language_specific_logic
+		@sequences = Hash.new {|h,k| 
+			h[k] = Hash.new {|h,k| 
+				h[k]=0
+			} 
+		}
+		@inv_sequences = Hash.new {|h,k| 
+			h[k] = Hash.new {|h2,k2| 
+				h2[k2]=0
+			} 
+		}
+	end
+
+	def self.from_context(language_specific_logic,context)
+		values_map = context.values_map
+		instance = new(language_specific_logic)	
+		values_map.each do |value,c|
+			value = value.to_s.strip
+			if language_specific_logic.terms_containing_value?(value)
+				words = language_specific_logic.to_words(value)				
+				first_words = words[0...-1]
+				instance.inv_sequences[words[0].downcase][:start] += c
+				first_words.each_with_index do |w,i|
+					instance.sequences[w.downcase][words[i+1].downcase] += c
+					instance.inv_sequences[words[i+1].downcase][w.downcase] += c
+				end
+				last_word = words.last
+				instance.sequences[last_word.downcase][:end] += c
+			else
+				# who cares, it will be never considered for composed names...
+			end
+		end
+		instance
+	end
+
+	def terms_in_value(value)
+		value = value.to_s.strip
+		if @language_specific_logic.terms_containing_value?(value)
+			words = @language_specific_logic.to_words(value)
+			group_words_in_terms(words).map{|w| w.downcase}			
+		else
+			[value]
+		end
+	end
+
+	private
+
+	def frequent_straight_sequence?(w1,w2)
+		w1 = w1.downcase
+		w2 = w2.downcase
+		all_sequences_of_w1 = 0
+		@sequences[w1].each do |k,v|
+			all_sequences_of_w1 += v
+		end
+		sequences_w1_w2 = @sequences[w1][w2]
+		(sequences_w1_w2.to_f/all_sequences_of_w1.to_f)>0.5
+	end
+
+	def frequent_inverse_sequence?(w1,w2)
+		w1 = w1.downcase
+		w2 = w2.downcase
+		all_inv_sequences_of_w1 = 0
+		@inv_sequences[w1].each do |k,v|
+			all_inv_sequences_of_w1 += v
+		end
+		inv_sequences_w1_w2 = @inv_sequences[w1][w2]
+		(inv_sequences_w1_w2.to_f/all_inv_sequences_of_w1.to_f)>0.5
+	end
+
+	def frequent_sequence?(w1,w2)
+		return false unless w2
+		frequent_straight_sequence?(w1,w2) && frequent_inverse_sequence?(w2,w1)
+	end
+
+	def group_words_in_terms(words)
+		# getNotSoGoodFieldName is not a term because
+		# notSoGoodFieldName is more frequently alone that preceded by get
+
+		return words if words.count==1
+		start_term = 0
+		end_term   = 0
+		term       = words[0]
+		while end_term < words.count && frequent_sequence?(words[end_term],words[end_term+1])
+			end_term += 1
+			term = @language_specific_logic.concat(term,words[end_term])
+		end
+		return [term] if end_term==(words.count-1)
+		[term] + group_words_in_terms(words[(end_term+1)..-1])
+	end
+
+end
+
+module InfoExtractionFunctionalities
+
+	def values_map
+		collect_values_with_count_subtree
+	end
+
+	def terms_map(language_specific_logic,context=nil)
+		# context default to root
+		unless context
+			context = self
+			while context.eContainer
+				context = context.eContainer
+			end		
+		end
+
+		# look into context to see how frequent are certain series of words,
+		# frequent series are recognized as composed terms
+		terms_breaker = TermsBreaker.from_context(language_specific_logic,context)
+
+		v_map = self.values_map
+		terms_map = Hash.new {|h,k| h[k]=0}
+		v_map.each do |v,n|
+			terms_breaker.terms_in_value(v).each do |t|
+				terms_map[t] += n
+			end
+		end
+		terms_map
+	end
+
+end
+
+class ::RGen::MetamodelBuilder::MMBase
+	include InfoExtractionFunctionalities
+end
+
+end
+end

data/lib/codemodels/language.rb

@@ -0,0 +1,65 @@
+module CodeModels
+
+@@languages = []
+
+class Parser
+
+	def parse_file(path)
+		code = IO.read(path)
+		parse_code(code)
+	end
+
+end
+
+module CodeModelsReflectionInfo
+	attr_accessor :source
+	attr_accessor :language
+end
+
+class Position
+	attr_accessor :line, :column
+end
+
+class SourceInfo
+	attr_accessor :filename
+	attr_accessor :begin_pos, :end_pos
+
+	def to_code
+		raise "Unimplemented"
+	end
+
+end
+
+class Language
+	attr_reader :name
+	attr_reader :extensions
+	attr_reader :parser
+
+	def initialize(name)
+		@name = name
+		@extensions = []
+	end	
+
+	def can_parse?(path)
+		extension = File.extname(path)
+		extension=extension[1..-1] if extension.length>0		
+		@extensions.include?(extension)
+	end
+
+end
+
+def self.register_language(language)
+	@@languages << language
+end
+
+def self.registered_languages
+	@@languages
+end
+
+def self.parse_file(path)
+	l = @@languages.find {|l| l.can_parse?(path) }
+	raise "I don't know how to parse #{path}. Languages known: #{@@languages.map(&:name)}" unless l
+	l.parser.parse_file(path)
+end
+
+end

data/lib/codemodels/metamodel.rb

@@ -0,0 +1,11 @@
+require 'rgen/metamodel_builder'
+require 'codemodels/language'
+
+module CodeModels
+
+# All nodes should derive from this one
+class CodeModelsAstNode < RGen::MetamodelBuilder::MMBase
+	include CodeModelsReflectionInfo
+end
+
+end

data/lib/codemodels/model_building.rb

@@ -0,0 +1,81 @@
+require 'codemodels/serialization'
+
+module CodeModels
+
+module ModelBuilding
+
+class << self
+	attr_accessor :verbose
+end
+
+def self.verbose_warn(msg)
+	warn(msg) if verbose
+end
+
+def self.handle_serialized_models_in_dir(src,src_extension,error_handler=nil,model_handler,&model_creator)
+	Dir["#{src}/**/*.#{src_extension}"].each do |fd|
+		verbose_warn "== #{fd} =="
+		handle_serialized_model_per_file(fd,error_handler,model_handler,&model_creator)		
+	end
+end
+
+def self.handle_models_in_dir(src,src_extension,error_handler=nil,model_handler,&model_creator)
+	Dir["#{src}/**/*.#{src_extension}"].each do |fd|	
+		verbose_warn "== #{fd} =="			
+		handle_model_per_file(fd,error_handler,model_handler,&model_creator)		
+	end
+end
+
+def self.generate_models_in_dir(src,dest,src_extension,dest_extension,max_nesting=500,error_handler=nil,&model_creator)
+	verbose_warn "== #{src} -> #{dest} =="
+	Dir["#{src}/**/*.#{src_extension}"].each do |fd|		
+		if File.directory? fd
+			basename = File.basename(fd)
+			generate_models_in_dir("#{src}/#{basename}","#{dest}/#{basename}",src_extension,dest_extension,max_nesting,error_handler,&model_creator)
+		else
+			if File.extname(fd)==".#{src_extension}"
+				translated_simple_name = "#{File.basename(fd, ".#{src_extension}")}.#{dest_extension}"
+				translated_name = "#{dest}/#{translated_simple_name}"
+				verbose_warn "* #{fd} --> #{translated_name}"
+				generate_model_per_file(fd,translated_name,max_nesting,error_handler,&model_creator)
+			end
+		end
+	end
+end
+
+def self.handle_model_per_file(src,error_handler=nil,model_handler,&models_generator)
+	verbose_warn "<Model from #{src}>"
+	begin
+		m = models_generator.call(src)
+		model_handler.call(src,m)
+	rescue => e
+		if error_handler
+			error_handler.call(src,e)
+		else
+			raise e
+		end
+	end
+end
+
+def self.generate_model_per_file(src,dest,max_nesting=500,error_handler=nil,&models_generator)
+	if not File.exist? dest 
+		verbose_warn "<Model from #{src}>"
+		
+		begin
+			m = models_generator.call(src)
+			LightModels::Serialization.save_model(m,dest,max_nesting)
+		rescue Exception => e
+			if error_handler
+				error_handler.call(src,e)
+			else
+				raise e
+			end
+		end		
+	else
+		verbose_warn "skipping #{src} because #{dest} found"
+	end
+end
+
+end
+
+end

data/lib/codemodels/monkey_patching.rb

@@ -0,0 +1,30 @@
+class Module
+
+  def simple_name
+    if (i = (r = name).rindex(':')) then r[0..i] = '' end
+    r
+  end
+  
+end
+
+class String
+
+	def remove_postfix(postfix)
+		raise "'#{self}'' have not the right postfix '#{postfix}'" unless end_with?(postfix)
+		self[0..-(1+postfix.length)]
+	end
+
+	def remove_prefix(prefix)
+		raise "'#{self}'' have not the right prefix '#{prefix}'" unless start_with?(prefix)
+		self[prefix.length..-1]
+	end
+	
+	def proper_capitalize 
+    	self[0, 1].upcase + self[1..-1]
+  	end
+
+	def proper_uncapitalize 
+    	self[0, 1].downcase + self[1..-1]
+  	end
+
+end	

data/lib/codemodels/parsing.rb

@@ -0,0 +1,184 @@
+require 'codemodels/monkey_patching'
+
+module CodeModels
+
+class ParsingError < Exception
+ 	attr_reader :node
+ 	attr_reader :line
+
+ 	def initialize(node,msg,line=nil)
+ 		@node = node
+ 		@msg = msg
+ 		@line = lin
+ 	end
+
+ 	def to_s
+ 		"#{@msg}, start line: #{@line}"
+ 	end
+
+end
+
+class UnknownNodeType < ParsingError
+
+ 	def initialize(node,line=nil,node_type=nil,where=nil)
+ 		super(node,"UnknownNodeType: type=#{node_type} , where: #{where}")
+ 	end
+
+end
+
+module ParserWrapper
+
+@@verbose = false
+
+JavaCollection = ::Java::JavaClass.for_name("java.util.Collection")
+	
+def adapter(model_class,ref)
+	if adapter_specific_class(model_class,ref)
+		adapter_specific_class(model_class,ref)
+	else
+		if model_class.superclass!=Object
+			adapter(model_class.superclass,ref) 
+		else
+			nil
+		end
+	end
+end
+
+def reference_to_method(model_class,ref)
+	s = ref.name
+	#s = 'value' if s=='body'
+	adapted = adapter(model_class,ref)
+	s = adapted if adapted		
+	s.to_sym
+end
+
+def attribute_to_method(model_class,att)
+	s = att.name
+	adapted = adapter(model_class,att)
+	s = adapted if adapted		
+	s.to_sym
+end
+
+def assign_ref_to_model(model,ref,value)
+	return unless value==nil # we do not need to assign a nil...
+	if ref.many
+		adder_method = :"add#{ref.name.capitalize}"
+		value.each {|el| model.send(adder_method,node_to_model(el))}
+	else
+		setter_method = :"#{ref.name}="
+		raise "Trying to assign an array to a single property. Class #{model.class}, property #{ref.name}" if value.is_a?(::Array)
+		model.send(setter_method,node_to_model(value))
+	end
+rescue Object => e
+	puts "Problem while assigning ref #{ref.name} (many? #{ref.many}) to #{model.class}. Value: #{value.class}"
+	puts "\t<<#{e}>>"
+	raise e
+end
+
+def assign_att_to_model(model,att,value)
+	if att.many
+		adder_method = :"add#{att.name.capitalize}"
+		value.each {|el| model.send(adder_method,el)}
+	else
+		setter_method = :"#{att.name}="
+		raise "Trying to assign an array to a single property. Class #{model.class}, property #{att.name}" if value.is_a?(::Array)
+		model.send(setter_method,value)
+	end
+end
+
+def populate_attr(node,att,model)	
+	value = get_feature_value(node,att)
+	model.send(:"#{att.name}=",value) if value!=nil
+rescue Object => e
+	puts "Problem while populating attribute #{att.name} of #{model} from #{node}. Value: #{value}"
+	raise e
+end
+
+def populate_ref(node,ref,model)
+	log("populate ref #{ref.name}, node: #{node.class}, model: #{model.class}")
+	value = get_feature_value(node,ref)
+	log("\tvalue=#{value.class}")
+	if value!=nil
+		if value==node
+			puts "avoiding loop... #{ref.name}, class #{node.class}" 
+			return
+		end
+		if JavaCollection.assignable_from?(value.java_class)
+			log("\tvalue is a collection")
+			capitalized_name = ref.name.proper_capitalize				
+			value.each do |el|
+				unless el.respond_to?(:parent)
+					class << el
+						attr_accessor :parent						
+					end
+				end
+				el.parent = node
+				model.send(:"add#{capitalized_name}",node_to_model(el))
+			end
+		else
+			log("\tvalue is not a collection")
+			unless value.respond_to?(:parent)
+				value.class.__persistent__ = true
+				class << value
+					attr_accessor :parent
+				end
+			end
+			value.parent = node
+			model.send(:"#{ref.name}=",node_to_model(value))
+		end
+	end
+end
+
+def log(msg)
+	puts msg if @@verbose
+end
+
+def node_to_model(node)
+	log("node_to_model #{node.class}")
+	metaclass = get_corresponding_metaclass(node)
+	instance = metaclass.new
+	metaclass.ecore.eAllAttributes.each do |attr|
+		populate_attr(node,attr,instance)
+	end
+	metaclass.ecore.eAllReferences.each do |ref|
+		populate_ref(node,ref,instance)
+	end
+	instance
+end
+
+def transform_enum_values(value)
+	if value.respond_to?(:java_class) && value.java_class.enum?
+		value.name
+	else
+		value
+	end
+end
+
+def get_feature_value_through_getter(node,feat_name)
+	capitalized_name = feat_name.proper_capitalize
+	methods = [:"get#{capitalized_name}",:"is#{capitalized_name}"]
+
+	methods.each do |m|
+		if node.respond_to?(m)
+			begin
+				return transform_enum_values(node.send(m))
+			rescue Object => e
+				raise "Problem invoking #{m} on #{node.class}: #{e}"
+			end
+		end
+	end
+	raise "how should I get this... #{feat_name} on #{node.class}. It does not respond to #{methods}"
+end
+
+def get_feature_value(node,feat)
+	adapter = adapter(node.class,feat)		
+	if adapter
+		adapter[:adapter].call(node)
+	else
+		get_feature_value_through_getter(node,feat.name)
+	end
+end
+
+end
+
+end