codemodels-html 0.1.0-java

data/lib/codemodels/html/parser.rb

@@ -0,0 +1,272 @@
+require 'jars/jericho-html-3.3.jar'
+require 'codemodels'
+require 'codemodels/html/monkey_patching'
+
+module CodeModels
+module Html
+
+class TextBlock
+	attr_accessor :source
+	attr_accessor :value
+
+	def begin_point=(data)
+		@source = SourceInfo.new unless @source
+		@source.begin_point= data
+	end
+
+	def end_point=(data)
+		@source = SourceInfo.new unless @source
+		@source.end_point= data
+	end	
+end
+
+class Java::NetHtmlparserJericho::Element 
+
+	def text_blocks(code)
+		blocks = []
+		break_content(self,code).each do |s,e|
+			text = code[s,e-s]			
+			unless text==nil or text.strip.empty?
+				#puts "<<<#{text}>>>"
+				block = TextBlock.new
+				block.value = text
+				block.source = SourceInfo.new
+				block.source.position = SourcePosition.from_code_indexes(code,s,e-1)
+
+				blocks << block
+			end
+		end
+		blocks
+	end
+
+	private
+
+	def break_content(node,code)
+		text_inside = code[(node.begin)...(node.end)]
+		#puts "Text inside #{node.name} ^#{text_inside}^ It has child elements #{node.child_elements}"
+		i  = text_inside.first_index('>') 
+		raise "No '>' found in node #{node}, text inside: '#{text_inside}'" unless i
+		#puts "Index i: #{i}"
+		raise "No Fixnum" unless node.begin.is_a?(Fixnum)
+		raise "No Fixnum" unless i.is_a?(Fixnum)
+		start_index = node.begin+i+1
+		li = text_inside.last_index('<')
+		#puts "Index li: #{li}"
+		end_index    = node.begin+li
+		#puts "Indexes #{start_index} #{end_index}"
+		#puts "Content of #{node.name} ^#{code[start_index,end_index-start_index]}^"
+
+		# no content
+		return [] if start_index==end_index
+
+		if node.child_elements.count==0
+			return [[start_index,end_index]]
+		else
+			segments = []
+			# before the first
+			segments << [start_index,node.child_elements.first.begin]
+			# between children
+			for i in 0...(node.child_elements.count-1)
+				s = node.child_elements[i].end
+				e = node.child_elements[i+1].begin
+				segments << [s,e]
+			end
+			# after the last
+			i_last = node.child_elements.size-1
+			last_child = node.child_elements[i_last]
+			segments << [last_child.end,end_index]			
+		end
+		segments
+	end
+
+end
+
+class Parser < CodeModels::Parser
+
+	def initialize
+		@embedded_parsers = Hash.new do |h,k|
+			h[k] = []
+		end
+	end
+
+	def parse_file(path)
+		code = IO.read(path)
+		parse_artifact(FileArtifact.new(path,code))
+	end
+
+	def raw_node_tree(code)
+		Java::net.htmlparser.jericho.Config.IsHTMLEmptyElementTagRecognised = true
+		xhtml = Java::net.htmlparser.jericho.Config::CompatibilityMode::XHTML
+		Java::net.htmlparser.jericho.Config.CurrentCompatibilityMode = xhtml
+		reader = java.io.StringReader.new code
+		source = Java::net.htmlparser.jericho.Source.new reader
+		source
+	end
+
+	def parse_code(code)
+		parse_artifact(FileArtifact.new('<code>',code))
+	end
+
+	def parse_artifact(artifact)
+		source = raw_node_tree(artifact.code)
+		node_to_model(source,artifact.code,artifact)
+	end
+
+	# It operates on original node, not on the model obtained because
+	# it could have less information. For example in parsing scripts I need the
+	# raw content
+	def register_embedded_parser(node_class,embedded_parser,&selector)
+		@embedded_parsers[node_class] << {embedded_parser: embedded_parser, selector: selector}
+	end
+
+	def self.node_content(node,code)
+		pos = node_content_pos(node,code)
+		code[pos[0]..pos[1]]
+	end
+
+	def self.node_content_pos(node,code)
+		text_inside = code[(node.begin)...(node.end)]
+		i  = text_inside.first_index('>') 
+		start_index = node.begin+i+1
+		li = text_inside.last_index('<')
+		end_index    = node.begin+li-1
+		raise "problem" if start_index>end_index
+		#content = code[start_index,end_index-start_index]
+		[start_index,end_index]
+	end
+
+	private
+
+	def add_source_info(node,model,code,artifact)
+		return if model==nil
+		model.language = LANGUAGE
+		model.source = CodeModels::SourceInfo.new
+		model.source.artifact = artifact
+		model.source.position = SourcePosition.from_code_indexes(code,node.begin,node.end)
+	end
+
+	def analyze_content(model,node,code,artifact)
+		node.text_blocks(code).each do |tb|
+			raise "GOTCHA #{node.name} TEXT INSIDE '#{code[(node.begin)..(node.end)]}'" if (tb.value=='</head>')
+			t = Html::Text.new
+			t.value = tb.value
+
+			t.language = LANGUAGE
+			t.source = tb.source
+			t.source.artifact = artifact
+
+			model.addChildren(t)
+		end
+	end
+
+	def node_to_model(node,code,artifact)
+		if node.is_a? Java::NetHtmlparserJericho::Source
+			model = Html::HtmlDocument.new
+			translate_many(code,node,model,:children,node.child_elements,artifact)
+			model
+		elsif node.is_a? Java::NetHtmlparserJericho::Element
+			if node.name=='!doctype'
+				model = Html::DTD.new
+				# I am naughty... and waiting for the Jericho parser to fix
+				# how they parse doctypes
+				model.name = 'html'
+				model		
+			elsif node.name=='script'
+				model = Html::Script.new	
+				model.name = node.name
+				# something is obfuscating the attributes method... damn...
+				attributes_method = node.java_method(:getAttributes)
+				attributes = attributes_method.call		
+				raise "Error, Attributes expected, instead it is '#{attributes.class}'. Node class #{node.class}" unless attributes.is_a?(Java::NetHtmlparserJericho::Attributes)
+				if attributes.get('type') && attributes.get('type').value=='text/ng-template'
+					content_pos = Parser.node_content_pos(node,code)
+					#raise "mismatch" unless content==embedded_artifact.code
+					embedded_artifact = EmbeddedArtifact.new
+					embedded_artifact.host_artifact = artifact
+					si = content_pos[0]
+					while code[si]==' '||code[si]=="\t"||code[si]=="\r"||code[si]=="\n"
+						si+=1
+					end
+					embedded_artifact.position_in_host = SourcePosition.from_code_indexes(code,si,content_pos[1])
+					script_doc = parse_artifact(embedded_artifact)				
+					model.addForeign_asts script_doc
+				end
+			else			
+				model = Html::Node.new
+				analyze_content(model,node,code,artifact)
+				model.name = node.name		
+				translate_many(code,node,model,:children,node.child_elements,artifact)
+			end		
+			translate_many(code,node,model,:attributes,artifact)
+			model	
+		elsif node.is_a? Java::NetHtmlparserJericho::Attribute
+			model = Html::Attribute.new
+			model.name  = node.name
+			model.value = node.value
+			model				
+		else
+			raise "Unknown node class: #{node.class}"
+		end
+
+		add_source_info(node,model,code,artifact)
+		check_foreign_parser(node,code,model,artifact)
+		model
+	end
+
+	def check_foreign_parser(node,code,model,artifact)
+		@embedded_parsers[node.class].each do |ep|
+			selector = ep[:selector]
+			embedded_parser = ep[:embedded_parser]
+			embedded_position = selector.call(node,code)
+			if embedded_position
+				unless embedded_position.is_a?(Array)
+					embedded_position = [embedded_position]
+				end
+				embedded_position.each do |ep|
+					embedded_artifact = EmbeddedArtifact.new
+					embedded_artifact.host_artifact = artifact
+					embedded_artifact.position_in_host = ep
+					#puts "<<<#{embedded_code}>>> #{ep}"
+					begin
+						embedded_root = embedded_parser.parse_artifact(embedded_artifact)
+					rescue Exception => e
+						raise "Problem embedded in '#{node}' at #{model.source.position} parsing '#{embedded_artifact.code}', from position #{ep}: #{e.inspect}"
+					end
+					model.addForeign_asts(embedded_root)
+				end
+			end
+		end
+	end
+
+	def translate_many(code,node,model,dest,node_value=(node.send(dest)),artifact)
+		return unless node_value!=nil
+		#puts "Considering #{model.class}.#{dest} (#{node_value.class})"
+		node_value.each do |el|
+			#puts "\t* #{el.class}"
+			model_el = node_to_model(el,code,artifact)
+			model.send(:"add#{dest.to_s.proper_capitalize}", model_el) if model_el!=nil
+		end
+	end
+
+end # class Parser
+
+DefaultParser = Parser.new
+
+def self.parse_artifact(artifact)
+	DefaultParser.parse_artifact(artifact)
+end
+
+def self.parse_code(code)
+	parse_file(code,'<code>')
+end
+
+def self.parse_file(code,filename)
+	parse_artifact(FileArtifact.new(filename,code))
+end
+
+def self.raw_node_tree(code)
+	DefaultParser.raw_node_tree(code)
+end
+
+end
+end

data/lib/codemodels/html/version.rb

@@ -0,0 +1,5 @@
+module CodeModels
+module Html
+	VERSION = "0.1.0"
+end
+end

data/lib/codemodels/html.rb

@@ -0,0 +1,8 @@
+require 'jars/jericho-html-3.3.jar'
+
+require "codemodels/html/version"
+require "codemodels/html/metamodel"
+require "codemodels/html/parser"
+require "codemodels/html/model_building"
+require "codemodels/html/language"
+require "codemodels/html/angular_embedding_rules"

data/test/data/puzzle.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>AngularJS puzzle</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+    <script type="text/javascript" src="js/lib/angular/angular.js"></script>
+    <script type="text/javascript" src="js/puzzle/slidingPuzzle.js"></script>
+	<script type="text/javascript" src="js/puzzle/wordSearchPuzzle.js"></script>
+	<script type="text/javascript" src="js/app.js"></script>
+    <link rel="stylesheet" type="text/css" href="css/styles.css"/>
+</head>
+<body ng-app="puzzleApp">
+	<ul id="types">
+		<li ng-repeat="t in types" ng-class="{'selected': t.id == type}">
+			<a ng-href="#/{{t.id}}">{{t.title}}</a>
+		</li>
+	</ul>
+
+	<div ng-include="type"></div>
+
+	<a id="fork" href="https://github.com/pdanis/angular-puzzle" title="Fork me on GitHub"></a>
+	<a id="powered" href="http://angularjs.org" title="powered by AngularJS"><img src="http://www.angularjs.org/img/AngularJS-large.png"/></a>
+
+	<script type="text/ng-template" id="sliding-puzzle">
+		<fieldset id="sliding-simple">
+			<legend>Basic</legend>
+			<sliding-puzzle size="3x3" src="img/angular.png"></sliding-puzzle>
+		</fieldset><br/>
+
+		<fieldset id="sliding-advanced" ng-controller="slidingAdvancedCtrl">
+			<legend>Advanced</legend>
+			<div ng-repeat="puzzle in puzzles">
+				<h2>{{puzzle.title}}</h2>
+				<div class="src">
+					<input type="text" ng-model="puzzle.src"/>
+				</div>
+				<div class="status">
+					moves: <strong>{{puzzle.api.moves}}</strong><br/>
+					solved: <strong>{{puzzle.api.isSolved()}}</strong>
+				</div>
+				<div class="size">
+					rows: <input type="text" ng-model="puzzle.rows" size="2"/>
+					cols: <input type="text" ng-model="puzzle.cols" size="2"/><br/>
+					<button ng-click="puzzle.api.shuffle()">shuffle</button>
+					<button ng-click="puzzle.api.solve()">solve</button>
+				</div>
+				<sliding-puzzle api="puzzle.api" size="{{puzzle.rows}}x{{puzzle.cols}}" src="{{puzzle.src}}"></sliding-puzzle>
+			</div>
+		</fieldset>
+	</script>
+
+	<script type="text/ng-template" id="word-search-puzzle">
+		<div id="word-search" ng-controller="wordSearchCtrl">
+			<ul class="words">
+				<li ng-repeat="word in puzzle.words" ng-class="{'found': word.found}">
+					{{word.name}}
+				</li>
+			</ul>
+			<word-search-puzzle matrix="matrix" words="words" api="puzzle"></word-search-puzzle>
+			<div class="status">
+				<button ng-click="puzzle.solve()" ng-show="!puzzle.solved" class="solve">solve</button>
+				<strong ng-show="puzzle.solved" ng-show="puzzle.solved">Solved!</strong>
+			</div>
+		</div>
+	</script>
+</body>
+</html>

data/test/test_basic_info.rb

@@ -0,0 +1,67 @@
+require 'test_helper'
+ 
+class TestBasicInfo < Test::Unit::TestCase
+
+	include TestHelper
+	include CodeModels
+	include CodeModels::Html
+
+	def test_source_line
+		code = %q{<html>
+			<body>
+				<p>ciao
+					come
+					stai?</p>
+				<p>io bene</p>
+				<div><p>
+					<span></span>
+				</p></div>
+			</body>
+			</html>}
+		r = Html.parse_code(code)
+		assert_class HtmlDocument, r
+		span = nil
+		r.traverse {|n| span = n if n.is_a?(Node) && n.name=='span'}
+		assert_not_nil span
+		assert_equal 8,span.source.position.begin_point.line
+		assert_class Node, span.eContainer
+		assert_equal 'p',span.eContainer.name
+		assert_equal 7,span.eContainer.source.position.begin_point.line # p
+		assert_equal 'div',span.eContainer.eContainer.name
+		assert_equal 7,span.eContainer.eContainer.source.position.begin_point.line # div
+		assert_equal 'body',span.eContainer.eContainer.eContainer.name
+		assert_equal 2,span.eContainer.eContainer.eContainer.source.position.begin_point.line # body
+	end
+
+	def test_source_line_of_text_elements
+		code = %q{<html>
+			<body>
+				<p>ciao
+					come
+					stai?</p>
+				<p>io bene</p>
+				<div><p>
+					<span></span>
+				</p></div>
+			</body>
+			</html>}
+		r = Html.parse_code(code)
+		assert_class HtmlDocument, r
+		body = nil
+		r.traverse {|n| body = n if n.is_a?(Node) && n.name=='body'}
+		assert_not_nil body
+		first_p = body.all_children[0]
+		first_p_text = first_p.all_children[0]
+		assert_class Text,first_p_text
+		assert_equal 3,first_p_text.source.position.begin_point.line
+		assert_equal 5,first_p_text.source.position.end_point.line
+	end	
+
+	def test_artifact_final_host_is_set_correctly_for_all
+		r = AngularJs.parser_considering_angular_embedded_code.parse_file('test/data/puzzle.html')
+		r.traverse(:also_foreign) do |n|
+			assert_equal 'test/data/puzzle.html',n.source.artifact.final_host.filename, "Node with wrong final_host: #{n}"
+		end
+	end
+
+end

data/test/test_basic_parsing.rb

@@ -0,0 +1,79 @@
+require 'test_helper'
+ 
+class TestBasicParsing < Test::Unit::TestCase
+
+	include TestHelper
+	include CodeModels
+	include CodeModels::Html
+
+	def test_basic_document
+		code = "<html></html>"
+		r = Html.parse_code(code)
+		assert_class HtmlDocument, r
+		assert_equal 1, r.children.count
+		assert_class Node, r.children[0] 	
+		assert_equal 'html', r.children[0].name
+	end
+
+	def test_basic_attributes
+		code = "<html id='ciao'></html>"
+		r = Html.parse_code(code)
+		html = r.children[0] 
+		assert_equal 1, html.attributes.count
+		assert_equal "id", html.attributes[0].name
+		assert_equal "ciao", html.attributes[0].value
+	end	
+
+	def test_basic_text
+		code = "<html>ciao</html>"
+		r = Html.parse_code(code)
+		html = r.children[0] 
+		assert_equal 1, html.children.count
+		assert_class Text, html.children[0]
+		assert_equal 'ciao', html.children[0].value
+	end
+
+	def test_basic_dtd
+		code = "<!DOCTYPE html>"
+		r = Html.parse_code(code)
+		dtd = r.children[0] 
+		assert_class DTD, dtd
+		assert_equal 'html', dtd.name
+	end
+
+	def test_no_text_blocks
+		code = "<head><script type='text/ng-template' id='sliding-puzzle'>\n<a/>\n</script></head>"
+		r = Html.raw_node_tree(code)
+		assert_equal 0,r.child_elements[0].text_blocks(code).count
+	end
+
+	def test_parse_scripts
+		code = "<html><head><script type='text/ng-template' id='sliding-puzzle'>\n<a/>\n</script></head></html>"
+		r = Html.parse_code(code)
+		script = r.children[0].children[0].children[0]		
+		assert_class Script, script
+		assert_class HtmlDocument, script.foreign_asts[0]
+		assert_class Node, script.foreign_asts[0].children[0]
+		assert_equal 'a', script.foreign_asts[0].children[0].name
+	end
+
+	def test_node_content
+		code = "<html><head><script type='text/ng-template' id='sliding-puzzle'>\n<a/>\n</script></head></html>"
+		r = Html.raw_node_tree(code)
+		script = r.child_elements[0].child_elements[0].child_elements[0]		
+		assert_equal "<head><script type='text/ng-template' id='sliding-puzzle'>\n<a/>\n</script></head>",Parser.node_content(r,code)
+		assert_equal "\n<a/>\n",Parser.node_content(script,code)
+	end
+
+	def test_text_blocks
+		code = "<html><head><div type='text/ng-template' id='sliding-puzzle'>ciao<a/>come</div></head></html>"
+		r = Html.raw_node_tree(code)
+		assert_equal 0,r.child_elements[0].text_blocks(code).count
+		assert_equal 0,r.child_elements[0].child_elements[0].text_blocks(code).count
+		div = r.child_elements[0].child_elements[0].child_elements[0]	
+		assert_equal 2,div.text_blocks(code).count
+		assert_equal "ciao",div.text_blocks(code)[0].value
+		assert_equal "come",div.text_blocks(code)[1].value
+	end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,74 @@
+require 'simplecov'
+SimpleCov.start do
+	add_filter "/test/"	
+end
+
+require 'json'
+require 'test/unit'
+require 'codemodels'
+require 'codemodels/html'
+
+module TestHelper
+
+include CodeModels
+
+def assert_metamodel(name,attrs,refs)
+	assert Html.const_defined?(name), "Metaclass '#{name}' not found"
+	c = Html.const_get name
+
+	assert_all_attrs attrs, c
+	assert_all_refs  refs, c	
+	c
+end
+
+def assert_class(expected_class,node)
+	assert node.class==expected_class, "Node expected to have class #{expected_class} instead it has class #{node.class}"
+end
+
+def relative_path(path)
+	File.join(File.dirname(__FILE__),path)
+end
+
+def assert_all_attrs(expected,c)
+	actual = c.ecore.eAllAttributes
+	assert_equal expected.count,actual.count,"Expected #{expected.count} attrs, found #{actual.count}. They are #{actual.name}"
+	expected.each do |e|
+		assert actual.find {|a| a.name==e}, "Attribute #{e} not found"	
+	end
+end
+
+def assert_all_refs(expected,c)
+	actual = c.ecore.eAllReferences
+	assert_equal expected.count,actual.count,"Expected #{expected.count} refs, found #{actual.count}. They are #{actual.name}"
+	expected.each do |e|
+		assert actual.find {|a| a.name==e}, "Reference #{e} not found"	
+	end
+end
+
+def assert_ref(c,name,type,many=false)
+	ref = c.ecore.eAllReferences.find {|r| r.name==name}
+	assert ref, "Reference '#{name}' not found"	
+	assert_equal type.ecore.name,ref.eType.name
+	assert_equal many, ref.many
+end
+
+def assert_attr(c,name,type,many=false)
+	att = c.ecore.eAllAttributes.find {|a| a.name==name}	
+	assert_equal type.name,att.eType.name
+	assert_equal many, att.many
+end
+
+def assert_map(exp,map)
+	assert_equal exp.count,map.count, "Expected to have keys: #{exp.keys}, it has #{map}"
+	exp.each do |k,v|
+		assert_equal exp[k],map[k], "Expected #{k} to have #{exp[k]} instances, it has #{map[k.to_s]}. Map: #{map}"
+	end
+end
+
+def assert_code_map_to(code,exp)
+	r = Html.parse_code(code)
+	map = r.values_map
+	assert_map(exp,map)
+end
+
+end

data/test/test_info_extraction.rb

@@ -0,0 +1,37 @@
+require 'test_helper'
+ 
+class TestInfoExtraction < Test::Unit::TestCase
+
+	include TestHelper
+	include CodeModels
+
+	def test_snippet_1
+		code = %q{
+			<html>
+			<body>
+				<p>ciao!</p>
+			</body>
+			</html>
+		}
+		assert_code_map_to(code, {
+			'html' =>1,
+			'body' =>1,
+			'p' => 1,
+			'ciao!'=> 1
+		})		
+	end
+
+	def test_no_extraneous_values
+		code = IO.read('test/data/puzzle.html')
+		r = Html::AngularJs.parser_considering_angular_embedded_code.parse_code(code)
+		r.traverse(:also_foreign) do |node|
+			node.collect_values_with_count.each do |value,count|
+				node_code = node.source.code
+				unless node_code.include?(value.to_s)
+					fail("Value '#{value}' expected in #{node}. Artifact: #{node.source.artifact}, abspos: #{node.source.position(:absolute)}, code: '#{node_code}'")
+				end
+			end
+		end
+	end
+
+end

data/test/test_monkey_patching.rb

@@ -0,0 +1,20 @@
+require 'test_helper'
+require 'codemodels/js'
+ 
+class TestParsingEmbeddedLanguages < Test::Unit::TestCase
+
+	include TestHelper
+	include CodeModels
+	include CodeModels::Html
+
+
+	def test_begin_index
+		code = "<sliding-puzzle api />"
+		assert_equal 0,code.first_index('<')
+		assert_equal 1,code.first_index('s')
+		assert_equal 8,code.first_index('-')
+		assert_equal 20,code.first_index('/')
+		assert_equal 21,code.first_index('>')
+	end	
+
+end

data/test/test_parsing_embedded_languages.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+require 'codemodels/js'
+ 
+class TestParsingEmbeddedLanguages < Test::Unit::TestCase
+
+	include TestHelper
+	include CodeModels
+	include CodeModels::Html
+
+	def setup
+		@p = AngularJs.parser_considering_angular_embedded_code
+	end
+
+	def test_source_line
+		code = 
+	 %q{<html>
+			<body ng-app="puzzleApp">
+				<ul id="types">
+					<li ng-repeat="t in types" ng-class="{'selected': t.id == type}">
+						<a ng-href="#/{{t.id}}">{{t.title}}</a>
+					</li>
+				</ul>
+			</body>
+		</html>}
+
+		r = @p.parse_code(code)
+		li = r.all_children_deep.find {|n| n.is_a?(Node) && n.name=='li'}
+		assert_not_nil li
+		a = li.attributes.find {|a| a.name=='ng-repeat'}
+		assert_not_nil a
+		assert_equal 1,a.foreign_asts.count
+		assert_class CodeModels::Js::InInfixExpression,a.foreign_asts[0]
+	end
+
+	def test_multiple_angular_expressions_in_attr
+		code = %q{<sliding-puzzle api="puzzle.api" size="{{puzzle.rows}}x{{puzzle.cols}}" src="{{puzzle.src}}"></sliding-puzzle>}
+
+		r = @p.parse_code(code)
+		assert_class HtmlDocument,r
+		assert_class Node,r.children[0]		
+		att_size = r.children[0].attributes.find {|a| a.name=='size'}
+		att_src = r.children[0].attributes.find {|a| a.name=='src'}
+		assert_equal 2,att_size.foreign_asts.count
+		assert_equal 1,att_src.foreign_asts.count
+	end
+
+	def test_parsing_empty_attr
+		code = %q{<sliding-puzzle api />}
+
+		r = @p.parse_code(code)
+		# it does not crash? It is ok!
+	end	
+
+end