trenni 1.7.0 → 2.0.1

data/lib/trenni/fallback/template.rl

@@ -0,0 +1,97 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+%%{
+	machine template;
+	
+	action instruction_begin {
+		instruction_begin = p
+	}
+	
+	action instruction_end {
+		instruction_end = p
+	}
+	
+	action emit_instruction {
+		delegate.instruction(data.byteslice(instruction_begin...instruction_end))
+	}
+	
+	action emit_instruction_line {
+		delegate.instruction(data.byteslice(instruction_begin...instruction_end), "\n")
+	}
+	
+	action instruction_error {
+		raise ParseError.new("failed to parse instruction", buffer, p)
+	}
+	
+	action expression_begin {
+		expression_begin = p
+	}
+	
+	action expression_end {
+		expression_end = p
+	}
+	
+	action emit_expression {
+		delegate.expression(data.byteslice(expression_begin...expression_end))
+	}
+	
+	action expression_error {
+		raise ParseError.new("failed to parse expression", buffer, p)
+	}
+	
+	action emit_text {
+		delegate.text(data.byteslice(ts...te))
+	}
+	
+	# This magic ensures that we process bytes.
+	getkey bytes[p];
+	
+	include template "trenni/template.rl";
+}%%
+
+require_relative '../parse_error'
+
+module Trenni
+	module Fallback
+		%% write data;
+		
+		def self.parse_template(buffer, delegate)
+			data = buffer.read
+			bytes = data.bytes
+			
+			p = 0
+			pe = eof = data.bytesize
+			stack = []
+			
+			expression_begin = expression_end = nil
+			instruction_begin = instruction_end = nil
+			
+			%% write init;
+			%% write exec;
+			
+			if p != eof
+				raise ParseError.new("could not consume all input", buffer, p)
+			end
+			
+			return nil
+		end
+	end
+end

data/lib/trenni/markup.rb

@@ -0,0 +1,76 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative 'substitutions'
+
+module Trenni
+	# A wrapper which indicates that `value` can be appended to the output buffer without any changes.
+	module Markup
+		# Generates a string suitable for concatenating with the output buffer.
+		def self.escape(value)
+			if value.is_a? Markup
+				value
+			elsif value
+				MarkupString.new(value.to_s)
+			else
+				# String#<< won't accept nil, so we return an empty string, thus ensuring a fixed point function:
+				EMPTY
+			end
+		end
+		
+		def escape(value)
+			Markup.escape(value)
+		end
+	end
+	
+	# Initialized from text which is escaped to use HTML entities.
+	class MarkupString < String
+		include Markup
+		
+		# This is only casually related to HTML, it's just enough so that it would not be mis-interpreted by `Trenni::Parser`.
+		ESCAPE = Substitutions.new("&" => "&amp;", "<" => "&lt;", ">" => "&gt;", "\"" => "&quot;")
+		
+		# Convert ESCAPE characters into their corresponding entities.
+		def initialize(string = nil, escape = true)
+			if string
+				super(string)
+				ESCAPE.gsub!(self) if escape
+			else
+				super()
+			end
+		end
+		
+		def self.raw(string)
+			self.new(string, false)
+		end
+	end
+	
+	module Script
+		def self.json(value)
+			MarkupString.new(JSON.dump(value), false)
+		end
+	end
+	
+	def self.MarkupString(value)
+		Markup.escape(value)
+	end
+	
+	Markup::EMPTY = String.new.extend(Markup).freeze
+end

data/lib/trenni/native.rb

@@ -0,0 +1,28 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative 'parse_error'
+
+begin
+	# Load native code:
+	require_relative 'trenni'
+rescue LoadError
+	warn "Could not load native implementation: #{$!}" if $VERBOSE
+end

data/lib/trenni/parse_delegate.rb

@@ -0,0 +1,34 @@
+# Copyright, 2012, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Trenni
+	# This is a sample delegate for capturing all events. It's only use is for testing.
+	class ParseDelegate
+		def initialize
+			@events = []
+		end
+		
+		attr :events
+		
+		def method_missing(*args)
+			@events << args
+		end
+	end
+end

data/lib/trenni/{scanner.rb → parse_error.rb}

@@ -20,37 +20,24 @@
 
 require_relative 'buffer'
 
-require 'strscan'
-
 module Trenni
 	class ParseError < StandardError
-		def initialize(message, scanner, positions = nil)
+		def initialize(message, buffer, offset)
 			super(message)
 			
-			@path = scanner.path
-			
-			@locations = []
-			
-			if positions
-				positions.each do |position|
-					@locations << Location.new(scanner.string, position)
-				end
-			else
-				@locations = [Location.new(scanner.string, scanner.pos)]
-			end
-			
-			@location = @locations.first
-			
-			@input_name = nil
+			@buffer = buffer
+			@offset = offset
 		end
 		
-		attr :locations
-		attr :location
+		def location
+			@location ||= Location.new(@buffer.read, @offset)
+		end
 		
+		attr :buffer
 		attr :path
 		
 		def to_s
-			"#{@path}#{@location}: #{super}\n#{location.line_text}"
+			"#{buffer.path}#{location}: #{super}\n#{location.line_text}"
 		end
 	end
 	
@@ -82,7 +69,7 @@ module Trenni
 		end
 		
 		def to_s
-			"[#{self.line_number}]"
+			"[#{self.line_number}:#{self.line_offset}]"
 		end
 		
 		# The line that contains the @offset (base 0 indexing).
@@ -103,36 +90,4 @@ module Trenni
 		
 		attr :line_text
 	end
-	
-	class StringScanner < ::StringScanner
-		def initialize(buffer)
-			@buffer = buffer
-			
-			super(buffer.read)
-		end
-		
-		attr :buffer
-		
-		def path
-			@buffer.path
-		end
-		
-		STUCK_MESSAGE = "Parser is stuck!".freeze
-		
-		def stuck?(position)
-			self.pos == position
-		end
-			
-		def raise_if_stuck(position, message = STUCK_MESSAGE)
-			if stuck?(position)
-				parse_error!(message)
-			end
-		end
-		
-		def parse_error!(message, positions = nil)
-			positions ||= [self.pos]
-			
-			raise ParseError.new(message, self, positions)
-		end
-	end
 end

data/lib/trenni/parsers.rb

@@ -0,0 +1,12 @@
+
+require_relative 'native'
+require_relative 'parse_delegate'
+
+if defined? Trenni::Native and !ENV['TRENNI_PREFER_FALLBACK']
+	Trenni::Parsers = Trenni::Native
+else
+	require_relative 'fallback/markup'
+	require_relative 'fallback/template'
+	
+	Trenni::Parsers = Trenni::Fallback
+end

data/lib/trenni/substitutions.rb

@@ -0,0 +1,45 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Trenni
+	class Substitutions
+		def initialize(tokens)
+			@tokens = tokens
+		end
+		
+		def patterns
+			@tokens.keys
+		end
+		
+		def pattern
+			@pattern ||= Regexp.union(patterns)
+		end
+		
+		attr :tokens
+		
+		def gsub!(string)
+			string.gsub!(pattern) {|match| @tokens[match]}
+		end
+		
+		def gsub(string)
+			string.gsub(pattern) {|match| @tokens[match]}
+		end
+	end
+end

data/lib/trenni/template.rb

@@ -18,197 +18,114 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-require_relative 'scanner'
+require_relative 'parsers'
 
 module Trenni
 	# The output variable that will be used in templates:
-	OUT = '_out'
+	OUT = :_out
 	
 	class Template
 		# Returns the output produced by calling the given block.
 		def self.capture(*args, &block)
-			out = eval(OUT, block.binding)
-			top = out.size
+			scope = block.binding
+			output_buffer = scope.local_variable_get(OUT)
 			
-			block.call(*args)
+			capture_buffer = String.new.force_encoding(output_buffer.encoding)
+			scope.local_variable_set(OUT, capture_buffer)
 			
-			return out.pop(out.size - top).join
+			begin
+				block.call(*args)
+			ensure
+				scope.local_variable_set(OUT, output_buffer)
+			end
+			
+			return capture_buffer
 		end
 		
 		# Returns the buffer used for capturing output.
 		def self.buffer(binding)
-			eval(OUT, binding)
+			binding.local_variable_get(OUT)
 		end
 		
 		class Assembler
-			def initialize
-				@parts = []
+			def initialize(filter: String, encoding: Encoding::UTF_8)
+				@code = String.new.force_encoding(encoding)
+				@filter = filter
 			end
 
-			attr :parts
+			attr :code
 
 			# Output raw text to the template.
 			def text(text)
 				text = text.gsub("'", "\\\\'")
-				@parts << "#{OUT}<<'#{text}';"
+				@code << "#{OUT}<<'#{text}';"
 				
 				# This is an interesting approach, but it doens't preserve newlines or tabs as raw characters, so template line numbers don't match up.
 				# @parts << "#{OUT}<<#{text.dump};"
 			end
 
 			# Output a ruby expression (or part of).
-			def expression(text)
-				@parts << "#{text};"
+			def instruction(text, postfix = nil)
+				@code << text << (postfix || ';')
 			end
 			
 			# Output a string interpolation.
-			def interpolation(text)
-				@parts << "#{OUT}<<(#{text});"
-			end
-
-			CODE_PREFIX = "#{OUT}=[];".freeze
-			CODE_POSTFIX = "#{OUT}".freeze
-
-			def code
-				return [CODE_PREFIX, *@parts, CODE_POSTFIX].join
+			def expression(text)
+				# Double brackets are required here to handle expressions like #{foo rescue "bar"}.
+				@code << "#{OUT}<<#{@filter}((#{text}));"
 			end
 		end
 		
-		class Scanner < StringScanner
-			def initialize(buffer, delegate)
-				super(buffer)
-				
-				@delegate = delegate
-			end
-			
-			def parse!
-				until eos?
-					start_pos = self.pos
-
-					scan_text
-					scan_expression or scan_interpolation
-					
-					raise_if_stuck(start_pos)
-				end
-			end
-
-			# This is formulated specifically so that it matches up until the start of a code block.
-			TEXT = /([^<#]|<(?!\?r)|#(?!\{)){1,}/m
-
-			def scan_text
-				if scan(TEXT)
-					@delegate.text(self.matched)
-				end
-			end
-
-			def scan_expression
-				start_pos = self.pos
-				
-				if scan(/<\?r/)
-					if scan_until(/(.*?)\?>/m)
-						@delegate.expression(self[1])
-					else
-						parse_error!("Could not find end of expression!", [start_pos, self.pos])
-					end
-					
-					return true
-				end
-				
-				return false
-			end
-
-			def scan_interpolation
-				start_pos = self.pos
-				
-				if scan(/\#\{/)
-					level = 1
-					code = String.new
-
-					until eos?
-						current_pos = self.pos
-						
-						# Scan anything other than something which causes nesting:
-						if scan(/[^"'\{\}]+/m)
-							code << matched
-						end
-						
-						# Scan a quoted string:
-						if scan(/'(\\'|[^'])*'/m) or scan(/"(\\"|[^"])*"/m)
-							code << matched
-						end
-						
-						# Scan something which nests:
-						if scan(/\{/)
-							code << matched
-							level += 1
-						end
-
-						if scan(/\}/)
-							level -= 1
-							if level == 0
-								@delegate.interpolation(code)
-								return true
-							else
-								code << matched
-							end
-						end
-						
-						break if stuck?(current_pos)
-					end
-					
-					parse_error!("Could not find end of interpolation!", [start_pos, self.pos])
-				end
-				
-				return false
-			end
+		def self.load_file(path, **options)
+			self.new(FileBuffer.new(path), **options)
 		end
 
-		def self.load_file(path)
-			self.new(FileBuffer.new(path))
-		end
-
-		def initialize(buffer)
+		def initialize(buffer, filter: String)
 			@buffer = buffer
+			@filter = filter
 		end
 
-		def to_string(scope = Object.new)
-			to_array(scope).join
+		def to_string(scope = Object.new, output = output_buffer)
+			output ||= output_buffer
+			
+			scope.instance_exec(output, &to_proc)
 		end
 		
 		def to_buffer(scope)
-			Buffer.new(to_array(scope).join, path: @buffer.path)
+			Buffer.new(to_string(scope), path: @buffer.path)
 		end
 
-		# Legacy functions:
-		alias evaluate to_string
-		alias result to_string
-
-		def to_array(scope)
-			if Binding === scope
-				# Slow code path, evaluate the code string in the given binding (scope).
-				eval(code, scope, @buffer.path)
-			else
-				# Faster code path, use instance_eval on a compiled Proc.
-				scope.instance_eval(&to_proc)
-			end
-		end
-		
-		def to_proc
-			@compiled_proc ||= eval("proc{;#{code};}", binding, @buffer.path)
+		def to_proc(scope = nil)
+			@compiled_proc ||= eval("proc{|#{OUT}|;#{code};#{OUT}}", scope, @buffer.path)
 		end
 		
 		protected
 		
+		def output_buffer
+			String.new.force_encoding(code.encoding)
+		end
+		
 		def code
 			@code ||= compile!
 		end
 		
 		def compile!
-			assembler = Assembler.new
+			assembler = Assembler.new(filter: @filter)
 			
-			Scanner.new(@buffer, assembler).parse!
+			Parsers.parse_template(@buffer, assembler)
 			
 			assembler.code
 		end
 	end
+	
+	class MarkupTemplate < Template
+		def initialize(buffer, filter: MarkupString)
+			super
+		end
+		
+		# The output of the markup template is encoded markup (e.g. with entities, tags, etc)
+		def output_buffer
+			MarkupString.new.force_encoding(code.encoding)
+		end
+	end
 end