slamdown 0.5.0

data/lib/slamdown/conversion/prepared_node.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Captures parser facts together with the immutable, conversion-specific decisions needed by normalisation and rendering.
+class PreparedNode
+	PARSER_ATTRIBUTES = [:kind, :source_name, :semantic_kind, :category, :content_model, :value, :properties, :source_form, :synthetic].freeze
+	PREPARATION_ATTRIBUTES = [:action, :attributes, :attribute_policies, :classes, :class_policies, :id, :id_policy, :semantic_values, :children].freeze
+	ATTRIBUTES = (PARSER_ATTRIBUTES + PREPARATION_ATTRIBUTES).freeze
+
+	attr_reader(*ATTRIBUTES)
+
+	def initialize(values)
+		ATTRIBUTES.each do |attribute|
+			instance_variable_set("@#{attribute}", Immutable.copy(values.fetch(attribute)))
+		end
+		freeze
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/preparer.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Produces a fresh prepared tree by combining one parsed tree with one validated conversion configuration.
+class Preparer
+	HTML_WHITESPACE = /[\t\n\f\r ]+/.freeze
+
+	class << self
+		def prepare(root, configuration)
+			new(configuration).prepare(root)
+		end
+	end
+
+	def initialize(configuration)
+		raise ArgumentError, "configuration must be a Slamdown conversion configuration" unless configuration.is_a?(Configuration)
+
+		@policy = Policy::Resolver.new(configuration)
+		@semantic_extractor = SemanticExtractor.new
+	end
+
+	def prepare(root)
+		raise ArgumentError, "root must be a parsed Slamdown node" unless root.is_a?(Parsing::Node)
+
+		prepare_node(root)
+	end
+
+	private
+
+	def prepare_node(node)
+		semantic_values = @semantic_extractor.extract(node)
+		attributes, attribute_policies = prepare_attributes(node, semantic_values)
+		classes, class_policies = prepare_classes(node, semantic_values)
+		id, id_policy = prepare_id(node)
+
+		values = parser_values(node).merge(
+			:action => @policy.action_for(node),
+			:attributes => attributes,
+			:attribute_policies => attribute_policies,
+			:classes => classes,
+			:class_policies => class_policies,
+			:id => id,
+			:id_policy => id_policy,
+			:semantic_values => semantic_values,
+			:children => node.children.map { |child| prepare_node(child) }
+		)
+
+		PreparedNode.new(values)
+	end
+
+	def parser_values(node)
+		PreparedNode::PARSER_ATTRIBUTES.each_with_object({}) do |attribute, result|
+			result[attribute] = node.public_send(attribute)
+		end
+	end
+
+	def prepare_attributes(node, semantic_values)
+		effective_attributes = node.attributes.reject { |name, _value| ["class", "id"].include?(name) }
+		if semantic_values.key?(:image_fallback_alt)
+			effective_attributes = effective_attributes + [["alt", semantic_values[:image_fallback_alt]]]
+		end
+
+		policies = {}
+		retained = effective_attributes.each_with_object([]) do |(name, value), result|
+			decision = @policy.attribute_for(node, name)
+			policies[name] = decision if decision
+			result << [name, decode_attribute(value)] if decision && decision.value == :preserve
+		end
+
+		[retained, policies]
+	end
+
+	def prepare_classes(node, semantic_values)
+		value = Hash[node.attributes]["class"]
+		return [[], {}] unless value.is_a?(String)
+
+		tokens = value.split(HTML_WHITESPACE).each_with_object([]) do |token, result|
+			result << token unless token.empty? || result.include?(token)
+		end
+		policies = {}
+		retained = tokens.each_with_object([]) do |token, result|
+			decision = @policy.class_for(node, token)
+			policies[token] = decision
+			consumed = semantic_values.key?(:code_language) && @semantic_extractor.consumed_class?(node, token)
+			result << token if decision.allowed? && !consumed
+		end
+
+		[retained, policies]
+	end
+
+	def prepare_id(node)
+		value = Hash[node.attributes]["id"]
+		return [nil, nil] unless value.is_a?(String)
+
+		decision = @policy.id_for(node, value)
+		valid = !value.empty? && !HTML_WHITESPACE.match(value)
+		[valid && decision.allowed? ? value : nil, decision]
+	end
+
+	def decode_attribute(value)
+		return value unless value.is_a?(String)
+
+		value.gsub(/&(?:#(\d+)|#x([0-9A-Fa-f]+)|([A-Za-z][A-Za-z0-9]+));/) do |reference|
+			begin
+				code_point = if $1
+					$1.to_i
+				elsif $2
+					$2.to_i(16)
+				else
+					Kramdown::Utils::Entities.entity($3).code_point
+				end
+				[code_point].pack("U")
+			rescue Kramdown::Error, RangeError
+				reference
+			end
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/renderer.rb

@@ -0,0 +1,432 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Traverses the normalised tree through shared inline and block primitives, delegating only genuine format differences to a strategy.
+class Renderer
+	ENTITY_KINDS = [:entity, :typographic_symbol, :smart_quote].freeze
+	LIST_KINDS = [:ordered_list, :unordered_list].freeze
+	FAILED_RENDERING = Object.new.freeze
+
+	class << self
+		def render(root, configuration, format)
+			new(configuration, format).render(root)
+		end
+	end
+
+	def initialize(configuration, format)
+		raise ArgumentError, "configuration must be a Slamdown conversion configuration" unless configuration.is_a?(Configuration)
+		raise ArgumentError, "format must be markdown or kramdown" unless ["markdown", "kramdown"].include?(format)
+
+		@configuration = configuration
+		@escaper = Rendering::Escaper.new
+		@entity_encoder = Rendering::EntityEncoder.new(configuration.entities)
+		@html = Rendering::Html.new(configuration, @entity_encoder)
+		@strategy = Rendering::Strategy.build(format)
+		@format = format
+		@diagnostics = []
+	end
+
+	def render(root)
+		raise ArgumentError, "root must be a normalised Slamdown document" unless root.is_a?(NormalisedNode) && root.kind == :document
+		@diagnostics = []
+
+		Rendering::Result.new(canonicalise(render_blocks(root.children)), @diagnostics)
+	end
+
+	# Renders a block sequence with exactly one blank line between neighbouring constructs.
+	def render_blocks(nodes)
+		parts = []
+		inline_run = []
+		flush = lambda do
+			unless inline_run.empty?
+				value = render_inlines(inline_run)
+				parts << value unless value.empty?
+				inline_run = []
+			end
+		end
+
+		nodes.each do |node|
+			if block_node?(node)
+				flush.call
+				value = safely_render(node) { render_block(node) }
+				parts << value unless value.equal?(FAILED_RENDERING) || value.empty?
+			else
+				inline_run << node
+			end
+		end
+		flush.call
+		parts.join("\n\n")
+	end
+
+	# Renders the children of structural containers whose parser shape may be inline or block-oriented.
+	def render_container_contents(node)
+		node.children.any? { |child| block_node?(child) } ? render_blocks(node.children) : render_inlines(node.children)
+	end
+
+	# Renders one inline sequence, with optional strong-marker suppression used by Markdown definition terms.
+	def render_inlines(nodes, suppress_strong: false, line_start: true)
+		result = ""
+		failed_rendering = false
+		nodes.each do |node|
+			value = safely_render(node) { render_inline(node, line_start && result.empty?, suppress_strong) }
+			if value.equal?(FAILED_RENDERING)
+				failed_rendering = true
+				next
+			end
+
+			# Dropping a failed inline node can make its surrounding text spaces newly adjacent, so restore the already-normalised boundary.
+			if failed_rendering
+				value = value.sub(/\A +/, "") if result.empty? || (result.end_with?(" ") && value.start_with?(" "))
+				failed_rendering = false
+			end
+			result += value
+		end
+		failed_rendering ? result.sub(/ +\z/, "") : result
+	end
+
+	private
+
+	def render_block(node)
+		return @html.render(node) if node.representation == :html
+
+		case node.semantic_kind
+		when :paragraph
+			@strategy.decorate_block(node, render_inlines(node.children))
+		when :heading
+			content = render_inlines(node.children, :line_start => false)
+			@strategy.decorate_block(node, "#{'#' * node.properties.fetch(:level)} #{content}")
+		when :blockquote
+			@strategy.decorate_block(node, render_blockquote(node))
+		when :ordered_list
+			@strategy.decorate_block(node, render_list(node, true))
+		when :unordered_list
+			@strategy.decorate_block(node, render_list(node, false))
+		when :definition_list
+			@strategy.render_definition_list(self, node)
+		when :horizontal_rule
+			@strategy.decorate_block(node, "---")
+		when :code_block
+			@strategy.decorate_block(node, render_code_block(node))
+		when :table
+			render_table(node)
+		else
+			render_container_contents(node)
+		end
+	end
+
+	def render_inline(node, line_start, suppress_strong)
+		return @escaper.text(node.value, :line_start => line_start, :escape_definition_marker => @strategy.escape_definition_marker?) if node.kind == :text
+		return render_entity(node, line_start) if ENTITY_KINDS.include?(node.kind)
+		return @html.render(node) if node.representation == :html
+
+		case node.semantic_kind
+		when :emphasis
+			content = render_inlines(node.children, :suppress_strong => suppress_strong, :line_start => false)
+			decorate_inline(node, "*#{content}*")
+		when :strong
+			content = render_inlines(node.children, :suppress_strong => suppress_strong, :line_start => false)
+			suppress_strong ? content : decorate_inline(node, "**#{content}**")
+		when :deletion, :line_break
+			@html.render(node)
+		when :code_span
+			content = node.value.to_s.strip
+			content.empty? ? "" : decorate_inline(node, render_code_span(content))
+		when :link
+			decorate_inline(node, render_link(node, suppress_strong))
+		when :image
+			decorate_inline(node, render_image(node))
+		else
+			if node.value.is_a?(String) && node.children.empty?
+				@escaper.text(node.value, :line_start => line_start, :escape_definition_marker => @strategy.escape_definition_marker?)
+			else
+				render_inlines(node.children, :suppress_strong => suppress_strong, :line_start => line_start)
+			end
+		end
+	end
+
+	def decorate_inline(node, syntax)
+		@strategy.decorate_inline(node, syntax)
+	end
+
+	def render_entity(node, line_start)
+		value = @entity_encoder.encode(node)
+		return value unless @entity_encoder.character_mode?
+
+		@escaper.text(value, :line_start => line_start, :escape_definition_marker => @strategy.escape_definition_marker?)
+	end
+
+	def render_link(node, suppress_strong)
+		attributes = Hash[node.attributes]
+		content = render_inlines(node.children, :suppress_strong => suppress_strong, :line_start => false)
+		destination = @escaper.destination(attributes.fetch("href"), :escape_ampersand => @strategy.escape_link_ampersands?)
+		"[#{content}](#{destination}#{render_title(attributes)})"
+	end
+
+	def render_image(node)
+		attributes = Hash[node.attributes]
+		alt = @escaper.label(attributes.fetch("alt", ""))
+		destination = @escaper.destination(attributes.fetch("src"), :escape_ampersand => @strategy.escape_link_ampersands?)
+		"![#{alt}](#{destination}#{render_title(attributes)})"
+	end
+
+	def render_title(attributes)
+		title = attributes["title"]
+		return "" unless title.is_a?(String) && !title.strip.empty?
+
+		" \"#{@escaper.title(title.strip, :escape_ampersand => @strategy.escape_link_ampersands?)}\""
+	end
+
+	def render_code_span(value)
+		content = value.gsub("\r\n", "\n").gsub("\r", "\n")
+		content = content.gsub(/\s+/, " ") if @configuration.inline
+		longest_run = content.scan(/`+/).map(&:length).max || 0
+		delimiter = "`" * [longest_run + 1, 1].max
+		padding_needed = content.start_with?("`") || content.end_with?("`") || (@strategy.pad_space_bounded_code_span? && surrounded_by_spaces?(content))
+		padding_needed ? "#{delimiter} #{content} #{delimiter}" : "#{delimiter}#{content}#{delimiter}"
+	end
+
+	def surrounded_by_spaces?(content)
+		content.start_with?(" ") && content.end_with?(" ") && !content.strip.empty?
+	end
+
+	def render_code_block(node)
+		content = node.value.to_s.gsub("\r\n", "\n").gsub("\r", "\n")
+		longest_run = content.scan(/~+/).map(&:length).max || 0
+		fence = "~" * [longest_run + 1, 3].max
+		language = node.semantic_values[:code_language].to_s
+		opening = language.empty? ? fence : fence + language
+		body = content.empty? || content.end_with?("\n") ? content : content + "\n"
+		"#{opening}\n#{body}#{fence}"
+	end
+
+	# Selects native pipe syntax only after structure, cell content and final padded width have proved representable.
+	def render_table(node)
+		analysis = Tables::Analyser.analyse(node)
+		if analysis.captions_merged?
+			@diagnostics << Diagnostic.new(
+				:code => :duplicate_table_captions_merged,
+				:severity => :info,
+				:message => "Merged duplicate plain table captions",
+				:node_path => node.source_path
+			)
+		end
+		html_table = analysis.table_for(@format)
+		return @html.render(html_table) if @configuration.tables[:format] == :html || analysis.html_required?
+
+		layout = analysis.layout(@format)
+		if layout.complexity_limited?
+			@diagnostics << Diagnostic.new(
+				:code => :table_complexity_limited,
+				:severity => :warning,
+				:message => table_complexity_message(layout.complexity_reason),
+				:node_path => node.source_path
+			)
+			return @html.render(html_table)
+		end
+		rows = render_table_rows(layout)
+		caption = render_table_caption(analysis.caption)
+		return @html.render(html_table) unless native_table?(layout, rows) && !caption.nil?
+		if layout.ragged_padded?
+			@diagnostics << Diagnostic.new(
+				:code => :ragged_table_padded,
+				:severity => :info,
+				:message => "Padded incomplete table rows with empty cells",
+				:node_path => node.source_path
+			)
+		end
+
+		# Native table syntax consumes structural classes and IDs because neither output dialect can attach them without changing the table representation.
+		syntax = render_pipe_table(layout, rows)
+		caption.empty? ? syntax : "#{caption}\n\n#{syntax}"
+	end
+
+	def table_complexity_message(reason)
+		detail = {
+			:row_count => "more than #{Tables::MAX_ROWS} rows",
+			:column_count => "more than #{Tables::MAX_COLUMNS} columns",
+			:grid_positions => "more than #{Tables::MAX_GRID_POSITIONS} expanded grid positions",
+			:span => "a row or column span greater than #{Tables::MAX_SPAN}"
+		}.fetch(reason)
+		"Rendered a complex table as canonical HTML because it contained #{detail}"
+	end
+
+	def render_table_rows(layout)
+		layout.rows.map do |row|
+			row.map { |cell| render_table_cell(cell) }
+		end
+	end
+
+	def render_table_cell(cell)
+		return "" if cell.placeholder?
+
+		children = cell.node.children
+		value = if children.length == 1 && children.first.semantic_kind == :paragraph
+			render_inlines(children.first.children, :line_start => false)
+		elsif children.none? { |child| block_node?(child) }
+			render_inlines(children, :line_start => false)
+		end
+		return unless value && !value.include?("\n")
+
+		value.gsub(/(?<!\\)\|/) { "\\|" }
+	end
+
+	def native_table?(layout, rows)
+		return false unless layout.rectangular? && rows.flatten.none?(&:nil?)
+		if @format == "markdown"
+			return false unless layout.head_rows.length == 1 && layout.body_groups.length == 1 && layout.foot_rows.empty?
+		end
+		if @format == "kramdown" && layout.body_groups.empty? && (layout.head_rows.any? || layout.foot_rows.any?)
+			return false
+		end
+
+		widths = table_column_widths(rows, table_has_separators?(layout))
+		max_width = @configuration.tables[:max_width]
+		max_width.nil? || pipe_row_width(widths) <= max_width
+	end
+
+	def render_pipe_table(layout, rows)
+		row_index = 0
+		head = rows.slice(row_index, layout.head_rows.length)
+		row_index += layout.head_rows.length
+		bodies = layout.body_groups.map do |group|
+			value = rows.slice(row_index, group.length)
+			row_index += group.length
+			value
+		end
+		foot = rows.slice(row_index, layout.foot_rows.length)
+		widths = table_column_widths(rows, table_has_separators?(layout))
+		lines = head.map { |row| pipe_row(row, widths) }
+		lines << separator_row(widths, "-") if !head.empty? && !bodies.empty?
+		bodies.each_with_index do |body, index|
+			lines << separator_row(widths, "-") if index > 0
+			lines.concat(body.map { |row| pipe_row(row, widths) })
+		end
+		unless foot.empty?
+			lines << separator_row(widths, "=")
+			lines.concat(foot.map { |row| pipe_row(row, widths) })
+		end
+		lines.join("\n")
+	end
+
+	def table_has_separators?(layout)
+		(!layout.head_rows.empty? && !layout.body_groups.empty?) || layout.body_groups.length > 1 || !layout.foot_rows.empty?
+	end
+
+	def table_column_widths(rows, separators)
+		(0...rows.first.length).map do |column|
+			width = rows.map { |row| row[column].length }.max
+			separators ? [width, 3].max : width
+		end
+	end
+
+	def pipe_row(values, widths)
+		"| " + values.each_with_index.map { |value, index| value.ljust(widths[index]) }.join(" | ") + " |"
+	end
+
+	def separator_row(widths, marker)
+		pipe_row(widths.map { |width| marker * width }, widths)
+	end
+
+	def pipe_row_width(widths)
+		widths.inject(1) { |total, width| total + width + 3 }
+	end
+
+	def render_table_caption(caption)
+		return "" unless caption
+
+		children = caption.children
+		content = if children.length == 1 && children.first.semantic_kind == :paragraph
+			render_inlines(children.first.children, :line_start => false)
+		elsif children.none? { |child| block_node?(child) }
+			render_inlines(children, :line_start => false)
+		end
+		return unless content && !content.include?("\n")
+
+		content = content.strip
+		content.empty? ? "" : content
+	end
+
+	def render_blockquote(node)
+		render_blocks(node.children).split("\n", -1).map do |line|
+			line.empty? ? ">" : "> #{line}"
+		end.join("\n")
+	end
+
+	def render_list(node, ordered)
+		items = node.children.select { |child| child.semantic_kind == :list_item }
+		loose = items.any? { |item| loose_list_item?(item) }
+		start = ordered ? node.properties.fetch(:start, 1) : nil
+		items.each_with_index.map do |item, index|
+			marker = ordered ? "#{start + index}. " : "* "
+			render_list_item(item, marker)
+		end.join(ordered && loose ? "\n\n" : "\n")
+	end
+
+	def render_list_item(item, marker)
+		children = item.children
+		return marker.rstrip if children.empty?
+
+		first = render_item_child(children.first)
+		result = prefix_lines(first, marker, " " * marker.length)
+		tight = tight_list_item?(item)
+		children[1..-1].to_a.each do |child|
+			separator = tight && LIST_KINDS.include?(child.semantic_kind) ? "\n" : "\n\n"
+			result += separator + indent_lines(render_item_child(child), marker.length)
+		end
+		result
+	end
+
+	def render_item_child(child)
+		value = safely_render(child) do
+			block_node?(child) ? render_block(child) : render_inline(child, true, false)
+		end
+		value.equal?(FAILED_RENDERING) ? "" : value
+	end
+
+	def tight_list_item?(item)
+		children = item.children
+		return true if children.length == 1 && children.first.semantic_kind == :paragraph
+
+		children.length == 2 && children.first.semantic_kind == :paragraph && LIST_KINDS.include?(children.last.semantic_kind)
+	end
+
+	def loose_list_item?(item)
+		!tight_list_item?(item)
+	end
+
+	def prefix_lines(value, first_prefix, continuation_prefix)
+		value.split("\n", -1).each_with_index.map do |line, index|
+			prefix = index.zero? ? first_prefix : continuation_prefix
+			line.empty? ? prefix.rstrip : prefix + line
+		end.join("\n")
+	end
+
+	def indent_lines(value, width)
+		prefix_lines(value, " " * width, " " * width)
+	end
+
+	def block_node?(node)
+		node.category == :block || Normaliser::BLOCK_SEMANTICS.include?(node.semantic_kind)
+	end
+
+	def safely_render(node)
+		yield
+	rescue StandardError => error
+		@diagnostics << Diagnostic.new(
+			:code => :conversion_error,
+			:severity => :error,
+			:message => "Unable to convert node: #{error.message}",
+			:node_path => node.source_path
+		)
+		FAILED_RENDERING
+	end
+
+	def canonicalise(value)
+		value.gsub("\r\n", "\n").gsub("\r", "\n").sub(/\A\n+/, "").sub(/\n+\z/, "")
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/rendering/entity_encoder.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+module Rendering
+
+# Converts parsed entity-like leaves according to the selected symbolic, character or numeric output policy.
+class EntityEncoder
+	def initialize(mode)
+		@mode = mode
+	end
+
+	def character_mode?
+		@mode == :character
+	end
+
+	def character(node)
+		entity_values(node)[:character]
+	end
+
+	def encode(node)
+		values = entity_values(node)
+		case @mode
+		when :character
+			values[:character]
+		when :numeric
+			"&##{values[:code_point]};"
+		else
+			values[:name] ? "&#{values[:name]};" : "&##{values[:code_point]};"
+		end
+	end
+
+	private
+
+	def entity_values(node)
+		return node.value if node.kind == :entity && node.value.is_a?(Hash)
+
+		entity = Kramdown::Utils::Entities.entity(node.value.to_s)
+		{
+			:code_point => entity.code_point,
+			:name => entity.name,
+			:character => entity.char
+		}
+	rescue Kramdown::Error
+		value = node.value.to_s
+		{
+			:code_point => value.ord,
+			:name => nil,
+			:character => value
+		}
+	end
+end
+
+end
+end
+end

data/lib/slamdown/conversion/rendering/escaper.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+module Rendering
+
+# Escapes generated Markdown contextually so source text cannot become unintended block or inline syntax when reparsed.
+class Escaper
+	INLINE_PUNCTUATION = /[*_\[\]`|]/.freeze
+	SPACE_OR_CONTROL = /[\x00-\x20]/.freeze
+
+	def text(value, line_start: false, escape_definition_marker: false)
+		result = value.to_s.gsub("\\", "\\\\").gsub("<", "&lt;").gsub(">", "&gt;")
+		result = result.gsub(INLINE_PUNCTUATION) { |character| "\\#{character}" }
+		result = result.gsub(/\{(?=:)/, "\\{")
+		line_start ? escape_block_start(result, escape_definition_marker) : result
+	end
+
+	def label(value)
+		text(value.to_s.gsub(/[\r\n]+/, " "))
+	end
+
+	def destination(value, escape_ampersand: true)
+		value.to_s.each_char.map do |character|
+			if SPACE_OR_CONTROL.match(character) || ["<", ">"].include?(character)
+				character.bytes.map { |byte| "%%%02X" % byte }.join
+			elsif character == "&" && escape_ampersand
+				"&amp;"
+			elsif ["\\", "(", ")"].include?(character)
+				"\\#{character}"
+			else
+				character
+			end
+		end.join
+	end
+
+	def title(value, escape_ampersand: true)
+		result = value.to_s.gsub(/[\r\n]+/, " ").gsub("\\", "\\\\")
+		result = result.gsub("&", "&amp;") if escape_ampersand
+		result.gsub("<", "\\<").gsub('"', '\\"')
+	end
+
+	private
+
+	def escape_block_start(value, escape_definition_marker)
+		return value.sub(/\A#/, "\\#") if /\A#+(?=\s)/.match(value)
+		return value.sub(/\A>/, "\\>") if /\A>(?=\s)/.match(value)
+		return value.sub(/\A[+-]/) { |marker| "\\#{marker}" } if /\A[+-](?=\s)/.match(value)
+		return value.sub(/\A(\d+)([.)])(?=\s)/) { "#{$1}\\#{$2}" } if /\A\d+[.)](?=\s)/.match(value)
+		return value.sub(/\A:/, "\\:") if escape_definition_marker && /\A:(?=\s)/.match(value)
+
+		value
+	end
+end
+
+end
+end
+end