slamdown 0.5.0

data/lib/slamdown/conversion/normaliser/content.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Owns format-neutral whitespace, phrasing-run and content-inspection rules shared by structural transformations.
+	class Content
+		HTML_WHITESPACE = /[\t\n\f\r ]+/.freeze
+		LEADING_SPACE = /\A +/.freeze
+		TRAILING_SPACE = / +\z/.freeze
+		BLOCK_SEMANTICS = [
+			:paragraph, :heading, :blockquote, :ordered_list, :unordered_list, :list_item, :definition_list, :definition_term, :definition_description,
+			:horizontal_rule, :code_block, :table, :table_head, :table_body, :table_foot, :table_row, :table_cell, :table_caption, :table_colgroup, :table_column
+		].freeze
+
+		def initialize(node_factory)
+			@node_factory = node_factory
+		end
+
+		def normalise_text(value)
+			value.to_s.gsub("\u00A0", " ").gsub(HTML_WHITESPACE, " ")
+		end
+
+		# Removes editor-layout whitespace around code while retaining meaningful indentation in already-left-aligned blocks.
+		def normalise_code_block(value)
+			lines = value.to_s.gsub("\r\n", "\n").gsub("\r", "\n").split("\n", -1)
+			lines.shift while lines.first && lines.first.strip.empty?
+			lines.pop while lines.last && lines.last.strip.empty?
+			return "" if lines.empty?
+
+			lines.map! { |line| line.sub(/[\t ]+\z/, "") }
+			prefixes = lines.reject { |line| line.empty? }.map { |line| line[/\A[\t ]*/] }
+			if !prefixes.empty? && prefixes.none?(&:empty?)
+				width = prefixes.map { |prefix| indentation_width(prefix) }.min
+				lines.map! { |line| remove_indentation(line, width) }
+			end
+			lines.join("\n")
+		end
+
+		# Splits inline runs around block children so a phrasing-only block can be closed and reopened canonically.
+		def split_around_blocks(nodes)
+			result = []
+			run = []
+			flush = lambda do
+				result << run unless run.empty?
+				run = []
+			end
+
+			nodes.each do |node|
+				if block_node?(node)
+					flush.call
+					result << node
+				else
+					run << node
+				end
+			end
+			flush.call
+			result
+		end
+
+		def normalise_inline_sequence(nodes, trim:)
+			result = []
+			nodes.each do |node|
+				if node.kind == :text && result.last && result.last.kind == :text
+					result[-1] = result.last.with(:value => normalise_text(result.last.value + node.value))
+				else
+					result << node
+				end
+			end
+
+			if trim
+				result[0] = result.first.with(:value => result.first.value.sub(LEADING_SPACE, "")) if result.first && result.first.kind == :text
+				result[-1] = result.last.with(:value => result.last.value.sub(TRAILING_SPACE, "")) if result.last && result.last.kind == :text
+			end
+			result.reject { |node| node.kind == :text && node.value.empty? }
+		end
+
+		def wrap_phrasing_runs(nodes)
+			result = []
+			run = []
+			flush = lambda do
+				run = normalise_inline_sequence(run, :trim => true)
+				result << @node_factory.paragraph(run, run.first.source_path) if visible_content?(run)
+				run = []
+			end
+
+			nodes.each do |node|
+				if node.semantic_kind == :paragraph_boundary
+					flush.call
+				elsif block_node?(node)
+					flush.call
+					result << node
+				else
+					run << node
+				end
+			end
+			flush.call
+			result
+		end
+
+		def plain_text(nodes)
+			nodes.map do |node|
+				if node.kind == :text
+					node.value
+				elsif node.kind == :entity && node.value.is_a?(Hash)
+					node.value[:character].to_s
+				elsif node.value.is_a?(String)
+					node.value
+				else
+					plain_text(node.children)
+				end
+			end.join
+		end
+
+		def textual_content?(nodes)
+			return true unless plain_text(nodes).strip.empty?
+
+			nodes.any? do |node|
+				image_alt = node.semantic_kind == :image ? Hash[node.attributes].fetch("alt", "").to_s : nil
+				[:typographic_symbol, :smart_quote].include?(node.kind) || !image_alt.to_s.strip.empty? || textual_content?(node.children)
+			end
+		end
+
+		def visible_content?(nodes)
+			nodes.any? do |node|
+				if node.kind == :text
+					!node.value.strip.empty?
+				elsif [:entity, :typographic_symbol, :smart_quote].include?(node.kind)
+					true
+				elsif [:line_break, :horizontal_rule, :paragraph_boundary].include?(node.semantic_kind)
+					false
+				else
+					textual_content?([node]) || [:image, :code_span].include?(node.semantic_kind) || node.representation == :html
+				end
+			end
+		end
+
+		def descendants(nodes)
+			nodes.flat_map { |node| [node] + descendants(node.children) }
+		end
+
+		def whitespace_text?(node)
+			node.kind == :text && node.value.strip.empty?
+		end
+
+		def block_node?(node)
+			node.category == :block || BLOCK_SEMANTICS.include?(node.semantic_kind)
+		end
+
+		private
+
+		# Tabs and complete four-space runs have equal indentation width, but a tab is never partially removed.
+		def indentation_width(value)
+			value.each_char.inject(0) { |width, character| width + (character == "\t" ? 4 : 1) }
+		end
+
+		def remove_indentation(line, width)
+			removed = 0
+			index = 0
+			while index < line.length && [" ", "\t"].include?(line[index])
+				character_width = line[index] == "\t" ? 4 : 1
+				break if removed + character_width > width
+
+				removed += character_width
+				index += 1
+			end
+			line[index..-1] || ""
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/correction_diagnostics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Collects ordinary corrective observations for one normalisation run without invoking Ruby's warning mechanism.
+	class CorrectionDiagnostics
+		def initialize
+			@diagnostics = []
+		end
+
+		def add(code, severity, message, path)
+			@diagnostics << Diagnostic.new(
+				:code => code,
+				:severity => severity,
+				:message => message,
+				:node_path => path
+			)
+		end
+
+		def to_a
+			@diagnostics.dup
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/element_actions.rb

@@ -0,0 +1,423 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Recursively applies resolved source-element actions and delegates specialised inline and break corrections to their focused collaborators.
+	class ElementActions
+		INTEGER_ATTRIBUTE = /\A[+-]?\d+\z/.freeze
+		ORDERED_LIST_START_RANGE = (2..100_000).freeze
+		TABLE_SEMANTICS = {
+			"caption" => :table_caption,
+			"colgroup" => :table_colgroup,
+			"col" => :table_column
+		}.freeze
+		SOURCE_SEMANTICS = {
+			"p" => :paragraph,
+			"blockquote" => :blockquote,
+			"ol" => :ordered_list,
+			"ul" => :unordered_list,
+			"li" => :list_item,
+			"dl" => :definition_list,
+			"dt" => :definition_term,
+			"dd" => :definition_description,
+			"details" => :definition_list,
+			"summary" => :definition_term,
+			"hr" => :horizontal_rule,
+			"br" => :line_break,
+			"code" => :code_span,
+			"kbd" => :code_span,
+			"samp" => :code_span,
+			"tt" => :code_span,
+			"var" => :code_span,
+			"pre" => :code_block,
+			"a" => :link,
+			"img" => :image,
+			"em" => :emphasis,
+			"i" => :emphasis,
+			"strong" => :strong,
+			"b" => :strong,
+			"s" => :deletion,
+			"strike" => :deletion,
+			"del" => :deletion,
+			"table" => :table,
+			"thead" => :table_head,
+			"tbody" => :table_body,
+			"tfoot" => :table_foot,
+			"tr" => :table_row,
+			"td" => :table_cell,
+			"th" => :table_cell
+		}.merge(TABLE_SEMANTICS).freeze
+
+		def initialize(configuration, node_factory, content, breaks, diagnostics)
+			@configuration = configuration
+			@node_factory = node_factory
+			@content = content
+			@breaks = breaks
+			@diagnostics = diagnostics
+			@policy = Policy::Resolver.new(configuration)
+			url_rewriter = UrlRewriter.new(configuration.urls)
+			@inline_semantics = InlineSemantics.new(@policy, node_factory, content, breaks, url_rewriter, diagnostics)
+		end
+
+		def normalise_root(root)
+			if root.kind == :document
+				normalise_children(root.children, [], @configuration.inline)
+			else
+				normalise_node(root, [], @configuration.inline)
+			end
+		end
+
+		private
+
+		def normalise_children(children, parent_path, force_inline, flatten_blocks = false)
+			children.each_with_index.each_with_object([]) do |(child, index), result|
+				result.concat(normalise_node(child, parent_path + [index], force_inline, flatten_blocks))
+			end
+		end
+
+		def normalise_node(node, path, force_inline, flatten_blocks = false)
+			return normalise_element(node, path, force_inline, flatten_blocks) if node.kind == :element
+
+			case node.kind
+			when :text
+				value = @content.normalise_text(node.value)
+				value.empty? ? [] : [@node_factory.text(value, path)]
+			when :entity
+				normalise_entity(node, path)
+			when :typographic_symbol, :smart_quote
+				[@node_factory.copy_leaf(node, path)]
+			when :raw, :comment, :processing_instruction
+				[]
+			else
+				normalise_children(node.children, path, force_inline, flatten_blocks)
+			end
+		end
+
+		def normalise_element(node, path, force_inline, flatten_blocks)
+			action = node.action ? node.action.value : :convert
+			return [] if action == :drop
+			return inline_contents(node, path) if action == :inline
+			if action == :convert && converted_semantic_kind(node) == :table && !table_has_usable_cell?(node)
+				@diagnostics.add(:unrepresentable_table_dropped, :info, "Dropped a table without any usable rows and cells", path)
+				return force_inline ? [@node_factory.text(" ", path)] : [@node_factory.paragraph_boundary(path)]
+			end
+			return [@node_factory.text(" ", path)] if force_inline && node.source_name == "br"
+			return flatten_block(node, path, action) if force_inline && block_output?(node, action)
+			return flatten_block(node, path, action) if flatten_blocks && block_output?(node, action)
+
+			children = normalise_children(node.children, path, force_inline, flatten_blocks || converted_semantic_kind(node) == :heading)
+			case action
+			when :preserve
+				normalise_preserved(node, children, path)
+			when :unwrap
+				node.category == :block ? @content.wrap_phrasing_runs(children) : children
+			when :convert
+				convert_node(node, children, path, force_inline)
+			else
+				raise "unhandled element action #{action.inspect}"
+			end
+		end
+
+		def convert_node(node, children, path, force_inline)
+			case node.source_name
+			when "div"
+				result = @content.wrap_phrasing_runs(children)
+				result.empty? && !force_inline ? [@node_factory.paragraph_boundary(path)] : result
+			when "span"
+				@inline_semantics.normalise_span(node, children, path)
+			else
+				convert_semantic_node(node, converted_semantic_kind(node), children, path, force_inline)
+			end
+		end
+
+		def convert_semantic_node(node, semantic_kind, children, path, force_inline)
+			case semantic_kind
+			when :paragraph
+				normalise_paragraph(node, children, path)
+			when :heading
+				normalise_heading(node, children, path)
+			when :emphasis, :strong
+				@inline_semantics.normalise_emphasis(node, semantic_kind, children, path)
+			when :deletion
+				node.source_name == "s" ? @inline_semantics.normalise_generic(node, semantic_kind, children, path) : @inline_semantics.trim_wrapper_whitespace(@node_factory.semantic(node, semantic_kind, children, path))
+			when :link
+				@inline_semantics.normalise_link(node, children, path)
+			when :image
+				@inline_semantics.normalise_image(node, path)
+			when :code_span
+				normalise_code_span(node, path)
+			when :code_block
+				value = @content.normalise_code_block(code_source(node))
+				[@node_factory.semantic(node, semantic_kind, [], path, :value => value)]
+			when :line_break
+				force_inline ? [@node_factory.text(" ", path)] : [@node_factory.semantic(node, semantic_kind, [], path)]
+			when :horizontal_rule
+				force_inline ? [] : [@node_factory.semantic(node, semantic_kind, [], path)]
+			when :ordered_list, :unordered_list
+				normalise_list(node, semantic_kind, children, path)
+			when :definition_list
+				node.source_name == "details" ? normalise_details(node, children, path) : [@node_factory.semantic(node, semantic_kind, children, path)]
+			when :list_item
+				@content.visible_content?(children) ? [@node_factory.semantic(node, semantic_kind, children, path)] : []
+			else
+				if node.category == :span
+					@inline_semantics.normalise_generic(node, semantic_kind, children, path)
+				else
+					children = children.reject { |child| @content.whitespace_text?(child) } if children.any? { |child| @content.block_node?(child) }
+					[@node_factory.semantic(node, semantic_kind, children, path)]
+				end
+			end
+		end
+
+		def normalise_paragraph(node, children, path)
+			retained_id = false
+			@content.split_around_blocks(children).each_with_object([]) do |part, result|
+				if part.is_a?(NormalisedNode)
+					result << part
+					next
+				end
+
+				@breaks.split_break_runs(part).each do |fragment|
+					fragment = @content.normalise_inline_sequence(fragment, :trim => true)
+					next unless @content.visible_content?(fragment)
+
+					if horizontal_rule_content?(fragment)
+						@diagnostics.add(:horizontal_rule_recovered, :info, "Converted a repeated-marker paragraph to a horizontal rule", path)
+						result << @node_factory.semantic(node, :horizontal_rule, [], path)
+					else
+						result << normalised_paragraph(node, fragment, path, !retained_id)
+						retained_id = true
+					end
+				end
+			end
+		end
+
+		def normalised_paragraph(node, fragment, path, retain_id)
+			paragraph_id = retain_id ? node.id : nil
+			if fragment.length == 1 && fragment.first.semantic_kind == :code_span && fragment.first.source_name == "code" && fragment.first.properties[:code_source_has_break]
+				code = fragment.first
+				return code.with(
+					:semantic_kind => :code_block,
+					:category => :block,
+					:value => @content.normalise_code_block(code.properties.fetch(:code_source)),
+					:id => paragraph_id || code.id,
+					:classes => (code.classes + node.classes).uniq
+				)
+			end
+			return @node_factory.semantic(node, :paragraph, fragment, path, :id => paragraph_id) unless fragment.length == 1 && fragment.first.semantic_kind == :code_span
+			return @node_factory.semantic(node, :paragraph, fragment, path, :id => paragraph_id) unless paragraph_id || !node.classes.empty?
+
+			code = fragment.first.with(
+				:id => paragraph_id || fragment.first.id,
+				:classes => (fragment.first.classes + node.classes).uniq
+			)
+			@node_factory.semantic(node, :paragraph, [code], path, :id => nil, :classes => [])
+		end
+
+		def normalise_heading(node, children, path)
+			fragments = @breaks.split_inline_fragments(children).map { |fragment| @content.normalise_inline_sequence(fragment, :trim => true) }
+			fragments.select! { |fragment| @content.textual_content?(fragment) }
+			if fragments.empty?
+				@diagnostics.add(:blank_heading_removed, :info, "Removed a heading without non-blank textual content", path)
+				return []
+			end
+
+			properties = node.properties.merge(:source_level => node.properties.fetch(:level))
+			fragments.map { |fragment| @node_factory.semantic(node, :heading, fragment, path, :properties => properties) }
+		end
+
+		def normalise_ordered_list(node, children, path)
+			start = Hash[node.attributes]["start"]
+			properties = node.properties
+			attributes = node.attributes
+			parsed_start = start.to_i if start.is_a?(String) && INTEGER_ATTRIBUTE.match(start.strip)
+			if parsed_start && ORDERED_LIST_START_RANGE.include?(parsed_start)
+				properties = properties.merge(:start => parsed_start)
+			elsif start
+				# Implausible or malformed list starts are editor debris, so remove the attribute and use the normal starting number.
+				attributes = attributes.reject { |name, _value| name == "start" }
+				@diagnostics.add(:ordered_list_start_removed, :info, "Removed an implausible ordered-list start", path)
+			end
+			@node_factory.semantic(node, :ordered_list, children, path, :properties => properties, :attributes => attributes)
+		end
+
+		def normalise_list(node, semantic_kind, children, path)
+			result = []
+			segment = []
+			items = nil
+			flush = lambda do
+				unless segment.empty?
+					if items
+						list = semantic_kind == :ordered_list ? normalise_ordered_list(node, segment, path) : @node_factory.semantic(node, semantic_kind, segment, path)
+						result << list
+					else
+						result.concat(@content.wrap_phrasing_runs(segment))
+					end
+				end
+				segment = []
+			end
+
+			children.each do |child|
+				list_item = child.semantic_kind == :list_item
+				next if !list_item && @content.whitespace_text?(child)
+				if !items.nil? && items != list_item
+					flush.call
+				end
+				items = list_item
+				segment << child
+			end
+			flush.call
+			result
+		end
+
+		def normalise_details(node, children, path)
+			children = children.reject { |child| @content.whitespace_text?(child) }
+			descriptions = children.map do |child|
+				if child.semantic_kind == :definition_term
+					child
+				else
+					@node_factory.build(
+						:kind => :element,
+						:semantic_kind => :definition_description,
+						:category => :block,
+						:children => [child],
+						:representation => :semantic,
+						:source_path => child.source_path
+					)
+				end
+			end
+			[@node_factory.semantic(node, :definition_list, descriptions, path)]
+		end
+
+		def normalise_code_span(node, path)
+			value = @content.normalise_text(code_source(node)).strip
+			value.empty? ? [] : [@node_factory.semantic(node, :code_span, [], path, :value => value)]
+		end
+
+		def code_source(node)
+			node.properties.fetch(:code_source, node.value.to_s)
+		end
+
+		def inline_contents(node, path)
+			children = normalise_children(node.children, path, true)
+			if children.empty? && node.value.is_a?(String) && !node.value.empty?
+				children = [@node_factory.text(@content.normalise_text(node.value), path)]
+			end
+			@content.normalise_inline_sequence(children, :trim => false)
+		end
+
+		def flatten_block(node, path, action)
+			semantic_kind = converted_semantic_kind(node)
+			return [] if semantic_kind == :horizontal_rule
+
+			if action == :convert && semantic_kind == :code_block
+				value = @content.normalise_text(node.value.to_s)
+				content = value.empty? ? [] : [@node_factory.build(
+					:kind => :element,
+					:source_name => node.source_name,
+					:semantic_kind => :code_span,
+					:category => :span,
+					:value => value,
+					:semantic_values => node.semantic_values,
+					:representation => :semantic,
+					:source_path => path
+				)]
+			else
+				content = normalise_children(node.children, path, true)
+				if content.empty? && node.value.is_a?(String) && !node.value.empty?
+					content = [@node_factory.text(@content.normalise_text(node.value), path)]
+				end
+			end
+			return [] unless @content.visible_content?(content)
+
+			[@node_factory.text(" ", path)] + content + [@node_factory.text(" ", path)]
+		end
+
+		def normalise_preserved(node, children, path)
+			semantic_kind = converted_semantic_kind(node)
+			if semantic_kind == :code_span
+				return [@node_factory.preserved(node, semantic_kind, [], path).with(:value => code_source(node))]
+			end
+			if node.category == :span || (semantic_kind && !Content::BLOCK_SEMANTICS.include?(semantic_kind))
+				return [@node_factory.preserved(node, semantic_kind, [], path)] if children.empty?
+
+				return @inline_semantics.wrap_inline_structure(children, path) do |fragment|
+					preserved = @node_factory.preserved(node, semantic_kind, fragment, path)
+					if semantic_kind == :deletion
+						@inline_semantics.trim_wrapper_whitespace(preserved)
+					elsif [:emphasis, :strong].include?(semantic_kind)
+						@inline_semantics.normalise_wrapper_whitespace(preserved, path)
+					else
+						[preserved]
+					end
+				end
+			end
+			unless children.any? { |child| @breaks.line_break?(child) || @breaks.paragraph_boundary?(child) }
+				preserved = @node_factory.preserved(node, semantic_kind, children, path)
+				return [preserved]
+			end
+			if node.category != :span
+				return @breaks.split_inline_fragments(children).map { |fragment| @node_factory.preserved(node, semantic_kind, fragment, path) }
+			end
+
+			@breaks.wrap_inline_fragments(children, path) do |fragment|
+				[@node_factory.preserved(node, semantic_kind, fragment, path)]
+			end
+		end
+
+		def normalise_entity(node, path)
+			return [@node_factory.text(" ", path)] if node.value.is_a?(Hash) && node.value[:code_point] == 160
+
+			[@node_factory.copy_leaf(node, path)]
+		end
+
+		def horizontal_rule_content?(children)
+			markers = @content.plain_text(children).gsub(/\s+/, "")
+			markers.length >= 3 && ["-", "*", "_"].include?(markers[0]) && markers.chars.all? { |marker| marker == markers[0] }
+		end
+
+		def converted_semantic_kind(node)
+			return :heading if node.source_name && /\Ah[1-6]\z/.match(node.source_name)
+			return SOURCE_SEMANTICS[node.source_name] if SOURCE_SEMANTICS.key?(node.source_name)
+
+			node.semantic_kind
+		end
+
+		def table_has_usable_cell?(table)
+			table.children.any? { |child| table_branch_has_usable_cell?(child) }
+		end
+
+		def table_branch_has_usable_cell?(node)
+			semantic_kind = converted_semantic_kind(node)
+			return false if semantic_kind == :table
+			return usable_table_row?(node) if semantic_kind == :table_row
+			return false if [:drop, :inline].include?(action_value(node))
+
+			node.children.any? { |child| table_branch_has_usable_cell?(child) }
+		end
+
+		def usable_table_row?(row)
+			return false unless [:convert, :preserve].include?(action_value(row))
+
+			row.children.any? do |cell|
+				converted_semantic_kind(cell) == :table_cell && [:convert, :preserve].include?(action_value(cell))
+			end
+		end
+
+		def action_value(node)
+			node.action ? node.action.value : :convert
+		end
+
+		def block_output?(node, action)
+			return true if node.category == :block
+			return false unless action == :convert
+
+			Content::BLOCK_SEMANTICS.include?(converted_semantic_kind(node))
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/heading_levels.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Calculates one conversion-root heading map after blank and policy-removed headings have left the tree.
+	class HeadingLevels
+		def initialize(configuration, content, diagnostics)
+			@configuration = configuration
+			@content = content
+			@diagnostics = diagnostics
+		end
+
+		def adjust(children)
+			headings = @content.descendants(children).select { |node| node.representation == :semantic && node.semantic_kind == :heading }
+			return children if headings.empty?
+
+			source_levels = headings.map { |heading| heading.properties[:source_level] }.uniq.sort
+			provisional_levels = source_levels.each_with_index.each_with_object({}) do |(level, index), result|
+				result[level] = @configuration.headings[:compact] ? index + 1 : level
+			end
+			offset = @configuration.headings[:start] - provisional_levels.values.min
+
+			children.map { |child| rewrite(child, provisional_levels, offset) }
+		end
+
+		private
+
+		def rewrite(node, provisional_levels, offset)
+			children = node.children.map { |child| rewrite(child, provisional_levels, offset) }
+			return node.with(:children => children) unless node.representation == :semantic && node.semantic_kind == :heading
+
+			source_level = node.properties[:source_level]
+			level = [[provisional_levels.fetch(source_level) + offset, 1].max, 6].min
+			if level != source_level
+				@diagnostics.add(:heading_level_adjusted, :info, "Adjusted heading level from #{source_level} to #{level}", node.source_path)
+			end
+			node.with(:children => children, :properties => node.properties.merge(:level => level))
+		end
+	end
+end
+
+end
+end