slamdown 0.5.0

data/lib/slamdown/conversion/normaliser/inline_semantics.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Normalises inline semantic constructs after their descendants have been prepared, including styled-span recovery and whitespace shifting.
+	class InlineSemantics
+		URL_SCHEME = /\A([A-Za-z][A-Za-z0-9+.-]*):/.freeze
+		STYLE_ELEMENTS = {
+			:strong => ["strong", :strong],
+			:emphasis => ["em", :emphasis],
+			:deletion => ["del", :deletion],
+			:underline => ["u", :underline]
+		}.freeze
+
+		def initialize(policy, node_factory, content, breaks, url_rewriter, diagnostics)
+			@policy = policy
+			@node_factory = node_factory
+			@content = content
+			@breaks = breaks
+			@url_rewriter = url_rewriter
+			@diagnostics = diagnostics
+		end
+
+		def normalise_span(node, children, path)
+			styles = node.semantic_values[:styled_as]
+			return children unless styles
+
+			@diagnostics.add(:styled_span_recovered, :info, "Converted presentational span styling to semantic content", path)
+			styles.reverse_each do |style|
+				children = apply_generated_style(style, children, path)
+			end
+			children
+		end
+
+		def normalise_emphasis(node, semantic_kind, children, path)
+			wrap_inline_structure(children, path) do |fragment|
+				fragment = @content.normalise_inline_sequence(fragment, :trim => false)
+				normalised = @node_factory.semantic(node, semantic_kind, fragment, path)
+				normalise_wrapper_whitespace(normalised, path)
+			end
+		end
+
+		def normalise_wrapper_whitespace(node, path)
+			moved = move_edge_whitespace(node)
+			if moved.length != 1 || moved.first != node
+				@diagnostics.add(:emphasis_whitespace_moved, :info, "Moved collapsible whitespace outside inline formatting", path)
+			end
+			moved
+		end
+
+		def trim_wrapper_whitespace(node)
+			children = @content.normalise_inline_sequence(node.children, :trim => true)
+			@content.visible_content?(children) ? [node.with(:children => children)] : []
+		end
+
+		# Reopens an inline wrapper around phrasing runs while allowing nested block output to bubble to its block parent.
+		def wrap_inline_structure(children, path)
+			@content.split_around_blocks(children).flat_map do |part|
+				if part.is_a?(NormalisedNode)
+					part
+				else
+					@breaks.wrap_inline_fragments(part, path) { |fragment| yield fragment }
+				end
+			end
+		end
+
+		def normalise_link(node, children, path)
+			children = @content.normalise_inline_sequence(children, :trim => false)
+			return [] unless @content.visible_content?(children)
+
+			attributes = Hash[node.attributes]
+			destination = attributes["href"]
+			return children unless usable_url?(destination) && !blacklisted_scheme?(destination.strip)
+
+			children = flatten_links(children)
+			attributes["href"] = @url_rewriter.rewrite(destination.strip)
+			wrap_inline_structure(children, path) do |fragment|
+				normalised = @node_factory.semantic(node, :link, fragment, path, :attributes => ordered_attributes(node.attributes, attributes))
+				moved = move_edge_whitespace(normalised)
+				@diagnostics.add(:link_whitespace_moved, :info, "Moved collapsible whitespace outside link text", path) unless moved == [normalised]
+				moved
+			end
+		end
+
+		def normalise_image(node, path)
+			attributes = Hash[node.attributes]
+			source = attributes["src"]
+			alt = @content.normalise_text(attributes.fetch("alt", "").to_s).strip
+			attributes["alt"] = alt if attributes.key?("alt")
+
+			unless source.is_a?(String) && !source.strip.empty?
+				return alt.empty? ? [] : [@node_factory.text(@content.normalise_text(alt), path)]
+			end
+
+			attributes["src"] = @url_rewriter.rewrite(source.strip)
+			[@node_factory.semantic(node, :image, [], path, :attributes => ordered_attributes(node.attributes, attributes))]
+		end
+
+		def normalise_generic(node, semantic_kind, children, path)
+			wrap_inline_structure(children, path) do |fragment|
+				[@node_factory.semantic(node, semantic_kind, fragment, path)]
+			end
+		end
+
+		private
+
+		def apply_generated_style(style, children, path)
+			source_name, semantic_kind = STYLE_ELEMENTS.fetch(style)
+			action = @policy.action_for_source_name(source_name).value
+
+			case action
+			when :drop
+				[]
+			when :unwrap, :inline
+				children
+			when :preserve
+				wrap_generated(source_name, semantic_kind, children, path, :html)
+			when :convert
+				wrap_generated(source_name, semantic_kind, children, path, :semantic)
+			else
+				raise "unhandled generated element action #{action.inspect}"
+			end
+		end
+
+		def wrap_generated(source_name, semantic_kind, children, path, representation)
+			wrap_inline_structure(children, path) do |fragment|
+				node = @node_factory.build(
+					:kind => :element,
+					:source_name => source_name,
+					:semantic_kind => semantic_kind,
+					:category => :span,
+					:children => fragment,
+					:representation => representation,
+					:source_path => path
+				)
+				if semantic_kind == :deletion
+					trim_wrapper_whitespace(node)
+				elsif [:emphasis, :strong].include?(semantic_kind)
+					normalise_wrapper_whitespace(node, path)
+				else
+					[node]
+				end
+			end
+		end
+
+		def flatten_links(children)
+			children.flat_map do |child|
+				if child.semantic_kind == :link
+					flatten_links(child.children)
+				elsif child.children.empty?
+					child
+				else
+					child.with(:children => flatten_links(child.children))
+				end
+			end
+		end
+
+		# Link labels and emphasis share the same rule: surrounding collapsible space belongs outside the generated syntax.
+		def move_edge_whitespace(node)
+			children = node.children.dup
+			leading = remove_leading_space(children)
+			trailing = remove_trailing_space(children)
+			return [node] if leading.empty? && trailing.empty?
+
+			children = @content.normalise_inline_sequence(children, :trim => false)
+			result = []
+			result << @node_factory.text(leading, node.source_path) unless leading.empty?
+			result << node.with(:children => children) if @content.visible_content?(children)
+			result << @node_factory.text(trailing, node.source_path) unless trailing.empty?
+			result
+		end
+
+		def remove_leading_space(children)
+			return "" unless children.first && children.first.kind == :text
+
+			match = Content::LEADING_SPACE.match(children.first.value)
+			return "" unless match
+
+			remaining = children.first.value[match[0].length..-1]
+			remaining.empty? ? children.shift : children[0] = children.first.with(:value => remaining)
+			match[0]
+		end
+
+		def remove_trailing_space(children)
+			return "" unless children.last && children.last.kind == :text
+
+			match = Content::TRAILING_SPACE.match(children.last.value)
+			return "" unless match
+
+			remaining = children.last.value[0...-match[0].length]
+			remaining.empty? ? children.pop : children[-1] = children.last.with(:value => remaining)
+			match[0]
+		end
+
+		def usable_url?(value)
+			value.is_a?(String) && !value.strip.empty?
+		end
+
+		def blacklisted_scheme?(destination)
+			match = URL_SCHEME.match(destination)
+			return false unless match
+
+			scheme = match[1].downcase
+			scheme.end_with?("script") || ["file", "data", "content", "intent"].include?(scheme) || scheme.start_with?("ms-")
+		end
+
+		def ordered_attributes(original, values)
+			original.each_with_object([]) do |(name, _value), result|
+				result << [name, values[name]] if values.key?(name)
+			end
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/node_factory.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Centralises construction of immutable normalised nodes so transformations only describe the fields they intend to change.
+	class NodeFactory
+		DEFAULTS = {
+			:kind => :element,
+			:source_name => nil,
+			:semantic_kind => nil,
+			:category => nil,
+			:value => nil,
+			:attributes => [],
+			:classes => [],
+			:id => nil,
+			:children => [],
+			:properties => {},
+			:semantic_values => {},
+			:representation => nil,
+			:synthetic => false,
+			:source_path => nil
+		}.freeze
+
+		def build(values)
+			NormalisedNode.new(DEFAULTS.merge(values))
+		end
+
+		def text(value, path)
+			build(
+				:kind => :text,
+				:category => :span,
+				:value => value,
+				:source_path => path
+			)
+		end
+
+		def semantic(node, semantic_kind, children, path, changes = {})
+			category = Content::BLOCK_SEMANTICS.include?(semantic_kind) ? :block : :span
+			build({
+				:kind => :element,
+				:source_name => node.source_name,
+				:semantic_kind => semantic_kind,
+				:category => category,
+				:value => node.value,
+				:attributes => node.attributes,
+				:classes => node.classes,
+				:id => node.id,
+				:children => children,
+				:properties => node.properties,
+				:semantic_values => node.semantic_values,
+				:representation => :semantic,
+				:synthetic => node.synthetic,
+				:source_path => path
+			}.merge(changes))
+		end
+
+		def preserved(node, semantic_kind, children, path)
+			build(
+				:kind => :element,
+				:source_name => node.source_name,
+				:semantic_kind => semantic_kind,
+				:category => node.category,
+				:value => node.value,
+				:attributes => node.attributes,
+				:classes => node.classes,
+				:id => node.id,
+				:children => children,
+				:properties => node.properties,
+				:semantic_values => node.semantic_values,
+				:representation => :html,
+				:synthetic => node.synthetic,
+				:source_path => path
+			)
+		end
+
+		def copy_leaf(node, path)
+			build(
+				:kind => node.kind,
+				:category => node.category,
+				:value => node.value,
+				:properties => node.properties,
+				:source_path => path
+			)
+		end
+
+		def paragraph(children, path)
+			build(
+				:kind => :element,
+				:semantic_kind => :paragraph,
+				:category => :block,
+				:children => children,
+				:representation => :semantic,
+				:source_path => path
+			)
+		end
+
+		def paragraph_boundary(path)
+			build(
+				:kind => :element,
+				:semantic_kind => :paragraph_boundary,
+				:category => :block,
+				:representation => :boundary,
+				:source_path => path
+			)
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/url_rewriter.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Applies the validated URL pattern to usable link and image destinations without changing unrelated domains or root paths.
+	class UrlRewriter
+		ABSOLUTE_SCHEME = /\A[A-Za-z][A-Za-z0-9+.-]*:\/\//.freeze
+
+		def initialize(options)
+			@force = options.fetch(:force)
+			@scheme = clean_scheme(options[:scheme])
+			@host = clean_host(options[:host])
+			@path = clean_path(options[:path])
+		end
+
+		def rewrite(value)
+			return value unless active?
+
+			uri = URI.parse(value)
+			return rewrite_absolute(uri, value) if absolute?(uri, value)
+			return rewrite_root_relative(value) if value.start_with?("/")
+			return value if value.start_with?("?", "#")
+
+			path = join_paths(@path, value)
+			@force == :absolute ? absolute_url(path) : root_relative(path)
+		rescue URI::InvalidURIError
+			value
+		end
+
+		private
+
+		def active?
+			@scheme || @host || @path
+		end
+
+		def clean_scheme(value)
+			return unless usable?(value)
+
+			value.strip.sub(/:\/\/\z/, "").sub(/:\z/, "")
+		end
+
+		def clean_host(value)
+			return unless usable?(value)
+
+			value.strip.sub(/\A\/\//, "").sub(/\/+\z/, "")
+		end
+
+		def clean_path(value)
+			return unless usable?(value)
+
+			value.strip.gsub(/\A\/+|\/+\z/, "")
+		end
+
+		def usable?(value)
+			value.is_a?(String) && !value.strip.empty?
+		end
+
+		def absolute?(uri, source)
+			uri.scheme || uri.host || ABSOLUTE_SCHEME.match(source) || source.start_with?("//")
+		end
+
+		def rewrite_absolute(uri, source)
+			return source unless @host && hosts_match?(uri.host, @host)
+			return source if @force == :relative && @scheme && uri.scheme.to_s.downcase != @scheme.downcase
+			return source unless path_matches?(uri.path)
+
+			path = uri.path.to_s.empty? ? "/" : uri.path
+			path = append_suffix(path, uri)
+			@force == :absolute ? absolute_url(path) : root_relative(path)
+		end
+
+		def rewrite_root_relative(source)
+			return source unless @force == :absolute && path_matches?(split_suffix(source).first)
+
+			absolute_url(source)
+		end
+
+		def hosts_match?(first, second)
+			canonical_host(first) == canonical_host(second)
+		end
+
+		def canonical_host(value)
+			value.to_s.downcase.sub(/\Awww\./, "")
+		end
+
+		def path_matches?(value)
+			return true unless @path
+
+			path = value.to_s.sub(/\A\/+/, "")
+			path == @path || path.start_with?("#{@path}/")
+		end
+
+		def join_paths(prefix, value)
+			path, suffix = split_suffix(value)
+			segments = [prefix, path].compact.map { |part| part.gsub(/\A\/+|\/+\z/, "") }.reject(&:empty?)
+			segments.join("/") + suffix
+		end
+
+		def split_suffix(value)
+			match = /\A([^?#]*)(.*)\z/.match(value)
+			[match[1], match[2]]
+		end
+
+		def append_suffix(path, uri)
+			result = path
+			result += "?#{uri.query}" if uri.query
+			result += "##{uri.fragment}" if uri.fragment
+			result
+		end
+
+		def root_relative(path)
+			"/#{path.sub(/\A\/+/, "")}"
+		end
+
+		def absolute_url(path)
+			"#{@scheme}://#{@host}#{root_relative(path)}"
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Coordinates one renderer-independent normalisation run through focused element, content, break and heading collaborators.
+class Normaliser
+	BLOCK_SEMANTICS = Content::BLOCK_SEMANTICS
+
+	class << self
+		def normalise(root, configuration)
+			new(configuration).normalise(root)
+		end
+	end
+
+	def initialize(configuration)
+		raise ArgumentError, "configuration must be a Slamdown conversion configuration" unless configuration.is_a?(Configuration)
+
+		@configuration = configuration
+	end
+
+	def normalise(root)
+		validate_root(root)
+		node_factory = NodeFactory.new
+		diagnostics = CorrectionDiagnostics.new
+		content = Content.new(node_factory)
+		breaks = Breaks.new(node_factory, content, diagnostics)
+		element_actions = ElementActions.new(@configuration, node_factory, content, breaks, diagnostics)
+		children = element_actions.normalise_root(root)
+
+		if @configuration.inline
+			children = content.normalise_inline_sequence(children, :trim => true)
+		else
+			children = content.wrap_phrasing_runs(children)
+			children = HeadingLevels.new(@configuration, content, diagnostics).adjust(children)
+		end
+
+		tree = node_factory.build(
+			:kind => :document,
+			:children => children,
+			:properties => root.kind == :document ? root.properties : {},
+			:source_path => []
+		)
+		Normalisation.new(tree, diagnostics.to_a)
+	end
+
+	private
+
+	def validate_root(root)
+		raise ArgumentError, "root must be a prepared Slamdown node" unless root.is_a?(PreparedNode)
+		return if root.kind == :document || root.category == :block
+
+		raise ArgumentError, "root must be a document or block-level prepared node"
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/policy.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Resolves caller policy over Slamdown's default semantic and retention policy while preserving where each decision originated.
+module Policy
+	CONVERTER_NAMES = ([
+		"p", "blockquote", "ol", "ul", "li", "dl", "dt", "dd", "details", "summary", "hr", "br", "code", "kbd", "pre", "samp", "tt", "var", "a", "img", "em", "i", "strong", "b", "s", "strike",
+		"table", "thead", "tbody", "tfoot", "tr", "td", "th", "caption", "col", "colgroup", "div", "span"
+	] + (1..6).map { |level| "h#{level}" }).freeze
+	PRESERVED_NAMES = %w[abbr cite del ins mark sub sup u].freeze
+
+	DEFAULT_ELEMENTS = begin
+		elements = {}
+		CONVERTER_NAMES.each { |name| elements[name] = {:action => :convert} }
+		PRESERVED_NAMES.each { |name| elements[name] = {:action => :preserve} }
+		%w[audio time video].each { |name| elements[name] = {:action => :inline} }
+		elements["a"][:attributes] = {"href" => :preserve}
+		elements["img"][:attributes] = {"src" => :preserve, "alt" => :preserve}
+		elements["ol"][:attributes] = {"start" => :preserve}
+		elements["abbr"][:attributes] = {"title" => :preserve}
+		%w[td th].each { |name| elements[name][:attributes] = {"colspan" => :preserve, "rowspan" => :preserve, "scope" => :preserve} }
+		%w[col colgroup].each { |name| elements[name][:attributes] = {"span" => :preserve} }
+		elements[:any] = {:attributes => {:any => :drop}}
+		Immutable.copy(elements)
+	end
+
+	# One immutable resolved value and the layer that supplied it.
+	class Decision
+		attr_reader :value, :provenance, :element_scope, :attribute_scope
+
+		def initialize(value, provenance, element_scope:, attribute_scope: nil)
+			@value = value
+			@provenance = provenance
+			@element_scope = element_scope
+			@attribute_scope = attribute_scope
+			freeze
+		end
+	end
+
+	# One immutable allow/deny result for combined class or ID matchers.
+	class MatchDecision
+		attr_reader :allowed, :provenance
+
+		def initialize(allowed, provenance)
+			@allowed = allowed
+			@provenance = provenance.freeze
+			freeze
+		end
+
+		def allowed?
+			allowed
+		end
+	end
+
+	# Looks up policy for source-aware parsed nodes. Parser-synthetic nodes deliberately receive no invented element key.
+	class Resolver
+		def initialize(configuration)
+			@configuration = configuration
+		end
+
+		def action_for(node)
+			return unless node.source_name
+
+			first_element_decision(node.source_name, :action) || Decision.new(generic_action(node), :default_any, :element_scope => :any)
+		end
+
+		# Resolves policy for a semantic element generated during normalisation, such as a strong wrapper recovered from span styling.
+		def action_for_source_name(source_name)
+			decision = first_element_decision(source_name, :action)
+			return decision if decision
+
+			raise ArgumentError, "no element action is registered for #{source_name.inspect}"
+		end
+
+		def attribute_for(node, attribute_name)
+			return unless node.source_name
+
+			first_attribute_decision(node.source_name, attribute_name.to_s.downcase)
+		end
+
+		def class_for(node, token)
+			match_decision(@configuration.classes, node, token)
+		end
+
+		def id_for(node, value)
+			match_decision(@configuration.ids, node, value)
+		end
+
+		private
+
+		def first_element_decision(element_name, property)
+			policy_layers(element_name).each do |rule, provenance, element_scope|
+				return Decision.new(rule[property], provenance, :element_scope => element_scope) if rule && rule.key?(property)
+			end
+			nil
+		end
+
+		def first_attribute_decision(element_name, attribute_name)
+			policy_groups.each do |elements, source|
+				[
+					[elements[element_name], :specific],
+					[elements[:any], :any]
+				].each do |rule, element_scope|
+					next unless rule && rule[:attributes]
+
+					attributes = rule[:attributes]
+					provenance = provenance_for(source, element_scope)
+					return Decision.new(attributes[attribute_name], provenance, :element_scope => element_scope, :attribute_scope => :named) if attributes.key?(attribute_name)
+					return Decision.new(attributes[:any], provenance, :element_scope => element_scope, :attribute_scope => :any) if attributes.key?(:any)
+				end
+			end
+			nil
+		end
+
+		def policy_layers(element_name)
+			policy_groups.flat_map do |elements, source|
+				[
+					[elements[element_name], provenance_for(source, :specific), :specific],
+					[elements[:any], provenance_for(source, :any), :any]
+				]
+			end
+		end
+
+		def policy_groups
+			[
+				[@configuration.elements, :user],
+				[DEFAULT_ELEMENTS, :default]
+			]
+		end
+
+		def provenance_for(source, element_scope)
+			return :overridden if source == :user && element_scope == :specific
+			return :user_any if source == :user
+
+			"default_#{element_scope}".to_sym
+		end
+
+		def generic_action(node)
+			usable_content?(node) ? :unwrap : :drop
+		end
+
+		def usable_content?(node)
+			node.children.any? { |child| ordinary_content?(child) }
+		end
+
+		def ordinary_content?(node)
+			return !node.value.empty? if node.kind == :text && node.value.is_a?(String)
+			return true if [:entity, :typographic_symbol, :smart_quote].include?(node.kind)
+			return false if [:raw, :comment, :processing_instruction].include?(node.kind)
+			return true if node.kind == :element && node.semantic_kind
+
+			node.children.any? { |child| ordinary_content?(child) }
+		end
+
+		def match_decision(rules, node, value)
+			return MatchDecision.new(false, []) unless node.source_name
+
+			provenance = []
+			provenance << :user_any if matches?(rules[:any], value)
+			provenance << :overridden if matches?(rules[node.source_name], value)
+			MatchDecision.new(!provenance.empty?, provenance)
+		end
+
+		def matches?(matchers, value)
+			return false unless matchers
+			return true if matchers == :any
+
+			matchers.any? do |matcher|
+				matcher.is_a?(Regexp) ? matcher.match(value) : matcher == value
+			end
+		end
+	end
+end
+
+end
+end