slamdown 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1022c0c170ad37a1dc791fd76797ac41173aead90dea397889d74813d083cc4d
+  data.tar.gz: 3d5fbf7537c7b012a21a7a1fe8eb256b9d4bebaabb6560609b7decac053d52c9
+SHA512:
+  metadata.gz: cefee43327b751e2e646c71858ea15825f62fb9fab7cf506ebfb3e6f0db283c49b0ae0bc3b7a00987eb40e37640a0f891a6bdbdd8aa2c7dd0c744a343219216b
+  data.tar.gz: 6f82a32a29404aac296028981d8cbeb4af38db44b7842d90172f785841642e6c5c71f23546b6f8a71fd3bf9e1b834f4e6e25744cabe1932d272ea988fd8ed254

data/changelog.md

@@ -0,0 +1,7 @@
+# Changelog
+
+
+## 0.5.0
+
+* Configurable conversion of messy HTML to clean Markdown and Kramdown.
+* Diagnostics for parsing and conversion issues.

data/lib/slamdown/conversion/configuration.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Holds the complete validated conversion options relied upon by all later conversion stages.
+class Configuration
+	ATTRIBUTES = [:inline, :headings, :entities, :html_indent, :urls, :tables, :elements, :classes, :ids].freeze
+
+	attr_reader(*ATTRIBUTES)
+
+	class << self
+		# Validates caller options and fills every omitted value from Slamdown's defaults.
+		def build(options = {})
+			ConfigurationValidator.new(options).validate
+		end
+	end
+
+	def initialize(values)
+		ATTRIBUTES.each do |attribute|
+			instance_variable_set("@#{attribute}", Immutable.copy(values.fetch(attribute)))
+		end
+		freeze
+	end
+
+	# Provides a plain immutable projection for consumers that need the complete effective configuration.
+	def to_h
+		ATTRIBUTES.each_with_object({}) do |attribute, result|
+			result[attribute] = public_send(attribute)
+		end.freeze
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/configuration_validator.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Performs small, section-local checks and normalisation before conversion code receives any options.
+class ConfigurationValidator
+	TOP_LEVEL_KEYS = [:inline, :headings, :entities, :html_indent, :urls, :tables, :elements, :classes, :ids].freeze
+	HEADING_KEYS = [:start, :compact].freeze
+	URL_KEYS = [:scheme, :host, :path, :force].freeze
+	TABLE_KEYS = [:format, :max_width].freeze
+	ELEMENT_RULE_KEYS = [:action, :attributes].freeze
+	ELEMENT_ACTIONS = [:convert, :preserve, :unwrap, :inline, :drop].freeze
+	ATTRIBUTE_ACTIONS = [:preserve, :drop].freeze
+	ENTITY_MODES = [:symbolic, :character, :numeric].freeze
+	URL_FORCES = [:relative, :absolute].freeze
+	TABLE_FORMATS = [:auto, :html].freeze
+	HTML_NAME = /\A[a-z][a-z0-9._:-]*\z/.freeze
+	ATTRIBUTE_NAME = /\A[a-z_:][a-z0-9._:-]*\z/.freeze
+	HTML_WHITESPACE = /[\t\n\f\r ]/.freeze
+
+	DEFAULTS = Immutable.copy(
+		:inline => false,
+		:headings => {:start => 2, :compact => true},
+		:entities => :character,
+		:html_indent => "\t",
+		:urls => {:scheme => nil, :host => nil, :path => nil, :force => :relative},
+		:tables => {:format => :auto, :max_width => 150},
+		:elements => {},
+		:classes => {},
+		:ids => {}
+	)
+
+	def initialize(options)
+		@options = options
+	end
+
+	def validate
+		expect_hash(@options, "options")
+		@options = normalise_option_keys(@options, TOP_LEVEL_KEYS, "options")
+
+		Configuration.new(
+			:inline => boolean_option(:inline),
+			:headings => validate_headings,
+			:entities => closed_option(:entities, ENTITY_MODES),
+			:html_indent => validate_html_indent,
+			:urls => validate_urls,
+			:tables => validate_tables,
+			:elements => validate_elements,
+			:classes => validate_name_rules(:classes),
+			:ids => validate_name_rules(:ids)
+		)
+	end
+
+	private
+
+	def value_or_default(name)
+		@options.key?(name) ? @options[name] : DEFAULTS[name]
+	end
+
+	def boolean_option(name)
+		value = value_or_default(name)
+		return value if value == true || value == false
+
+		raise ArgumentError, "#{name} must be true or false"
+	end
+
+	def closed_option(name, allowed)
+		value = value_or_default(name)
+		return value if allowed.include?(value)
+
+		raise ArgumentError, "#{name} must be one of #{allowed.map(&:inspect).join(', ')}"
+	end
+
+	def validate_headings
+		values = merged_section(:headings, HEADING_KEYS)
+		start = values[:start]
+		compact = values[:compact]
+
+		raise ArgumentError, "headings[:start] must be an integer from 1 to 6" unless start.is_a?(Integer) && (1..6).include?(start)
+		raise ArgumentError, "headings[:compact] must be true or false" unless compact == true || compact == false
+
+		values
+	end
+
+	def validate_html_indent
+		value = value_or_default(:html_indent)
+		valid = value.is_a?(String) && (value.empty? || value == "\t" || /\A {1,8}\z/.match(value))
+		return value if valid
+
+		raise ArgumentError, "html_indent must be empty, one tab or one to eight spaces"
+	end
+
+	def validate_urls
+		values = merged_section(:urls, URL_KEYS)
+
+		[:scheme, :host, :path].each do |name|
+			value = values[name]
+			raise ArgumentError, "urls[:#{name}] must be a String or nil" unless value.nil? || value.is_a?(String)
+		end
+		raise ArgumentError, "urls[:force] must be :relative or :absolute" unless URL_FORCES.include?(values[:force])
+		if values[:force] == :absolute && (!usable_string?(values[:scheme]) || !usable_string?(values[:host]))
+			raise ArgumentError, "urls with force :absolute require usable scheme and host values"
+		end
+
+		values
+	end
+
+	def validate_tables
+		values = merged_section(:tables, TABLE_KEYS)
+
+		raise ArgumentError, "tables[:format] must be :auto or :html" unless TABLE_FORMATS.include?(values[:format])
+		max_width = values[:max_width]
+		raise ArgumentError, "tables[:max_width] must be nil or an integer of at least 20" unless max_width.nil? || (max_width.is_a?(Integer) && max_width >= 20)
+
+		values
+	end
+
+	def merged_section(name, keys)
+		provided = @options.key?(name) ? @options[name] : {}
+		expect_hash(provided, name.to_s)
+		provided = normalise_option_keys(provided, keys, name.to_s)
+
+		DEFAULTS.fetch(name).merge(provided)
+	end
+
+	def validate_elements
+		provided = value_or_default(:elements)
+		expect_hash(provided, "elements")
+
+		normalise_named_hash(provided, "elements") do |element_name, rule|
+			expect_hash(rule, "elements[#{element_name.inspect}]")
+			rule = normalise_option_keys(rule, ELEMENT_RULE_KEYS, "elements[#{element_name.inspect}]")
+			validate_element_rule(element_name, rule)
+		end
+	end
+
+	def validate_element_rule(element_name, rule)
+		result = {}
+		if rule.key?(:action)
+			action = rule[:action]
+			raise ArgumentError, "invalid action for #{element_name.inspect}: #{action.inspect}" unless ELEMENT_ACTIONS.include?(action)
+			if action == :convert && (element_name == :any || !Policy::CONVERTER_NAMES.include?(element_name))
+				raise ArgumentError, "#{element_name.inspect} has no registered Slamdown converter"
+			end
+			result[:action] = action
+		end
+
+		if rule.key?(:attributes)
+			attributes = rule[:attributes]
+			expect_hash(attributes, "attributes for #{element_name.inspect}")
+			result[:attributes] = normalise_named_hash(attributes, "attributes for #{element_name.inspect}", :attribute => true) do |attribute_name, action|
+				if ["class", "id"].include?(attribute_name)
+					raise ArgumentError, "#{attribute_name} must be configured through its dedicated section"
+				end
+				raise ArgumentError, "invalid attribute action for #{attribute_name.inspect}: #{action.inspect}" unless ATTRIBUTE_ACTIONS.include?(action)
+				action
+			end
+		end
+
+		result
+	end
+
+	def validate_name_rules(section)
+		provided = value_or_default(section)
+		expect_hash(provided, section.to_s)
+
+		normalise_named_hash(provided, section.to_s) do |element_name, matcher|
+			validate_matcher(matcher, "#{section}[#{element_name.inspect}]")
+		end
+	end
+
+	def validate_matcher(matcher, location)
+		return :any if matcher == :any
+
+		matchers = matcher.is_a?(Array) ? matcher : [matcher]
+		matchers.each do |candidate|
+			valid_string = candidate.is_a?(String) && !candidate.empty? && !HTML_WHITESPACE.match(candidate)
+			next if valid_string || candidate.is_a?(Regexp)
+
+			raise ArgumentError, "#{location} must contain exact strings, regular expressions or :any"
+		end
+		matchers
+	end
+
+	def normalise_named_hash(values, location, attribute: false)
+		result = {}
+		values.each do |source_name, value|
+			name = normalise_name(source_name, location, attribute)
+			raise ArgumentError, "#{location} contains duplicate normalised key #{name.inspect}" if result.key?(name)
+
+			result[name] = yield(name, value)
+		end
+		result
+	end
+
+	def normalise_name(source_name, location, attribute)
+		unless source_name.is_a?(String) || source_name.is_a?(Symbol)
+			raise ArgumentError, "#{location} keys must be strings or symbols"
+		end
+
+		name = source_name.to_s.downcase
+		return :any if name == "any"
+
+		pattern = attribute ? ATTRIBUTE_NAME : HTML_NAME
+		raise ArgumentError, "invalid name #{source_name.inspect} in #{location}" unless pattern.match(name)
+
+		name
+	end
+
+	def expect_hash(value, location)
+		raise ArgumentError, "#{location} must be a Hash" unless value.is_a?(Hash)
+	end
+
+	def normalise_option_keys(values, allowed, location)
+		values.each_with_object({}) do |(source_name, value), result|
+			name = source_name.is_a?(String) ? source_name.to_sym : source_name
+			unless allowed.include?(name)
+				raise ArgumentError, "unknown #{location} keys: #{source_name.inspect}"
+			end
+			if result.key?(name)
+				raise ArgumentError, "#{location} contains duplicate normalised key #{name.inspect}"
+			end
+
+			result[name] = value
+		end
+	end
+
+	def usable_string?(value)
+		value.is_a?(String) && !value.strip.empty?
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/converter.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Coordinates the validated public conversion pipeline and applies the conversion error-reporting contract.
+class Converter
+	FORMATS = {
+		"markdown" => Output::Markdown,
+		"kramdown" => Output::Kramdown
+	}.freeze
+	ERROR_MODES = [:raise, :warn, :silent].freeze
+
+	class << self
+		def convert(input, format:, options: {}, errors: :warn)
+			new(format, errors).convert(input, options)
+		end
+	end
+
+	def initialize(format, errors)
+		@format = normalise_format(format)
+		@output_class = FORMATS.fetch(@format)
+		validate_error_mode(errors)
+		@errors = errors
+	end
+
+	def convert(input, options)
+		root = conversion_root(input)
+		configuration = Configuration.build(options)
+		prepared = Preparer.prepare(root, configuration)
+		normalisation = Normaliser.normalise(prepared, configuration)
+		rendering = Renderer.render(normalisation.tree, configuration, @format)
+		report_rendering_errors(rendering.diagnostics)
+
+		@output_class.new(rendering.output, normalisation.diagnostics + rendering.diagnostics)
+	rescue ArgumentError
+		raise
+	rescue ConversionError
+		raise
+	rescue StandardError => error
+		handle_conversion_error(error, defined?(normalisation) && normalisation ? normalisation.diagnostics : [])
+	end
+
+	private
+
+	def normalise_format(format)
+		name = format.is_a?(Symbol) ? format.to_s : format
+		return name if name.is_a?(String) && FORMATS.key?(name)
+
+		raise ArgumentError, "format must be :markdown, :kramdown, \"markdown\" or \"kramdown\""
+	end
+
+	def validate_error_mode(errors)
+		return if ERROR_MODES.include?(errors)
+
+		raise ArgumentError, "errors must be :raise, :warn or :silent"
+	end
+
+	def conversion_root(input)
+		root = input.is_a?(Document) ? input.tree : input
+		unless root.is_a?(Parsing::Node)
+			raise ArgumentError, "input must be a Slamdown document or parsed tree"
+		end
+		unless root.kind == :document || root.category == :block
+			raise ArgumentError, "input tree must be a document or block-level subtree"
+		end
+
+		root
+	end
+
+	def handle_conversion_error(error, diagnostics)
+		diagnostic = Diagnostic.new(
+			:code => :conversion_error,
+			:severity => :error,
+			:message => "Unable to convert document: #{error.message}",
+			:node_path => nil
+		)
+		conversion_error = ConversionError.new(diagnostic)
+
+		raise conversion_error if @errors == :raise
+		Kernel.warn(conversion_error.message) if @errors == :warn
+
+		@output_class.new("", diagnostics + [diagnostic])
+	end
+
+	def report_rendering_errors(diagnostics)
+		errors = diagnostics.select { |diagnostic| diagnostic.severity == :error }
+		return if errors.empty?
+
+		raise ConversionError.new(errors.first) if @errors == :raise
+		return unless @errors == :warn
+
+		errors.each { |diagnostic| Kernel.warn(ConversionError.new(diagnostic).message) }
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normalisation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Carries the immutable normalised conversion tree together with the corrective diagnostics produced while building it.
+class Normalisation
+	attr_reader :tree, :diagnostics
+
+	def initialize(tree, diagnostics)
+		raise ArgumentError, "tree must be a normalised Slamdown node" unless tree.is_a?(NormalisedNode)
+		raise ArgumentError, "diagnostics must contain only Slamdown diagnostics" unless diagnostics.is_a?(Array) && diagnostics.all? { |diagnostic| diagnostic.is_a?(Diagnostic) }
+
+		@tree = tree
+		@diagnostics = diagnostics.dup.freeze
+		freeze
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normalised_node.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+# Represents one immutable, renderer-independent construct after element policy and corrective normalisation have been applied.
+class NormalisedNode
+	ATTRIBUTES = [:kind, :source_name, :semantic_kind, :category, :value, :attributes, :classes, :id, :children, :properties, :semantic_values, :representation, :synthetic, :source_path].freeze
+
+	attr_reader(*ATTRIBUTES)
+
+	def initialize(values)
+		ATTRIBUTES.each do |attribute|
+			instance_variable_set("@#{attribute}", Immutable.copy(values.fetch(attribute)))
+		end
+		freeze
+	end
+
+	# Creates a changed immutable node without making transformation code repeat every unchanged field.
+	def with(changes)
+		unknown = changes.keys - ATTRIBUTES
+		raise ArgumentError, "unknown normalised-node attributes: #{unknown.map(&:inspect).join(', ')}" unless unknown.empty?
+
+		values = ATTRIBUTES.each_with_object({}) do |attribute, result|
+			result[attribute] = changes.key?(attribute) ? changes[attribute] : public_send(attribute)
+		end
+		self.class.new(values)
+	end
+
+	# Provides a stable plain-Ruby projection for focused normalisation tests and later rendering diagnostics.
+	def to_h
+		ATTRIBUTES.each_with_object({}) do |attribute, result|
+			value = public_send(attribute)
+			result[attribute] = attribute == :children ? value.map(&:to_h) : value
+		end
+	end
+end
+
+end
+end

data/lib/slamdown/conversion/normaliser/breaks.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Slamdown
+module Conversion
+
+class Normaliser
+	# Interprets line-break runs and bubbles recovered paragraph boundaries through inline wrappers without leaking internal sentinels.
+	class Breaks
+		def initialize(node_factory, content, diagnostics)
+			@node_factory = node_factory
+			@content = content
+			@diagnostics = diagnostics
+		end
+
+		def split_break_runs(children)
+			fragments = []
+			current = []
+			index = 0
+
+			while index < children.length
+				if paragraph_boundary?(children[index])
+					if @content.visible_content?(current) && @content.visible_content?(children[(index + 1)..-1] || [])
+						fragments << current
+						current = []
+					end
+					index += 1
+					next
+				end
+
+				unless line_break?(children[index])
+					current << children[index]
+					index += 1
+					next
+				end
+
+				run_path = children[index].source_path
+				break_count = 0
+				interstitial = []
+				cursor = index
+				loop do
+					if cursor < children.length && line_break?(children[cursor])
+						break_count += 1
+						cursor += 1
+					elsif cursor < children.length && @content.whitespace_text?(children[cursor]) && next_break_index(children, cursor)
+						interstitial << children[cursor]
+						cursor += 1
+					else
+						break
+					end
+				end
+
+				content_before = @content.visible_content?(current)
+				content_after = @content.visible_content?(children[cursor..-1] || [])
+				if break_count >= 2
+					if content_before && content_after
+						fragments << current
+						current = []
+						@diagnostics.add(:paragraph_break_recovered, :info, "Converted a repeated line-break run to a paragraph boundary", run_path)
+					end
+				elsif content_before && content_after
+					current << children[index]
+					current.concat(interstitial)
+				end
+				index = cursor
+			end
+
+			fragments << current if @content.visible_content?(current)
+			fragments
+		end
+
+		def wrap_inline_fragments(children, path)
+			fragments = split_inline_fragments(children)
+			fragments.each_with_index.each_with_object([]) do |(fragment, index), result|
+				result << @node_factory.paragraph_boundary(fragment.first ? fragment.first.source_path : path) if index > 0
+				result.concat(yield(fragment))
+			end
+		end
+
+		def split_inline_fragments(children)
+			return [children] unless children.any? { |child| line_break?(child) || paragraph_boundary?(child) }
+
+			split_break_runs(children)
+		end
+
+		def line_break?(node)
+			node.representation == :semantic && node.semantic_kind == :line_break
+		end
+
+		def paragraph_boundary?(node)
+			node.representation == :boundary && node.semantic_kind == :paragraph_boundary
+		end
+
+		private
+
+		def next_break_index(children, index)
+			cursor = index
+			cursor += 1 while cursor < children.length && @content.whitespace_text?(children[cursor])
+			cursor < children.length && line_break?(children[cursor])
+		end
+	end
+end
+
+end
+end