markbridge 0.1.3 → 0.2.1

data/lib/markbridge.rb

@@ -1,166 +1,329 @@
 # frozen_string_literal: true
 
 require_relative "markbridge/version"
-require_relative "markbridge/configuration"
+require_relative "markbridge/parse"
+require_relative "markbridge/conversion"
 
 require_relative "markbridge/ast"
 require_relative "markbridge/renderers/discourse"
-require_relative "markbridge/processors"
 
 module Markbridge
   class << self
-    # Parse BBCode to AST
+    # Parse BBCode to AST.
+    #
     # @param input [String] BBCode source
-    # @param handlers [HandlerRegistry, nil] custom handler registry or use default
-    # @return [AST::Document]
+    # @param handlers [Parsers::BBCode::HandlerRegistry, nil] custom handlers (defaults to .default)
+    # @return [Parse]
     def parse_bbcode(input, handlers: nil)
-      handlers ||= default_handlers
-      parse_with(Parsers::BBCode::Parser, input, handlers:)
+      raise ArgumentError, "input cannot be nil" if input.nil?
+
+      parser = Parsers::BBCode::Parser.new(handlers:)
+      ast = parser.parse(input.to_s)
+
+      Parse.new(
+        ast:,
+        format: :bbcode,
+        unknown_tags: parser.unknown_tags,
+        diagnostics: bbcode_diagnostics(parser),
+      )
     end
 
-    # Convert BBCode to Discourse Markdown
+    # Convert BBCode to Discourse Markdown.
+    #
+    # If a block is given, it is called with the parsed AST between
+    # parse and render — the caller can append/remove/replace nodes
+    # before rendering. Mutations to the yielded AST persist in
+    # {Conversion#ast}.
+    #
     # @param input [String] BBCode source
-    # @param handlers [HandlerRegistry, nil] custom handler registry or use default
-    # @param tag_library [TagLibrary, nil] custom tag library or use default
-    # @return [String] Markdown output
-    def bbcode_to_markdown(input, handlers: nil, tag_library: nil)
-      ast = parse_bbcode(input, handlers:)
-      render_to_markdown(ast, tag_library:)
+    # @param handlers [Parsers::BBCode::HandlerRegistry, nil] custom handlers
+    # @param renderer [Renderers::Discourse::Renderer, nil] custom renderer
+    #   (build with {.discourse_renderer}); defaults to a fresh default Renderer
+    # @param raise_on_error [Boolean] when true (default), let render-time
+    #   exceptions propagate; when false, swallow them, return a
+    #   {Conversion} with an empty +markdown+ string, and surface the
+    #   exceptions via {Conversion#errors}.
+    # @yieldparam ast [AST::Document] mutate before rendering (optional)
+    # @return [Conversion]
+    def bbcode_to_markdown(input, handlers: nil, renderer: nil, raise_on_error: true)
+      parse = parse_bbcode(input, handlers:)
+      yield(parse.ast) if block_given?
+      build_conversion(parse, renderer:, raise_on_error:)
     end
 
-    # Parse HTML to AST
-    # @param input [String] HTML source
-    # @param handlers [HandlerRegistry, nil] custom handler registry or use default
-    # @return [AST::Document]
+    # Parse HTML to AST.
+    #
+    # @param input [String, Nokogiri::XML::Node] HTML source or
+    #   pre-parsed Nokogiri tree (e.g. the +DocumentFragment+ returned
+    #   by +Nokogiri::HTML.fragment+). Passing a pre-parsed tree lets
+    #   callers run their own Nokogiri-driven pre-processing without
+    #   forcing Markbridge to re-parse the same bytes.
+    # @param handlers [Parsers::HTML::HandlerRegistry, nil] custom handlers
+    # @return [Parse]
     def parse_html(input, handlers: nil)
-      handlers ||= default_html_handlers
-      parse_with(Parsers::HTML::Parser, input, handlers:)
+      raise ArgumentError, "input cannot be nil" if input.nil?
+
+      parser = Parsers::HTML::Parser.new(handlers:)
+      ast = parser.parse(input)
+
+      Parse.new(ast:, format: :html, unknown_tags: parser.unknown_tags, diagnostics: {})
     end
 
-    # Convert HTML to Discourse Markdown
-    # @param input [String] HTML source
-    # @param handlers [HandlerRegistry, nil] custom handler registry or use default
-    # @param tag_library [TagLibrary, nil] custom tag library or use default
-    # @return [String] Markdown output
-    def html_to_markdown(input, handlers: nil, tag_library: nil)
-      ast = parse_html(input, handlers:)
-      render_to_markdown(ast, tag_library:)
+    # Convert HTML to Discourse Markdown.
+    #
+    # If a block is given, it is called with the parsed AST between
+    # parse and render — the caller can append/remove/replace nodes
+    # before rendering. Mutations to the yielded AST persist in
+    # {Conversion#ast}.
+    #
+    # @param input [String, Nokogiri::XML::Node] HTML source or
+    #   pre-parsed Nokogiri tree (see {.parse_html})
+    # @param handlers [Parsers::HTML::HandlerRegistry, nil] custom handlers
+    # @param renderer [Renderers::Discourse::Renderer, nil] custom renderer
+    # @param raise_on_error [Boolean] when true (default), let render-time
+    #   exceptions propagate; when false, swallow them, return a
+    #   {Conversion} with an empty +markdown+ string, and surface the
+    #   exceptions via {Conversion#errors}.
+    # @yieldparam ast [AST::Document] mutate before rendering (optional)
+    # @return [Conversion]
+    def html_to_markdown(input, handlers: nil, renderer: nil, raise_on_error: true)
+      parse = parse_html(input, handlers:)
+      yield(parse.ast) if block_given?
+      build_conversion(parse, renderer:, raise_on_error:)
     end
 
-    # Parse s9e/TextFormatter XML to AST
-    # @param input [String] XML source in s9e/TextFormatter format
-    # @param handlers [Parsers::TextFormatter::HandlerRegistry, nil] custom handler registry or use default
-    # @return [AST::Document]
+    # Parse s9e/TextFormatter XML to AST.
+    #
+    # @param input [String, Nokogiri::XML::Node] XML source or
+    #   pre-parsed Nokogiri tree. A +Nokogiri::XML::Document+ is
+    #   unwrapped via +#root+; any other node is treated as the root.
+    # @param handlers [Parsers::TextFormatter::HandlerRegistry, nil] custom handlers
+    # @return [Parse]
     def parse_text_formatter_xml(input, handlers: nil)
-      handlers ||= default_text_formatter_handlers
-      parse_with(Parsers::TextFormatter::Parser, input, handlers:)
+      raise ArgumentError, "input cannot be nil" if input.nil?
+
+      parser = Parsers::TextFormatter::Parser.new(handlers:)
+      ast = parser.parse(input)
+      unknown_tags = parser.unknown_tags
+
+      Parse.new(ast:, format: :text_formatter_xml, unknown_tags:, diagnostics: {})
     end
 
-    # Convert s9e/TextFormatter XML to Discourse Markdown
-    # @param input [String] XML source in s9e/TextFormatter format
-    # @param handlers [Parsers::TextFormatter::HandlerRegistry, nil] custom handler registry or use default
-    # @param tag_library [TagLibrary, nil] custom tag library or use default
-    # @return [String] Markdown output
-    def text_formatter_xml_to_markdown(input, handlers: nil, tag_library: nil)
-      ast = parse_text_formatter_xml(input, handlers:)
-      render_to_markdown(ast, tag_library:)
+    # Convert s9e/TextFormatter XML to Discourse Markdown.
+    #
+    # If a block is given, it is called with the parsed AST between
+    # parse and render — the caller can append/remove/replace nodes
+    # before rendering. Mutations to the yielded AST persist in
+    # {Conversion#ast}.
+    #
+    # @param input [String, Nokogiri::XML::Node] XML source or
+    #   pre-parsed Nokogiri tree (see {.parse_text_formatter_xml})
+    # @param handlers [Parsers::TextFormatter::HandlerRegistry, nil] custom handlers
+    # @param renderer [Renderers::Discourse::Renderer, nil] custom renderer
+    # @param raise_on_error [Boolean] see {.bbcode_to_markdown}
+    # @yieldparam ast [AST::Document] mutate before rendering (optional)
+    # @return [Conversion]
+    def text_formatter_xml_to_markdown(input, handlers: nil, renderer: nil, raise_on_error: true)
+      parse = parse_text_formatter_xml(input, handlers:)
+      yield(parse.ast) if block_given?
+      build_conversion(parse, renderer:, raise_on_error:)
     end
 
-    # Parse MediaWiki wikitext to AST
+    # Parse MediaWiki wikitext to AST.
+    #
     # @param input [String] MediaWiki source
-    # @param inline_tag_registry [Parsers::MediaWiki::InlineTagRegistry, nil] custom registry
-    # @return [AST::Document]
-    def parse_mediawiki(input, inline_tag_registry: nil)
+    # @param handlers [Parsers::MediaWiki::InlineTagRegistry, nil] custom inline-tag registry
+    # @return [Parse]
+    def parse_mediawiki(input, handlers: nil)
       raise ArgumentError, "input cannot be nil" if input.nil?
 
-      input = input.to_s
-      parser = Parsers::MediaWiki::Parser.new(inline_tag_registry:)
-      parser.parse(input)
+      parser = Parsers::MediaWiki::Parser.new(handlers:)
+      ast = parser.parse(input.to_s)
+
+      Parse.new(ast:, format: :mediawiki, unknown_tags: parser.unknown_tags, diagnostics: {})
     end
 
-    # Convert MediaWiki wikitext to Discourse Markdown
+    # Convert MediaWiki wikitext to Discourse Markdown.
+    #
+    # If a block is given, it is called with the parsed AST between
+    # parse and render — the caller can append/remove/replace nodes
+    # before rendering. Mutations to the yielded AST persist in
+    # {Conversion#ast}.
+    #
     # @param input [String] MediaWiki source
-    # @param inline_tag_registry [Parsers::MediaWiki::InlineTagRegistry, nil] custom registry
-    # @param tag_library [TagLibrary, nil] custom tag library or use default
-    # @return [String] Markdown output
-    def mediawiki_to_markdown(input, inline_tag_registry: nil, tag_library: nil)
-      ast = parse_mediawiki(input, inline_tag_registry:)
-      render_to_markdown(ast, tag_library:)
+    # @param handlers [Parsers::MediaWiki::InlineTagRegistry, nil]
+    # @param renderer [Renderers::Discourse::Renderer, nil] custom renderer
+    # @param raise_on_error [Boolean] see {.bbcode_to_markdown}
+    # @yieldparam ast [AST::Document] mutate before rendering (optional)
+    # @return [Conversion]
+    def mediawiki_to_markdown(input, handlers: nil, renderer: nil, raise_on_error: true)
+      parse = parse_mediawiki(input, handlers:)
+      yield(parse.ast) if block_given?
+      build_conversion(parse, renderer:, raise_on_error:)
     end
 
-    # Get default handler registry
-    # @return [Parsers::BBCode::HandlerRegistry]
-    def default_handlers
-      @default_handlers ||= Parsers::BBCode::HandlerRegistry.default
+    # Convert input in the given format. Thin dispatcher over the
+    # four +*_to_markdown+ methods; useful when the format is data-
+    # driven (e.g. iterating posts whose +:format+ column varies).
+    # An optional block is forwarded to the dispatched method.
+    #
+    # @param input [String, Nokogiri::XML::Node] source content; the
+    #   HTML and TextFormatter dispatch targets also accept pre-parsed
+    #   Nokogiri trees.
+    # @param format [Symbol] one of +:bbcode+, +:html+,
+    #   +:text_formatter_xml+, +:mediawiki+
+    # @param kwargs [Hash] forwarded to the underlying convenience method
+    #   (e.g. +handlers:+, +renderer:+, +raise_on_error:+).
+    # @yieldparam ast [AST::Document] mutate before rendering (optional)
+    # @return [Conversion]
+    def convert(input, format:, **kwargs, &block)
+      case format
+      when :bbcode
+        bbcode_to_markdown(input, **kwargs, &block)
+      when :html
+        html_to_markdown(input, **kwargs, &block)
+      when :text_formatter_xml
+        text_formatter_xml_to_markdown(input, **kwargs, &block)
+      when :mediawiki
+        mediawiki_to_markdown(input, **kwargs, &block)
+      else
+        raise ArgumentError,
+              "unknown format #{format.inspect} " \
+                "(expected :bbcode, :html, :text_formatter_xml, or :mediawiki)"
+      end
     end
 
-    # Get default HTML handler registry
-    # @return [Parsers::HTML::HandlerRegistry]
-    def default_html_handlers
-      @default_html_handlers ||= Parsers::HTML::HandlerRegistry.default
-    end
+    # Render a {Parse} or a bare AST node to Discourse Markdown.
+    # Useful when the caller has mutated the AST between parse and
+    # render (e.g. appending attachments not present in the source),
+    # or built an AST programmatically.
+    #
+    # When given a {Parse}, the returned {Conversion} carries the
+    # parser's +unknown_tags+, +diagnostics+, and source +format+
+    # forward. When given an AST node, those fields default to empty
+    # and +format+ is +nil+ — there was no source document, so there
+    # is no source format to report. A bare node that isn't already a
+    # {AST::Document} is wrapped in one, so {Conversion#ast} is always
+    # a Document (and tree helpers like +each_descendant+ are always
+    # available on it).
+    #
+    # @param parse_or_ast [Parse, AST::Node]
+    # @param format [Symbol] :discourse (only renderer currently shipped)
+    # @param renderer [Renderers::Discourse::Renderer, nil]
+    # @param raise_on_error [Boolean]
+    # @return [Conversion]
+    def render(parse_or_ast, format: :discourse, renderer: nil, raise_on_error: true)
+      raise ArgumentError, "unknown render format #{format.inspect}" unless format == :discourse
 
-    # Get default tag library
-    # @return [Renderers::Discourse::TagLibrary]
-    def default_tag_library
-      @default_tag_library ||= Renderers::Discourse::TagLibrary.default
-    end
+      parse =
+        case parse_or_ast
+        when Parse
+          parse_or_ast
+        when AST::Document
+          Parse.new(ast: parse_or_ast, format: nil, unknown_tags: {}, diagnostics: {})
+        when AST::Node
+          document = AST::Document.new([parse_or_ast])
+          Parse.new(ast: document, format: nil, unknown_tags: {}, diagnostics: {})
+        else
+          raise ArgumentError, "expected Parse or AST::Node, got #{parse_or_ast.class}"
+        end
 
-    # Get default s9e/TextFormatter handler registry
-    # @return [Parsers::TextFormatter::HandlerRegistry]
-    def default_text_formatter_handlers
-      @default_text_formatter_handlers ||= Parsers::TextFormatter::HandlerRegistry.default
+      build_conversion(parse, renderer:, raise_on_error:)
     end
 
-    # Get the global configuration
-    # @return [Configuration]
-    def configuration
-      @configuration ||= Configuration.new
-    end
+    # Build a configured Discourse {Renderers::Discourse::Renderer}
+    # for use with the +renderer:+ kwarg on the +*_to_markdown+
+    # convenience methods.
+    #
+    # @param tags [Hash{Class => Tag, nil}, nil] mappings to merge on
+    #   top of the default library; +nil+ values unregister the class.
+    # @param tag_library [Renderers::Discourse::TagLibrary, nil] base
+    #   library to start from. Defaults to a fresh {TagLibrary.default}.
+    #   When supplied, it is +dup+'d before any +tags:+ / +unregister:+
+    #   mutation, so the caller's library is left untouched.
+    # @param unregister [Array<Class>, nil] AST classes to drop from
+    #   the library so they fall through to +render_children+.
+    # @param escaper [#escape, nil] when given, used as-is; +escape:+,
+    #   +escape_hard_line_breaks:+, and +allow:+ are then ignored.
+    # @param escape [Boolean] when +false+, the renderer is built with
+    #   {Renderers::Discourse::IdentityEscaper} (no Markdown escaping).
+    #   Mutually exclusive with +escape_hard_line_breaks:+ / +allow:+.
+    # @param escape_hard_line_breaks [Boolean] forwarded to a fresh
+    #   {MarkdownEscaper} when no explicit +escaper:+ is given.
+    # @param allow [Symbol, Array<Symbol>, nil] block-level constructs to
+    #   pass through unescaped (e.g. +:lists+, +:bullet_list+,
+    #   +:ordered_list+, +:atx_heading+, +:block_quote+); forwarded to a
+    #   fresh {MarkdownEscaper}.
+    # @param postprocessor [Renderers::Discourse::Postprocessor, nil] when given,
+    #   used as-is; +strip_trailing_invisibles:+ is then ignored.
+    # @param strip_trailing_invisibles [Boolean] forwarded to a fresh
+    #   {Renderers::Discourse::Postprocessor} when no explicit
+    #   +postprocessor:+ is given. Strips NBSP and zero-width format
+    #   characters from the end of each line.
+    # @return [Renderers::Discourse::Renderer]
+    def discourse_renderer(
+      tags: nil,
+      tag_library: nil,
+      unregister: nil,
+      escaper: nil,
+      escape: true,
+      escape_hard_line_breaks: false,
+      allow: nil,
+      postprocessor: nil,
+      strip_trailing_invisibles: false
+    )
+      # Dup the caller's library before mutating so successive
+      # +discourse_renderer+ calls against the same +tag_library:+ don't
+      # see each other's overrides. +TagLibrary.default+ already returns
+      # a fresh instance, so the dup is only needed in the explicit
+      # +tag_library:+ branch.
+      library = tag_library ? tag_library.dup : Renderers::Discourse::TagLibrary.default
+      library.merge!(tags) if tags
+      Array(unregister).each { |klass| library.unregister(klass) }
 
-    # Configure Markbridge with a block
-    # @yield [Configuration]
-    def configure
-      yield configuration
-    end
+      escaper ||= build_escaper(escape:, escape_hard_line_breaks:, allow:)
+      postprocessor ||= Renderers::Discourse::Postprocessor.new(strip_trailing_invisibles:)
 
-    # Reset defaults (useful for testing)
-    def reset_defaults!
-      @default_handlers = nil
-      @default_html_handlers = nil
-      @default_tag_library = nil
-      @default_text_formatter_handlers = nil
-      @configuration = nil
+      Renderers::Discourse::Renderer.new(tag_library: library, escaper:, postprocessor:)
     end
 
     private
 
-    def parse_with(parser_class, input, handlers:)
-      raise ArgumentError, "input cannot be nil" if input.nil?
-
-      parser = parser_class.new(handlers:)
-      parser.parse(input.to_s)
+    def bbcode_diagnostics(parser)
+      {
+        auto_closed_tags_count: parser.auto_closed_tags_count,
+        depth_exceeded_count: parser.depth_exceeded_count,
+        unclosed_raw_tags: parser.unclosed_raw_tags,
+      }
     end
 
-    def render_to_markdown(ast, tag_library:)
-      tag_library ||= default_tag_library
-      renderer = build_renderer(tag_library:)
-      cleanup_markdown(renderer.render(ast))
+    def build_conversion(parse, renderer:, raise_on_error:)
+      renderer ||= Renderers::Discourse::Renderer.new
+      markdown, errors = render_through(renderer, parse.ast, raise_on_error:)
+
+      Conversion.new(parsed: parse, markdown:, errors:)
     end
 
-    def build_renderer(tag_library:)
-      escaper =
-        Renderers::Discourse::MarkdownEscaper.new(
-          escape_hard_line_breaks: configuration.escape_hard_line_breaks,
-        )
-      Renderers::Discourse::Renderer.new(tag_library:, escaper:)
+    def build_escaper(escape:, escape_hard_line_breaks:, allow:)
+      if escape == false
+        if escape_hard_line_breaks || allow
+          raise ArgumentError,
+                "escape: false is mutually exclusive with " \
+                  "escape_hard_line_breaks: / allow: (those configure " \
+                  "MarkdownEscaper, which escape: false replaces wholesale)"
+        end
+        Renderers::Discourse::IdentityEscaper.new
+      else
+        Renderers::Discourse::MarkdownEscaper.new(escape_hard_line_breaks:, allow:)
+      end
     end
 
-    def cleanup_markdown(text)
-      text
-        .gsub(/\n{3,}/, "\n\n") # Max 2 consecutive newlines
-        .gsub(/^[ \t]+$/, "") # Remove whitespace-only lines
-        .strip # Trim leading/trailing whitespace
+    def render_through(renderer, ast, raise_on_error:)
+      raw = renderer.render(ast)
+      [renderer.postprocessor.call(raw), []]
+    rescue StandardError => e
+      raise if raise_on_error
+      ["", [e]]
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: markbridge
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.1
 platform: ruby
 authors:
 - Discourse Team
@@ -25,6 +25,7 @@ files:
 - lib/markbridge/ast/bold.rb
 - lib/markbridge/ast/code.rb
 - lib/markbridge/ast/color.rb
+- lib/markbridge/ast/details.rb
 - lib/markbridge/ast/document.rb
 - lib/markbridge/ast/element.rb
 - lib/markbridge/ast/email.rb
@@ -53,10 +54,11 @@ files:
 - lib/markbridge/ast/upload.rb
 - lib/markbridge/ast/url.rb
 - lib/markbridge/bbcode.rb
-- lib/markbridge/configuration.rb
+- lib/markbridge/conversion.rb
 - lib/markbridge/gem_loader.rb
 - lib/markbridge/html.rb
 - lib/markbridge/mediawiki.rb
+- lib/markbridge/parse.rb
 - lib/markbridge/parsers/bbcode.rb
 - lib/markbridge/parsers/bbcode/closing_strategies/base.rb
 - lib/markbridge/parsers/bbcode/closing_strategies/reordering.rb
@@ -102,7 +104,9 @@ files:
 - lib/markbridge/parsers/html/handlers/paragraph_handler.rb
 - lib/markbridge/parsers/html/handlers/quote_handler.rb
 - lib/markbridge/parsers/html/handlers/raw_handler.rb
+- lib/markbridge/parsers/html/handlers/self_closing_handler.rb
 - lib/markbridge/parsers/html/handlers/simple_handler.rb
+- lib/markbridge/parsers/html/handlers/span_handler.rb
 - lib/markbridge/parsers/html/handlers/table_cell_handler.rb
 - lib/markbridge/parsers/html/handlers/table_handler.rb
 - lib/markbridge/parsers/html/handlers/table_row_handler.rb
@@ -126,19 +130,12 @@ files:
 - lib/markbridge/parsers/text_formatter/handlers/table_cell_handler.rb
 - lib/markbridge/parsers/text_formatter/handlers/url_handler.rb
 - lib/markbridge/parsers/text_formatter/parser.rb
-- lib/markbridge/processors.rb
-- lib/markbridge/processors/discourse_markdown.rb
-- lib/markbridge/processors/discourse_markdown/code_block_tracker.rb
-- lib/markbridge/processors/discourse_markdown/detectors/base.rb
-- lib/markbridge/processors/discourse_markdown/detectors/event.rb
-- lib/markbridge/processors/discourse_markdown/detectors/mention.rb
-- lib/markbridge/processors/discourse_markdown/detectors/poll.rb
-- lib/markbridge/processors/discourse_markdown/detectors/upload.rb
-- lib/markbridge/processors/discourse_markdown/scanner.rb
 - lib/markbridge/renderers/discourse.rb
 - lib/markbridge/renderers/discourse/builders/list_item_builder.rb
 - lib/markbridge/renderers/discourse/html_escaper.rb
+- lib/markbridge/renderers/discourse/identity_escaper.rb
 - lib/markbridge/renderers/discourse/markdown_escaper.rb
+- lib/markbridge/renderers/discourse/postprocessor.rb
 - lib/markbridge/renderers/discourse/render_context.rb
 - lib/markbridge/renderers/discourse/renderer.rb
 - lib/markbridge/renderers/discourse/rendering_interface.rb
@@ -149,6 +146,7 @@ files:
 - lib/markbridge/renderers/discourse/tags/bold_tag.rb
 - lib/markbridge/renderers/discourse/tags/code_tag.rb
 - lib/markbridge/renderers/discourse/tags/color_tag.rb
+- lib/markbridge/renderers/discourse/tags/details_tag.rb
 - lib/markbridge/renderers/discourse/tags/email_tag.rb
 - lib/markbridge/renderers/discourse/tags/event_tag.rb
 - lib/markbridge/renderers/discourse/tags/heading_tag.rb

data/lib/markbridge/configuration.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Markbridge
-  class Configuration
-    attr_accessor :escape_hard_line_breaks
-
-    def initialize
-      @escape_hard_line_breaks = false
-    end
-  end
-end