turndown 1.0.0

data/lib/turndown/converter.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Turndown
+  class Converter
+    attr_reader :options, :rules
+
+    def initialize(options = {})
+      normalized_options = normalize_options(options)
+      @options = Utilities.extend(default_options, normalized_options)
+      @rules = Rules.new(@options)
+    end
+
+    def convert(input)
+      raise TypeError, "#{input} is not a string, or an element/document/fragment node." unless can_convert?(input)
+      return "" if input == ""
+
+      output = process(RootNode.build(input, options))
+      post_process(output)
+    end
+
+    def use(plugin)
+      if plugin.is_a?(Array)
+        plugin.each { |entry| use(entry) }
+      elsif plugin.respond_to?(:call)
+        plugin.call(self)
+      else
+        raise TypeError, "plugin must be a callable or an Array of callables"
+      end
+
+      self
+    end
+
+    def add_rule(name, filter:, replacement:, append: nil)
+      rules.add(name, Rule.new(filter: filter, replacement: replacement, append: append))
+      self
+    end
+
+    def keep(filter)
+      rules.keep(filter)
+      self
+    end
+
+    def remove(filter)
+      rules.remove(filter)
+      self
+    end
+
+    def escape(string)
+      Utilities.escape_markdown(string)
+    end
+
+    private
+
+    def process(parent_raw)
+      parent = parent_raw.is_a?(Node) ? parent_raw : Node.new(parent_raw, options)
+
+      parent.child_nodes.reduce("") do |output, node|
+        replacement =
+          if node.text_node?
+            node.code? ? node.node_value : escape(node.node_value)
+          elsif node.element?
+            replacement_for_node(node)
+          else
+            ""
+          end
+
+        join(output, replacement)
+      end
+    end
+
+    def post_process(output)
+      rules.each do |rule|
+        appended = rule.append(options)
+        output = join(output, appended) unless appended.empty?
+      end
+
+      output.sub(/\A[\t\r\n]+/, "").sub(/\s+\z/, "")
+    end
+
+    def replacement_for_node(node)
+      rule = rules.for_node(node)
+      content = process(node)
+      whitespace = node.flanking_whitespace
+      if !whitespace[:leading].empty? || !whitespace[:trailing].empty?
+        content = content.gsub(/\A[[:space:]]+|[[:space:]]+\z/, "")
+      end
+      "#{whitespace[:leading]}#{rule.replacement(content, node, options)}#{whitespace[:trailing]}"
+    end
+
+    def join(output, replacement)
+      s1 = Utilities.trim_trailing_newlines(output)
+      s2 = Utilities.trim_leading_newlines(replacement)
+      newlines = [output.length - s1.length, replacement.length - s2.length].max
+      separator = "\n\n"[0, newlines]
+      "#{s1}#{separator}#{s2}"
+    end
+
+    def normalize_options(options)
+      options.each_with_object({}) do |(key, value), normalized|
+        normalized[normalize_option_key(key)] = value
+      end
+    end
+
+    def normalize_option_key(key)
+      key.to_s.gsub(/([A-Z])/, '_\1').downcase.sub(/\A_/, "").to_sym
+    end
+
+    def can_convert?(input)
+      input.is_a?(String) || input.is_a?(Nokogiri::XML::Node) || input.is_a?(Nokogiri::XML::DocumentFragment)
+    end
+
+    def default_options
+      {
+        rules: CommonMarkRules.build,
+        heading_style: "setext",
+        hr: "* * *",
+        bullet_list_marker: "*",
+        code_block_style: "indented",
+        fence: "```",
+        em_delimiter: "_",
+        strong_delimiter: "**",
+        link_style: "inlined",
+        link_reference_style: "full",
+        br: "  ",
+        preformatted_code: false,
+        blank_replacement: ->(_content, node, _options) { node.block? ? "\n\n" : "" },
+        keep_replacement: lambda do |_content, node, _options|
+          node.block? ? "\n\n#{node.outer_html}\n\n" : node.outer_html
+        end,
+        default_replacement: lambda do |content, node, _options|
+          node.block? ? "\n\n#{content}\n\n" : content
+        end
+      }
+    end
+  end
+end

data/lib/turndown/node.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+module Turndown
+  class Node
+    attr_reader :raw, :options
+
+    def initialize(raw, options)
+      @raw = raw
+      @options = options
+    end
+
+    def node_name
+      raw.name.to_s.upcase
+    end
+
+    def node_name_downcase
+      raw.name.to_s.downcase
+    end
+
+    def node_type
+      return 9 if raw.is_a?(Nokogiri::HTML5::Document) || raw.is_a?(Nokogiri::HTML4::Document)
+      return 11 if raw.is_a?(Nokogiri::HTML5::DocumentFragment) || raw.is_a?(Nokogiri::XML::DocumentFragment)
+      return 1 if raw.element?
+      return 4 if raw.cdata?
+      return 3 if raw.text?
+
+      0
+    end
+
+    def text_node?
+      node_type == 3 || node_type == 4
+    end
+
+    def element?
+      node_type == 1
+    end
+
+    def document?
+      node_type == 9
+    end
+
+    def document_fragment?
+      node_type == 11
+    end
+
+    def block?
+      Utilities.is_block?(self)
+    end
+
+    def void?
+      Utilities.is_void?(self)
+    end
+
+    def meaningful_when_blank?
+      Utilities.is_meaningful_when_blank?(self)
+    end
+
+    def code?
+      node_name == "CODE" || parent_node&.code?
+    end
+
+    def blank?
+      !void? &&
+        !meaningful_when_blank? &&
+        text_content.match?(/\A\s*\z/) &&
+        !Utilities.has_void?(raw) &&
+        !Utilities.has_meaningful_when_blank?(raw)
+    end
+
+    def flanking_whitespace
+      return { leading: "", trailing: "" } if block? || (options[:preformatted_code] && code?)
+
+      edges = edge_whitespace(text_content)
+      if edges[:leading_ascii] != "" && flanked_by_whitespace?(:left)
+        edges[:leading] = edges[:leading_non_ascii]
+      end
+      if edges[:trailing_ascii] != "" && flanked_by_whitespace?(:right)
+        edges[:trailing] = edges[:trailing_non_ascii]
+      end
+
+      { leading: edges[:leading], trailing: edges[:trailing] }
+    end
+
+    def node_value
+      raw.content
+    end
+
+    def text_content
+      raw.text
+    end
+
+    def outer_html
+      raw.to_html
+    end
+
+    def class_name
+      get_attribute("class").to_s
+    end
+
+    def type
+      get_attribute("type")
+    end
+
+    def checked?
+      !get_attribute("checked").nil?
+    end
+
+    def get_attribute(name)
+      raw[name]
+    end
+
+    def child_nodes
+      raw.children.map { |child| self.class.new(child, options) }
+    end
+
+    def parent_node
+      return nil unless raw.respond_to?(:parent) && raw.parent
+
+      self.class.new(raw.parent, options)
+    end
+
+    def previous_sibling
+      return nil unless raw.respond_to?(:previous_sibling) && raw.previous_sibling
+
+      self.class.new(raw.previous_sibling, options)
+    end
+
+    def next_sibling
+      return nil unless raw.respond_to?(:next_sibling) && raw.next_sibling
+
+      self.class.new(raw.next_sibling, options)
+    end
+
+    def first_child
+      child = raw.children.first
+      return nil unless child
+
+      self.class.new(child, options)
+    end
+
+    def last_element_child
+      child = raw.element_children.last
+      child && self.class.new(child, options)
+    end
+
+    def children
+      raw.element_children.map { |child| self.class.new(child, options) }
+    end
+
+    def rows
+      raw.css("tr").map { |row| self.class.new(row, options) }
+    end
+
+    def same_node?(other)
+      raw == other.raw
+    end
+
+    def inspect
+      "#<Turndown::Node #{node_name}>"
+    end
+
+    private
+
+    def edge_whitespace(string)
+      chars = string.chars
+      ascii_whitespace = +""
+      non_ascii_leading = +""
+      index = 0
+
+      while index < chars.length && ascii_whitespace?(chars[index])
+        ascii_whitespace << chars[index]
+        index += 1
+      end
+
+      while index < chars.length && whitespace?(chars[index])
+        non_ascii_leading << chars[index]
+        index += 1
+      end
+
+      if index == chars.length
+        leading = ascii_whitespace + non_ascii_leading
+        return {
+          leading: leading,
+          leading_ascii: ascii_whitespace,
+          leading_non_ascii: non_ascii_leading,
+          trailing: "",
+          trailing_non_ascii: "",
+          trailing_ascii: ""
+        }
+      end
+
+      trailing_ascii = +""
+      trailing_non_ascii = +""
+      index = chars.length - 1
+
+      while index >= 0 && ascii_whitespace?(chars[index])
+        trailing_ascii.prepend(chars[index])
+        index -= 1
+      end
+
+      while index >= 0 && whitespace?(chars[index])
+        trailing_non_ascii.prepend(chars[index])
+        index -= 1
+      end
+
+      {
+        leading: ascii_whitespace + non_ascii_leading,
+        leading_ascii: ascii_whitespace,
+        leading_non_ascii: non_ascii_leading,
+        trailing: trailing_non_ascii + trailing_ascii,
+        trailing_non_ascii: trailing_non_ascii,
+        trailing_ascii: trailing_ascii
+      }
+    end
+
+    def ascii_whitespace?(char)
+      [" ", "\t", "\r", "\n"].include?(char)
+    end
+
+    def whitespace?(char)
+      char.match?(/[[:space:]]/)
+    end
+
+    def flanked_by_whitespace?(side)
+      sibling, pattern =
+        if side == :left
+          [raw.previous_sibling, / $/]
+        else
+          [raw.next_sibling, /^ /]
+        end
+
+      return false unless sibling
+
+      if sibling.text? || sibling.cdata?
+        sibling.content.match?(pattern)
+      elsif options[:preformatted_code] && sibling.element? && sibling.name.casecmp("code").zero?
+        false
+      elsif sibling.element? && !Utilities.is_block?(self.class.new(sibling, options))
+        sibling.text.match?(pattern)
+      else
+        false
+      end
+    end
+  end
+end

data/lib/turndown/plugins/gfm.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+# Ported from turndown-plugin-gfm by Dom Christie
+# (https://github.com/mixmark-io/turndown-plugin-gfm), released under the
+# MIT License. See LICENSE for the full notice.
+
+module Turndown
+  module Plugins
+    module GFM
+      HIGHLIGHT_REGEXP = /highlight-(?:text|source)-([a-z0-9]+)/
+      ALIGN_MAP = {
+        "left" => ":--",
+        "right" => "--:",
+        "center" => ":-:"
+      }.freeze
+
+      module_function
+
+      def call(converter)
+        converter.use([
+          HighlightedCodeBlock,
+          Strikethrough,
+          Tables,
+          TaskListItems
+        ])
+      end
+
+      module Strikethrough
+        module_function
+
+        def call(converter)
+          converter.add_rule(
+            :strikethrough,
+            filter: %w[del s strike],
+            replacement: ->(content, _node, _options) { "~#{content}~" }
+          )
+        end
+      end
+
+      module TaskListItems
+        module_function
+
+        def call(converter)
+          converter.add_rule(
+            :task_list_items,
+            filter: lambda do |node, _options|
+              node.type == "checkbox" && node.parent_node&.node_name == "LI"
+            end,
+            replacement: ->(_content, node, _options) { "#{node.checked? ? '[x]' : '[ ]'} " }
+          )
+        end
+      end
+
+      module HighlightedCodeBlock
+        module_function
+
+        def call(converter)
+          converter.add_rule(
+            :highlighted_code_block,
+            filter: lambda do |node, _options|
+              first_child = node.first_child
+              node.node_name == "DIV" &&
+                node.class_name.match?(HIGHLIGHT_REGEXP) &&
+                first_child &&
+                first_child.node_name == "PRE"
+            end,
+            replacement: lambda do |_content, node, options|
+              language = node.class_name[HIGHLIGHT_REGEXP, 1].to_s
+              "\n\n#{options[:fence]}#{language}\n#{node.first_child.text_content}\n#{options[:fence]}\n\n"
+            end
+          )
+        end
+      end
+
+      module Tables
+        module_function
+
+        def call(converter)
+          converter.keep(lambda { |node, _options| node.node_name == "TABLE" && node.rows.none? { |row| heading_row?(row) } })
+
+          converter.add_rule(
+            :table_cell,
+            filter: %w[th td],
+            replacement: ->(content, node, _options) { cell(content, node) }
+          )
+
+          converter.add_rule(
+            :table_row,
+            filter: "tr",
+            replacement: lambda do |content, node, _options|
+              next "" if empty_row?(node)
+
+              border_cells = String.new
+
+              if heading_row?(node)
+                node.child_nodes.each do |child|
+                  next unless %w[TH TD].include?(child.node_name)
+
+                  border = ALIGN_MAP.fetch(child.get_attribute("align").to_s.downcase, "---")
+                  border_cells << cell(border, child)
+                end
+              end
+
+              "\n#{content}#{border_cells.empty? ? '' : "\n#{border_cells}"}"
+            end
+          )
+
+          converter.add_rule(
+            :table,
+            filter: lambda do |node, _options|
+              node.node_name == "TABLE" && node.rows.any? { |row| heading_row?(row) }
+            end,
+            replacement: lambda do |content, _node, _options|
+              "\n\n#{content.sub("\n\n", "\n")}\n\n"
+            end
+          )
+
+          converter.add_rule(
+            :table_section,
+            filter: %w[thead tbody tfoot],
+            replacement: ->(content, _node, _options) { content }
+          )
+        end
+
+        def heading_row?(row)
+          return false unless row
+
+          parent = row.parent_node
+          cells = row.child_nodes.select(&:element?)
+          return false if cells.empty?
+
+          if parent&.node_name == "THEAD"
+            cells.any? { |child| !child.text_content.empty? }
+          else
+            return false unless cells.all? { |child| child.node_name == "TH" }
+
+            parent &&
+              row.same_node?(parent.first_child || row) &&
+              (parent.node_name == "TABLE" || first_tbody?(parent))
+          end
+        end
+
+        def first_tbody?(element)
+          previous_sibling = element.previous_sibling
+          element.node_name == "TBODY" &&
+            (
+              previous_sibling.nil? ||
+              (
+                previous_sibling.node_name == "THEAD" &&
+                previous_sibling.text_content.match?(/\A\s*\z/)
+              )
+            )
+        end
+
+        def cell(content, node)
+          index = node.parent_node.children.find_index { |child| child.same_node?(node) } || 0
+          prefix = index.zero? ? "| " : " "
+          "#{prefix}#{content} |"
+        end
+
+        def empty_row?(row)
+          return false if heading_row?(row)
+
+          cells = row.child_nodes.select(&:element?)
+          cells.any? && cells.all? { |cell| cell.text_content.empty? }
+        end
+      end
+    end
+  end
+end

data/lib/turndown/root_node.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Turndown
+  class RootNode
+    ROOT_ID = "turndown-root".freeze
+
+    class << self
+      def build(input, options)
+        root =
+          if input.is_a?(String)
+            document = Nokogiri::HTML5::Document.parse(%(<x-turndown id="#{ROOT_ID}">#{input}</x-turndown>))
+            document.at_css("##{ROOT_ID}")
+          else
+            input.dup
+          end
+
+        normalize_pre(root)
+        CollapseWhitespace.call(
+          element: root,
+          is_block: ->(node) { Utilities.is_block?(node) },
+          is_void: ->(node) { Utilities.is_void?(node) },
+          is_pre: options[:preformatted_code] ? ->(node) { pre_or_code?(node) } : nil
+        )
+
+        root
+      end
+
+      private
+
+      def pre_or_code?(node)
+        return false unless node.element?
+
+        %w[pre code].include?(node.name.downcase)
+      end
+
+      def normalize_pre(root)
+        root.css("pre br").each do |node|
+          node.replace(Nokogiri::XML::Text.new("\n", node.document))
+        end
+      end
+    end
+  end
+end

data/lib/turndown/rule.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Turndown
+  class Rule
+    attr_reader :filter
+
+    def initialize(filter: nil, replacement:, append: nil)
+      @filter = filter
+      @replacement = replacement
+      @append = append
+    end
+
+    def matches?(node, options)
+      case filter
+      when String, Symbol
+        filter.to_s == node.node_name_downcase
+      when Array
+        filter.map(&:to_s).include?(node.node_name_downcase)
+      when Proc
+        filter.call(node, options)
+      when nil
+        false
+      else
+        raise TypeError, "`filter` needs to be a string, array, or function"
+      end
+    end
+
+    def replacement(content, node, options)
+      @replacement.call(content, node, options)
+    end
+
+    def append(options)
+      return "" unless @append
+
+      @append.call(options)
+    end
+  end
+end

data/lib/turndown/rules.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Turndown
+  class Rules
+    def initialize(options)
+      @options = options
+      @keep = []
+      @remove = []
+      @blank_rule = Rule.new(replacement: options[:blank_replacement])
+      @keep_replacement = options[:keep_replacement]
+      @default_rule = Rule.new(replacement: options[:default_replacement])
+      @array = options.fetch(:rules).values.dup
+    end
+
+    def add(_key, rule)
+      @array.unshift(rule)
+    end
+
+    def keep(filter)
+      @keep.unshift(
+        Rule.new(
+          filter: filter,
+          replacement: @keep_replacement
+        )
+      )
+    end
+
+    def remove(filter)
+      @remove.unshift(
+        Rule.new(
+          filter: filter,
+          replacement: ->(_content, _node, _options) { "" }
+        )
+      )
+    end
+
+    def for_node(node)
+      return @blank_rule if node.blank?
+
+      find_rule(@array, node) || find_rule(@keep, node) || find_rule(@remove, node) || @default_rule
+    end
+
+    def each(&block)
+      @array.each(&block)
+    end
+
+    private
+
+    def find_rule(rules, node)
+      rules.find { |rule| rule.matches?(node, @options) }
+    end
+  end
+end