markdown_composer 0.7.0

data/lib/markdown_composer/diagnostics.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module MarkdownComposer
+  Diagnostic = MarkdownComposer.value_object(:severity, :code, :message, :path, :details) do
+    def to_h
+      {
+        severity: severity,
+        code: code,
+        message: message,
+        path: path,
+        details: details
+      }.compact
+    end
+  end
+
+  class Diagnostics
+    attr_reader :items
+
+    def initialize(items = [])
+      @items = items
+    end
+
+    def info(code, message, path: nil, details: nil)
+      add(:info, code, message, path: path, details: details)
+    end
+
+    def warn(code, message, path: nil, details: nil)
+      add(:warning, code, message, path: path, details: details)
+    end
+
+    def error(code, message, path: nil, details: nil)
+      add(:error, code, message, path: path, details: details)
+    end
+
+    def concat(other)
+      @items.concat(other.to_a)
+      self
+    end
+
+    def errors
+      @items.select { |item| item.severity == :error }
+    end
+
+    def any_errors?
+      errors.any?
+    end
+
+    def to_a
+      @items
+    end
+
+    def as_json
+      @items.map(&:to_h)
+    end
+
+    private
+
+    def add(severity, code, message, path:, details:)
+      @items << Diagnostic.new(severity: severity, code: code, message: message, path: path, details: details)
+      self
+    end
+  end
+end

data/lib/markdown_composer/document_index/html_parser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+begin
+  require "nokogiri"
+rescue LoadError
+  nil
+end
+
+module MarkdownComposer
+  class DocumentIndex
+    class HtmlParser
+      attr_reader :source, :diagnostics
+
+      def initialize(source, diagnostics:)
+        @source = source
+        @diagnostics = diagnostics
+      end
+
+      def index
+        fragment = Nokogiri::HTML5.fragment(source.html.to_s)
+        markdown = html_to_markdown(fragment)
+        DocumentIndex.from_markdown(markdown, source_key: source.key, diagnostics: diagnostics)
+      end
+
+      private
+
+      def html_to_markdown(fragment)
+        fragment.children.map { |node| node_to_markdown(node) }.join.lines.map(&:lstrip).join
+      end
+
+      def node_to_markdown(node)
+        return node.text if node.text?
+
+        case node.name
+        when /^h([1-6])$/
+          "#{"#" * Regexp.last_match(1).to_i} #{node.text.strip}\n\n"
+        when "p"
+          "#{inline_html(node).strip}\n\n"
+        when "a"
+          "[#{node.text.strip}](#{node["href"]})"
+        when "img"
+          "![#{node["alt"]}](#{node["src"]})\n\n"
+        when "ul"
+          node.css("> li").map { |li| "- #{inline_html(li).strip}\n" }.join + "\n"
+        when "ol"
+          node.css("> li").each_with_index.map { |li, index| "#{index + 1}. #{inline_html(li).strip}\n" }.join + "\n"
+        when "blockquote"
+          node.text.lines.map { |line| "> #{line.strip}\n" }.join + "\n"
+        when "pre"
+          code = node.at_css("code")
+          language = code&.classes&.find { |klass| klass.start_with?("language-") }&.sub("language-", "")
+          "```#{language}\n#{code&.text || node.text}\n```\n\n"
+        when "table"
+          rows = node.css("tr").map { |tr| tr.css("th,td").map { |cell| cell.text.strip } }
+          return "" if rows.empty?
+
+          header = "| #{rows.first.join(" | ")} |\n"
+          separator = "| #{rows.first.map { "---" }.join(" | ")} |\n"
+          body = rows[1..].to_a.map { |row| "| #{row.join(" | ")} |\n" }.join
+          "#{header}#{separator}#{body}\n"
+        else
+          children = node.children.map { |child| node_to_markdown(child) }.join
+          children.empty? ? "#{node.to_html}\n" : children
+        end
+      end
+
+      def inline_html(node)
+        node.children.map do |child|
+          if child.text?
+            child.text
+          elsif child.name == "a"
+            "[#{child.text.strip}](#{child["href"]})"
+          elsif child.name == "img"
+            "![#{child["alt"]}](#{child["src"]})"
+          elsif child.name == "code"
+            "`#{child.text}`"
+          else
+            inline_html(child)
+          end
+        end.join
+      end
+    end
+  end
+end

data/lib/markdown_composer/document_index/markdown_parser.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+module MarkdownComposer
+  class DocumentIndex
+    class MarkdownParser
+      attr_reader :lines, :source_key, :diagnostics
+
+      def initialize(markdown, source_key:, diagnostics:)
+        @lines = markdown.lines
+        @source_key = source_key
+        @diagnostics = diagnostics
+        @nodes = []
+        @position = 0
+      end
+
+      def index
+        scan_nodes
+        build_sections
+      end
+
+      private
+
+      def scan_nodes
+        index = 0
+        if lines.first&.match?(/\A---\s*$/)
+          closing = lines[1..]&.find_index { |line| line.match?(/\A---\s*$/) }
+          if closing
+            end_index = closing + 1
+            add_node("data_block", lines[0..end_index], 0, end_index, text: lines[1...end_index].join.strip, attributes: { "format" => "yaml", "location" => "frontmatter" })
+            index = end_index + 1
+          end
+        end
+
+        while index < lines.length
+          line = lines[index]
+          if line.strip.empty?
+            index += 1
+            next
+          end
+
+          case line
+          when /\A\$\$\s*$/
+            start = index
+            index += 1
+            index += 1 while index < lines.length && !lines[index].match?(/\A\$\$\s*$/)
+            index += 1 if index < lines.length
+            add_node("math_block", lines[start...index], start, index - 1)
+          when /\A(```|~~~)\s*([A-Za-z0-9_-]+)?/
+            fence = Regexp.last_match(1)
+            language = Regexp.last_match(2).to_s
+            start = index
+            index += 1
+            index += 1 while index < lines.length && !lines[index].start_with?(fence)
+            closing_index = index
+            closed = index < lines.length
+            index += 1 if closed
+            body_lines = lines[(start + 1)...(closed ? closing_index : index)].to_a
+            language_token = language.downcase
+            type = language_token == "mermaid" ? "mermaid" : "code_block"
+            type = "data_block" if %w[yaml yml json].include?(language_token)
+            format = language_token == "yml" ? "yaml" : language_token
+            attributes = { "language" => language_token, "format" => format }
+            attributes["location"] = "body" if type == "data_block"
+            attributes["diagram_type"] = mermaid_diagram_type(body_lines.join) if type == "mermaid"
+            add_node(type, lines[start...index], start, index - 1, text: body_lines.join.strip, attributes: attributes)
+          when /\A\s{0,3}(\#{1,6})\s+(.+?)\s*#*\s*$/
+            level = Regexp.last_match(1).length
+            text = clean_text(Regexp.last_match(2))
+            add_node("heading_#{level}", [ line ], index, index, level: level, text: text)
+            index += 1
+          when /\A\|.*\|\s*$/
+            start = index
+            index += 1 while index < lines.length && lines[index].match?(/\A\|.*\|\s*$/)
+            table_lines = lines[start...index]
+            add_node("table", table_lines, start, index - 1, attributes: table_attributes(table_lines))
+            add_table_part_nodes(table_lines, start)
+          when /\A\s*([-*+])\s+/
+            start = index
+            index += 1 while index < lines.length && lines[index].match?(/\A\s*([-*+])\s+/)
+            item_total = index - start
+            add_node("unordered_list", lines[start...index], start, index - 1, attributes: { "ordered" => false, "item_count" => item_total })
+            lines[start...index].each_with_index do |item_line, offset|
+              add_node("list_item", [ item_line ], start + offset, start + offset, text: clean_text(item_line.sub(/\A\s*[-*+]\s+/, "")), attributes: { "ordered" => false, "item_index" => offset + 1, "derived" => true })
+            end
+          when /\A\s*\d+\.\s+/
+            start = index
+            index += 1 while index < lines.length && lines[index].match?(/\A\s*\d+\.\s+/)
+            item_total = index - start
+            add_node("ordered_list", lines[start...index], start, index - 1, attributes: { "ordered" => true, "item_count" => item_total })
+            lines[start...index].each_with_index do |item_line, offset|
+              add_node("list_item", [ item_line ], start + offset, start + offset, text: clean_text(item_line.sub(/\A\s*\d+\.\s+/, "")), attributes: { "ordered" => true, "item_index" => offset + 1, "derived" => true })
+            end
+          when /\A>\s?/
+            start = index
+            index += 1 while index < lines.length && lines[index].match?(/\A>\s?/)
+            add_node("blockquote", lines[start...index], start, index - 1)
+          when /\A<!--.*-->\s*$/
+            add_node("comment", [ line ], index, index, text: line.gsub(/\A<!--|-->\s*\z/, "").strip)
+            index += 1
+          when /\A<[^>]+>/
+            start = index
+            index += 1
+            add_node("raw_html", lines[start...index], start, index - 1)
+          else
+            start = index
+            index += 1
+            while index < lines.length && paragraph_continues?(lines[index])
+              index += 1
+            end
+            add_node("paragraph", lines[start...index], start, index - 1)
+          end
+        end
+      end
+
+      def paragraph_continues?(line)
+        !line.strip.empty? &&
+          !line.match?(/\A(```|~~~)|\A\$\$\s*$|\A\s{0,3}\#{1,6}\s+|\A\|.*\|\s*$|\A\s*([-*+]|\d+\.)\s+|\A>\s?|\A<[^>]+>/)
+      end
+
+      def add_node(type, raw_lines, start_index, end_index, level: nil, text: nil, attributes: {})
+        raw = raw_lines.join
+        node = ComposerNode.new(
+          id: "#{source_key}:n#{@position + 1}",
+          source_key: source_key,
+          type: type,
+          source_position: @position + 1,
+          level: level,
+          text: text || clean_text(raw),
+          attributes: attributes,
+          children: inline_nodes(raw, start_index),
+          raw: raw.end_with?("\n") ? raw : "#{raw}\n",
+          start_line: start_index + 1,
+          end_line: end_index + 1
+        )
+        @nodes << node
+        @position += 1
+        node
+      end
+
+      def add_table_part_nodes(table_lines, start_index)
+        return if table_lines.empty?
+
+        header_line = table_lines.first
+        separator_index = table_lines.find_index { |line| line.match?(/\A\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/) }
+        body_start = separator_index ? separator_index + 1 : 1
+        column_count = table_columns(header_line).length
+        add_node("table_head", [ header_line ], start_index, start_index, attributes: { "section" => "table_head", "row_count" => 1, "column_count" => column_count, "derived" => true })
+        visible_row_index = 0
+        table_lines.each_with_index do |row_line, offset|
+          next if offset == separator_index
+
+          visible_row_index += 1
+          row_type = offset.zero? ? "table_head" : "table_body"
+          cells = table_columns(row_line)
+          add_node("table_row", [ row_line ], start_index + offset, start_index + offset, attributes: { "section" => row_type, "row_index" => visible_row_index, "column_count" => cells.length, "derived" => true })
+          cells.each_with_index do |cell, column_index|
+            add_node(offset < body_start ? "table_header" : "table_cell", [ "#{cell}\n" ], start_index + offset, start_index + offset, text: cell, attributes: { "section" => row_type, "row_index" => visible_row_index, "column_index" => column_index + 1, "derived" => true })
+          end
+        end
+        body_lines = table_lines[body_start..].to_a
+        if body_lines.any?
+          body_row_count = body_lines.count { |line| !line.match?(/\A\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/) }
+          add_node("table_body", body_lines, start_index + body_start, start_index + table_lines.length - 1, attributes: { "section" => "table_body", "row_count" => body_row_count, "column_count" => column_count, "derived" => true })
+        end
+      end
+
+      def table_attributes(table_lines)
+        visible_rows = table_lines.reject { |line| line.match?(/\A\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/) }
+        { "row_count" => visible_rows.length, "column_count" => table_columns(table_lines.first).length }
+      end
+
+      def table_columns(row_line)
+        row_line.to_s.strip.sub(/\A\|/, "").sub(/\|\z/, "").split("|").map(&:strip)
+      end
+
+      def inline_nodes(raw, start_index)
+        children = []
+        raw.scan(/!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]+)")?\)/).each_with_index do |(alt, src, title), offset|
+          children << inline_node("image", alt, start_index, offset, { "alt" => alt, "src" => src, "title" => title }.compact)
+        end
+        raw.scan(/(?<!!)\[([^\]]+)\]\(([^)\s]+)(?:\s+"([^"]+)")?\)/).each_with_index do |(text, href, title), offset|
+          children << inline_node("link", text, start_index, offset, { "href" => href, "title" => title }.compact)
+        end
+        raw.scan(/`([^`\n]+)`/).each_with_index do |(code), offset|
+          children << inline_node("inline_code", code, start_index, offset, {})
+        end
+        raw.scan(/\$([^$\n]+)\$/).each_with_index do |(math), offset|
+          children << inline_node("inline_math", math, start_index, offset, {})
+        end
+        children
+      end
+
+      def inline_node(type, text, start_index, offset, attributes)
+        ComposerNode.new(
+          id: "#{source_key}:i#{@position + 1}:#{type}:#{offset}",
+          source_key: source_key,
+          type: type,
+          source_position: @position + 1,
+          level: nil,
+          text: text,
+          attributes: attributes,
+          children: [],
+          raw: inline_raw(type, text, attributes),
+          start_line: start_index + 1,
+          end_line: start_index + 1
+        )
+      end
+
+      def inline_raw(type, text, attributes)
+        case type
+        when "link" then "[#{text}](#{attributes["href"]})"
+        when "image" then "![#{text}](#{attributes["src"]})"
+        when "inline_code" then "`#{text}`"
+        when "inline_math" then "$#{text}$"
+        else text.to_s
+        end
+      end
+
+      def build_sections
+        root_body = []
+        root_children = []
+        sections = []
+        stack = []
+
+        @nodes.each do |node|
+          if node.heading?
+            stack.pop while stack.any? && stack.last.level >= node.level
+            parent = stack.last
+            section = MutableSection.new(
+              id: "#{source_key}:s#{sections.length + 1}",
+              source_key: source_key,
+              heading_node: node,
+              level: node.level,
+              title_text: node.text,
+              body_nodes: [],
+              child_sections: [],
+              source_position: node.source_position,
+              parent_section_id: parent&.id,
+              start_line: node.start_line
+            )
+            parent ? parent.child_sections << section : root_children << section
+            sections << section
+            stack << section
+          elsif stack.any?
+            stack.last.body_nodes << node
+          else
+            root_body << node
+          end
+        end
+
+        immutable_sections = sections.map { |section| section.to_immutable(lines.length) }
+        by_id = immutable_sections.to_h { |section| [ section.id, section ] }
+        resolved_sections = {}
+        immutable_sections = immutable_sections.map { |section| resolve_section_children(section, by_id, resolved_sections) }
+        by_id = immutable_sections.to_h { |section| [ section.id, section ] }
+        root = ComposerSection.new(
+          id: "#{source_key}:root",
+          source_key: source_key,
+          heading_node: nil,
+          level: 0,
+          title_text: nil,
+          body_nodes: root_body,
+          child_sections: root_children.map { |child| by_id.fetch(child.id) },
+          all_nodes: (@nodes - []),
+          source_position: 0,
+          parent_section_id: nil,
+          start_line: 1,
+          end_line: lines.length
+        )
+
+        DocumentIndex.new(source_key: source_key, root: root, nodes: @nodes, sections: immutable_sections, diagnostics: diagnostics, lines: lines)
+      end
+
+      def resolve_section_children(section, by_id, resolved_sections)
+        return resolved_sections[section.id] if resolved_sections.key?(section.id)
+
+        resolved = section.with(
+          child_sections: section.child_sections.map do |child|
+            resolve_section_children(by_id.fetch(child.id), by_id, resolved_sections)
+          end
+        )
+        resolved_sections[section.id] = resolved
+      end
+
+      def clean_text(raw)
+        raw.to_s
+          .gsub(/```.*?```/m, "")
+          .gsub(/!\[([^\]]*)\]\([^)]+\)/, "\\1")
+          .gsub(/\[([^\]]+)\]\([^)]+\)/, "\\1")
+          .gsub(/[`*_>#|-]/, " ")
+          .gsub(/\s+/, " ")
+          .strip
+      end
+
+      def mermaid_diagram_type(source)
+        first = source.to_s.lines.map(&:strip).reject { |line| line.empty? || line.start_with?("%%") }.first.to_s
+        case first
+        when /\A(?:flowchart|graph)\b/i then "flowchart"
+        when /\AsequenceDiagram\b/i then "sequence"
+        when /\AclassDiagram\b/i then "class"
+        when /\AstateDiagram(?:-v2)?\b/i then "state"
+        when /\AerDiagram\b/i then "er"
+        when /\Agantt\b/i then "gantt"
+        when /\Apie\b/i then "pie"
+        when /\Ajourney\b/i then "journey"
+        when /\AgitGraph\b/i then "gitgraph"
+        when /\Amindmap\b/i then "mindmap"
+        when /\Atimeline\b/i then "timeline"
+        when /\AquadrantChart\b/i then "quadrant"
+        when /\Axychart-beta\b/i then "xy"
+        when /\Asankey-beta\b/i then "sankey"
+        else "unknown"
+        end
+      end
+
+      MutableSection = Struct.new(:id, :source_key, :heading_node, :level, :title_text, :body_nodes, :child_sections, :source_position, :parent_section_id, :start_line, keyword_init: true) do
+        def to_immutable(document_end)
+          all = body_nodes + child_sections.flat_map { |child| [ child.heading_node, *child.to_immutable(document_end).all_nodes ] }
+          end_line = ([ heading_node&.end_line, *all.map(&:end_line) ].compact.max || start_line)
+          ComposerSection.new(
+            id: id,
+            source_key: source_key,
+            heading_node: heading_node,
+            level: level,
+            title_text: title_text,
+            body_nodes: body_nodes,
+            child_sections: child_sections,
+            all_nodes: all,
+            source_position: source_position,
+            parent_section_id: parent_section_id,
+            start_line: start_line,
+            end_line: [ end_line, document_end ].min
+          )
+        end
+      end
+    end
+  end
+end

data/lib/markdown_composer/document_index.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module MarkdownComposer
+  ComposerNode = MarkdownComposer.value_object(:id, :source_key, :type, :source_position, :level, :text, :attributes, :children, :raw, :start_line, :end_line) do
+    def heading?
+      type.start_with?("heading_")
+    end
+
+    def to_h
+      {
+        id: id,
+        source_key: source_key,
+        type: type,
+        source_position: source_position,
+        level: level,
+        text: text,
+        attributes: attributes,
+        children: children.map(&:to_h),
+        raw: raw,
+        start_line: start_line,
+        end_line: end_line
+      }
+    end
+  end
+
+  ComposerSection = MarkdownComposer.value_object(:id, :source_key, :heading_node, :level, :title_text, :body_nodes, :child_sections, :all_nodes, :source_position, :parent_section_id, :start_line, :end_line) do
+    def text
+      [ title_text, *all_nodes.map(&:text) ].compact.join("\n")
+    end
+
+    def raw
+      [ heading_node, *all_nodes.reject { |node| node.attributes["derived"] } ].compact.uniq(&:id).sort_by(&:source_position).map(&:raw).join
+    end
+
+    def to_h
+      {
+        id: id,
+        source_key: source_key,
+        heading_node: heading_node&.to_h,
+        level: level,
+        title_text: title_text,
+        body_nodes: body_nodes.map(&:to_h),
+        child_sections: child_sections.map(&:id),
+        all_nodes: all_nodes.map(&:id),
+        source_position: source_position,
+        parent_section_id: parent_section_id,
+        start_line: start_line,
+        end_line: end_line
+      }
+    end
+  end
+
+  class DocumentIndex
+    attr_reader :source_key, :root, :nodes, :sections, :diagnostics, :lines
+
+    def self.build(source, diagnostics: Diagnostics.new)
+      source = Source.build(source)
+      if source.format == :html
+        from_html(source, diagnostics: diagnostics)
+      else
+        from_markdown(source.markdown.to_s, source_key: source.key, diagnostics: diagnostics)
+      end
+    end
+
+    def self.from_markdown(markdown, source_key: "buffer", diagnostics: Diagnostics.new)
+      MarkdownParser.new(markdown.to_s, source_key: source_key, diagnostics: diagnostics).index
+    end
+
+    def self.from_html(source, diagnostics:)
+      if defined?(Nokogiri)
+        HtmlParser.new(source, diagnostics: diagnostics).index
+      else
+        diagnostics.error("source.html_parser_missing", "HTML input requires nokogiri", path: "sources.#{source.key}")
+        from_markdown(source.html.to_s, source_key: source.key, diagnostics: diagnostics)
+      end
+    end
+
+    def initialize(source_key:, root:, nodes:, sections:, diagnostics:, lines:)
+      @source_key = source_key
+      @root = root
+      @nodes = nodes
+      @sections = sections
+      @diagnostics = diagnostics
+      @lines = lines
+    end
+
+    def markdown_for_range(start_line, end_line)
+      lines[(start_line - 1)..(end_line - 1)].join
+    end
+  end
+end
+
+require_relative "document_index/markdown_parser"
+require_relative "document_index/html_parser"