markly 0.1.0

data/ext/markly/utf8.h

@@ -0,0 +1,35 @@
+#ifndef CMARK_UTF8_H
+#define CMARK_UTF8_H
+
+#include <stdint.h>
+#include "buffer.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+CMARK_GFM_EXPORT
+void cmark_utf8proc_case_fold(cmark_strbuf *dest, const uint8_t *str,
+                              bufsize_t len);
+
+CMARK_GFM_EXPORT
+void cmark_utf8proc_encode_char(int32_t uc, cmark_strbuf *buf);
+
+CMARK_GFM_EXPORT
+int cmark_utf8proc_iterate(const uint8_t *str, bufsize_t str_len, int32_t *dst);
+
+CMARK_GFM_EXPORT
+void cmark_utf8proc_check(cmark_strbuf *dest, const uint8_t *line,
+                          bufsize_t size);
+
+CMARK_GFM_EXPORT
+int cmark_utf8proc_is_space(int32_t uc);
+
+CMARK_GFM_EXPORT
+int cmark_utf8proc_is_punctuation(int32_t uc);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/ext/markly/xml.c

@@ -0,0 +1,181 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include "config.h"
+#include "cmark-gfm.h"
+#include "node.h"
+#include "buffer.h"
+#include "houdini.h"
+#include "syntax_extension.h"
+
+#define BUFFER_SIZE 100
+
+// Functions to convert cmark_nodes to XML strings.
+
+static void escape_xml(cmark_strbuf *dest, const unsigned char *source,
+                       bufsize_t length) {
+  houdini_escape_html0(dest, source, length, 0);
+}
+
+struct render_state {
+  cmark_strbuf *xml;
+  int indent;
+};
+
+static CMARK_INLINE void indent(struct render_state *state) {
+  int i;
+  for (i = 0; i < state->indent; i++) {
+    cmark_strbuf_putc(state->xml, ' ');
+  }
+}
+
+static int S_render_node(cmark_node *node, cmark_event_type ev_type,
+                         struct render_state *state, int options) {
+  cmark_strbuf *xml = state->xml;
+  bool literal = false;
+  cmark_delim_type delim;
+  bool entering = (ev_type == CMARK_EVENT_ENTER);
+  char buffer[BUFFER_SIZE];
+
+  if (entering) {
+    indent(state);
+    cmark_strbuf_putc(xml, '<');
+    cmark_strbuf_puts(xml, cmark_node_get_type_string(node));
+
+    if (options & CMARK_OPT_SOURCEPOS && node->start_line != 0) {
+      snprintf(buffer, BUFFER_SIZE, " sourcepos=\"%d:%d-%d:%d\"",
+               node->start_line, node->start_column, node->end_line,
+               node->end_column);
+      cmark_strbuf_puts(xml, buffer);
+    }
+
+    if (node->extension && node->extension->xml_attr_func) {
+      const char* r = node->extension->xml_attr_func(node->extension, node);
+      if (r != NULL)
+        cmark_strbuf_puts(xml, r);
+    }
+
+    literal = false;
+
+    switch (node->type) {
+    case CMARK_NODE_DOCUMENT:
+      cmark_strbuf_puts(xml, " xmlns=\"http://commonmark.org/xml/1.0\"");
+      break;
+    case CMARK_NODE_TEXT:
+    case CMARK_NODE_CODE:
+    case CMARK_NODE_HTML_BLOCK:
+    case CMARK_NODE_HTML_INLINE:
+      cmark_strbuf_puts(xml, " xml:space=\"preserve\">");
+      escape_xml(xml, node->as.literal.data, node->as.literal.len);
+      cmark_strbuf_puts(xml, "</");
+      cmark_strbuf_puts(xml, cmark_node_get_type_string(node));
+      literal = true;
+      break;
+    case CMARK_NODE_LIST:
+      switch (cmark_node_get_list_type(node)) {
+      case CMARK_ORDERED_LIST:
+        cmark_strbuf_puts(xml, " type=\"ordered\"");
+        snprintf(buffer, BUFFER_SIZE, " start=\"%d\"",
+                 cmark_node_get_list_start(node));
+        cmark_strbuf_puts(xml, buffer);
+        delim = cmark_node_get_list_delim(node);
+        if (delim == CMARK_PAREN_DELIM) {
+          cmark_strbuf_puts(xml, " delim=\"paren\"");
+        } else if (delim == CMARK_PERIOD_DELIM) {
+          cmark_strbuf_puts(xml, " delim=\"period\"");
+        }
+        break;
+      case CMARK_BULLET_LIST:
+        cmark_strbuf_puts(xml, " type=\"bullet\"");
+        break;
+      default:
+        break;
+      }
+      snprintf(buffer, BUFFER_SIZE, " tight=\"%s\"",
+               (cmark_node_get_list_tight(node) ? "true" : "false"));
+      cmark_strbuf_puts(xml, buffer);
+      break;
+    case CMARK_NODE_HEADING:
+      snprintf(buffer, BUFFER_SIZE, " level=\"%d\"", node->as.heading.level);
+      cmark_strbuf_puts(xml, buffer);
+      break;
+    case CMARK_NODE_CODE_BLOCK:
+      if (node->as.code.info.len > 0) {
+        cmark_strbuf_puts(xml, " info=\"");
+        escape_xml(xml, node->as.code.info.data, node->as.code.info.len);
+        cmark_strbuf_putc(xml, '"');
+      }
+      cmark_strbuf_puts(xml, " xml:space=\"preserve\">");
+      escape_xml(xml, node->as.code.literal.data, node->as.code.literal.len);
+      cmark_strbuf_puts(xml, "</");
+      cmark_strbuf_puts(xml, cmark_node_get_type_string(node));
+      literal = true;
+      break;
+    case CMARK_NODE_CUSTOM_BLOCK:
+    case CMARK_NODE_CUSTOM_INLINE:
+      cmark_strbuf_puts(xml, " on_enter=\"");
+      escape_xml(xml, node->as.custom.on_enter.data,
+                 node->as.custom.on_enter.len);
+      cmark_strbuf_putc(xml, '"');
+      cmark_strbuf_puts(xml, " on_exit=\"");
+      escape_xml(xml, node->as.custom.on_exit.data,
+                 node->as.custom.on_exit.len);
+      cmark_strbuf_putc(xml, '"');
+      break;
+    case CMARK_NODE_LINK:
+    case CMARK_NODE_IMAGE:
+      cmark_strbuf_puts(xml, " destination=\"");
+      escape_xml(xml, node->as.link.url.data, node->as.link.url.len);
+      cmark_strbuf_putc(xml, '"');
+      cmark_strbuf_puts(xml, " title=\"");
+      escape_xml(xml, node->as.link.title.data, node->as.link.title.len);
+      cmark_strbuf_putc(xml, '"');
+      break;
+    default:
+      break;
+    }
+    if (node->first_child) {
+      state->indent += 2;
+    } else if (!literal) {
+      cmark_strbuf_puts(xml, " /");
+    }
+    cmark_strbuf_puts(xml, ">\n");
+
+  } else if (node->first_child) {
+    state->indent -= 2;
+    indent(state);
+    cmark_strbuf_puts(xml, "</");
+    cmark_strbuf_puts(xml, cmark_node_get_type_string(node));
+    cmark_strbuf_puts(xml, ">\n");
+  }
+
+  return 1;
+}
+
+char *cmark_render_xml(cmark_node *root, int options) {
+  return cmark_render_xml_with_mem(root, options, cmark_node_mem(root));
+}
+
+char *cmark_render_xml_with_mem(cmark_node *root, int options, cmark_mem *mem) {
+  char *result;
+  cmark_strbuf xml = CMARK_BUF_INIT(mem);
+  cmark_event_type ev_type;
+  cmark_node *cur;
+  struct render_state state = {&xml, 0};
+
+  cmark_iter *iter = cmark_iter_new(root);
+
+  cmark_strbuf_puts(state.xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+  cmark_strbuf_puts(state.xml,
+                    "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n");
+  while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
+    cur = cmark_iter_get_node(iter);
+    S_render_node(cur, ev_type, &state, options);
+  }
+  result = (char *)cmark_strbuf_detach(&xml);
+
+  cmark_iter_free(iter);
+  return result;
+}

data/lib/markly.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# The compiled library.
+require_relative 'markly/markly'
+
+require_relative 'markly/flags'
+require_relative 'markly/node'
+require_relative 'markly/renderer'
+require_relative 'markly/renderer/html_renderer'
+require_relative 'markly/version'
+
+module Markly
+  # Public: Parses a Markdown string into a `document` node.
+  #
+  # string - {String} to be parsed
+  # option - A {Symbol} or {Array of Symbol}s indicating the parse options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
+  #
+  # Returns the `parser` node.
+  def self.parse(text, flags: DEFAULT, extensions: nil)
+    parser = Parser.new(flags)
+    
+    extensions&.each do |extension|
+      parser.enable(extension)
+    end
+    
+    return parser.parse(text.encode(Encoding::UTF_8))
+  end
+  
+  # Public:  Parses a Markdown string into an HTML string.
+  #
+  # text - A {String} of text
+  # option - Either a {Symbol} or {Array of Symbol}s indicating the render options
+  # extensions - An {Array of Symbol}s indicating the extensions to use
+  #
+  # Returns a {String} of converted HTML.
+  def self.render_html(text, flags: DEFAULT, parse_flags: flags, render_flags: flags, extensions: [])
+    root = self.parse(text, flags: parse_flags, extensions: extensions)
+    
+    return root.to_html(flags: render_flags, extensions: extensions)
+  end
+end

data/lib/markly/flags.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Markly
+	DEFAULT = 0
+	VALIDATE_UTF8 = 1 << 9
+	SMART = 1 << 10
+	LIBERAL_HTML_TAG = 1 << 12
+	FOOTNOTES = 1 << 13
+	STRIKETHROUGH_DOUBLE_TILDE = 1 << 14
+	UNSAFE = 1 << 17
+	
+	PARSE_FLAGS = {
+		validate_utf8: VALIDATE_UTF8,
+		smart_quotes: SMART,
+		liberal_html_tags: LIBERAL_HTML_TAG,
+		footnotes: FOOTNOTES,
+		strikethrough_double_tilde: STRIKETHROUGH_DOUBLE_TILDE,
+		unsafe: UNSAFE,
+	}
+	
+	SOURCE_POSITION = 1 << 1
+	HARD_BREAKS = 1 << 2
+	NO_BREAKS = 1 << 4
+	GITHUB_PRE_LANG = 1 << 11
+	TABLE_PREFER_STYLE_ATTRIBUTES = 1 << 15
+	FULL_INFO_STRING = 1 << 16
+	
+	RENDER_FLAGS = {
+		source_position: SOURCE_POSITION,
+		hard_breaks: HARD_BREAKS,
+		no_breaks: NO_BREAKS,
+		pre_lang: GITHUB_PRE_LANG,
+		table_prefer_style_attributes: TABLE_PREFER_STYLE_ATTRIBUTES,
+		full_info_string: FULL_INFO_STRING,
+		unsafe: UNSAFE,
+	}
+end

data/lib/markly/node.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative 'node/inspect'
+
+module Markly
+  class Node
+    include Enumerable
+    include Inspect
+
+    # Public: An iterator that "walks the tree," descending into children recursively.
+    #
+    # blk - A {Proc} representing the action to take for each child
+    def walk(&block)
+      return enum_for(:walk) unless block_given?
+
+      yield self
+      each do |child|
+        child.walk(&block)
+      end
+    end
+
+    # Public: Convert the node to an HTML string.
+    #
+    # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    # extensions - An {Array of Symbol}s indicating the extensions to use
+    #
+    # Returns a {String}.
+    def to_html(flags: DEFAULT, extensions: [])
+      _render_html(flags, extensions).force_encoding('utf-8')
+    end
+
+    # Public: Convert the node to a CommonMark string.
+    #
+    # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    # width - Column to wrap the output at
+    #
+    # Returns a {String}.
+    def to_commonmark(flags: DEFAULT, width: 120)
+      _render_commonmark(flags, width).force_encoding('utf-8')
+    end
+
+    # Public: Convert the node to a plain text string.
+    #
+    # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    # width - Column to wrap the output at
+    #
+    # Returns a {String}.
+    def to_plaintext(flags: DEFAULT, width: 120)
+      _render_plaintext(flags, width).force_encoding('utf-8')
+    end
+
+    # Public: Iterate over the children (if any) of the current pointer.
+    def each
+      return enum_for(:each) unless block_given?
+
+      child = first_child
+      while child
+        nextchild = child.next
+        yield child
+        child = nextchild
+      end
+    end
+
+    # Deprecated: Please use `each` instead
+    def each_child(&block)
+      warn '[DEPRECATION] `each_child` is deprecated.  Please use `each` instead.'
+      each(&block)
+    end
+  end
+end

data/lib/markly/node/inspect.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'pp'
+
+module Markly
+  class Node
+    module Inspect
+      PP_INDENT_SIZE = 2
+
+      def inspect
+        PP.pp(self, +'', Float::INFINITY)
+      end
+
+      # @param printer [PrettyPrint] pp
+      def pretty_print(printer)
+        printer.group(PP_INDENT_SIZE, "#<#{self.class}(#{type}):", '>') do
+          printer.breakable
+
+          attrs = %i[
+            source_position
+            string_content
+            url
+            title
+            header_level
+            list_type
+            list_start
+            list_tight
+            fence_info
+          ].map do |name|
+            begin
+              [name, __send__(name)]
+            rescue Error
+              nil
+            end
+          end.compact
+
+          printer.seplist(attrs) do |name, value|
+            printer.text "#{name}="
+            printer.pp value
+          end
+
+          if first_child
+            printer.breakable
+            printer.group(PP_INDENT_SIZE) do
+              children = []
+              node = first_child
+              while node
+                children << node
+                node = node.next
+              end
+              printer.text 'children='
+              printer.pp children
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/markly/renderer.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'stringio'
+
+module Markly
+  class Renderer
+    attr_accessor :in_tight, :warnings, :in_plain
+    def initialize(flags: DEFAULT, extensions: [])
+      @flags = flags
+      @stream = StringIO.new(+'')
+      @need_blocksep = false
+      @warnings = Set.new
+      @in_tight = false
+      @in_plain = false
+      @tagfilter = extensions.include?(:tagfilter)
+    end
+
+    def out(*args)
+      args.each do |arg|
+        if arg == :children
+          @node.each { |child| out(child) }
+        elsif arg.is_a?(Array)
+          arg.each { |x| render(x) }
+        elsif arg.is_a?(Node)
+          render(arg)
+        else
+          @stream.write(arg)
+        end
+      end
+    end
+
+    def render(node)
+      @node = node
+      if node.type == :document
+        document(node)
+        @stream.string
+      elsif @in_plain && node.type != :text && node.type != :softbreak
+        node.each { |child| render(child) }
+      else
+        begin
+          send(node.type, node)
+        rescue NoMethodError => e
+          @warnings.add("WARNING: #{node.type} not implemented.")
+          raise e
+        end
+      end
+    end
+
+    def document(_node)
+      out(:children)
+    end
+
+    def code_block(node)
+      code_block(node)
+    end
+
+    def reference_def(_node); end
+
+    def cr
+      return if @stream.string.empty? || @stream.string[-1] == "\n"
+
+      out("\n")
+    end
+
+    def blocksep
+      out("\n")
+    end
+
+    def containersep
+      cr unless @in_tight
+    end
+
+    def block
+      cr
+      yield
+      cr
+    end
+
+    def container(starter, ender)
+      out(starter)
+      yield
+      out(ender)
+    end
+
+    def plain
+      old_in_plain = @in_plain
+      @in_plain = true
+      yield
+      @in_plain = old_in_plain
+    end
+
+    private
+
+    def escape_href(str)
+      @node.html_escape_href(str)
+    end
+
+    def escape_html(str)
+      @node.html_escape_html(str)
+    end
+
+    def tagfilter(str)
+      if @tagfilter
+        str.gsub(
+          %r{
+            <
+            (
+            title|textarea|style|xmp|iframe|
+            noembed|noframes|script|plaintext
+            )
+            (?=\s|>|/>)
+          }xi,
+          '&lt;\1'
+        )
+      else
+        str
+      end
+    end
+
+    def source_position(node)
+      return '' unless flag_enabled?(SOURCE_POSITION)
+
+      s = node.source_position
+      " data-sourcepos=\"#{s[:start_line]}:#{s[:start_column]}-" \
+        "#{s[:end_line]}:#{s[:end_column]}\""
+    end
+
+    def flag_enabled?(flag)
+      (@flags & flag) != 0
+    end
+  end
+end