markly 0.7.0 → 0.8.0

data/ext/markly/scanners.h

@@ -15,6 +15,10 @@ bufsize_t _scan_autolink_uri(const unsigned char *p);
 bufsize_t _scan_autolink_email(const unsigned char *p);
 bufsize_t _scan_html_tag(const unsigned char *p);
 bufsize_t _scan_liberal_html_tag(const unsigned char *p);
+bufsize_t _scan_html_comment(const unsigned char *p);
+bufsize_t _scan_html_pi(const unsigned char *p);
+bufsize_t _scan_html_declaration(const unsigned char *p);
+bufsize_t _scan_html_cdata(const unsigned char *p);
 bufsize_t _scan_html_block_start(const unsigned char *p);
 bufsize_t _scan_html_block_start_7(const unsigned char *p);
 bufsize_t _scan_html_block_end_1(const unsigned char *p);
@@ -37,6 +41,10 @@ bufsize_t _scan_footnote_definition(const unsigned char *p);
 #define scan_autolink_email(c, n) _scan_at(&_scan_autolink_email, c, n)
 #define scan_html_tag(c, n) _scan_at(&_scan_html_tag, c, n)
 #define scan_liberal_html_tag(c, n) _scan_at(&_scan_liberal_html_tag, c, n)
+#define scan_html_comment(c, n) _scan_at(&_scan_html_comment, c, n)
+#define scan_html_pi(c, n) _scan_at(&_scan_html_pi, c, n)
+#define scan_html_declaration(c, n) _scan_at(&_scan_html_declaration, c, n)
+#define scan_html_cdata(c, n) _scan_at(&_scan_html_cdata, c, n)
 #define scan_html_block_start(c, n) _scan_at(&_scan_html_block_start, c, n)
 #define scan_html_block_start_7(c, n) _scan_at(&_scan_html_block_start_7, c, n)
 #define scan_html_block_end_1(c, n) _scan_at(&_scan_html_block_end_1, c, n)

data/ext/markly/scanners.re

@@ -54,16 +54,15 @@ bufsize_t _scan_at(bufsize_t (*scanner)(const unsigned char *), cmark_chunk *c,
   opentag = tagname attribute* spacechar* [/]? [>];
   closetag = [/] tagname spacechar* [>];
 
-  htmlcomment = "!---->" | ("!--" ([-]? [^\x00>-]) ([-]? [^\x00-])* "-->");
+  htmlcomment = "--" ([^\x00-]+ | "-" [^\x00-] | "--" [^\x00>])* "-->";
 
-  processinginstruction = "?" ([^?>\x00]+ | [?][^>\x00] | [>])* "?>";
+  processinginstruction = ([^?>\x00]+ | [?][^>\x00] | [>])+;
 
-  declaration = "!" [A-Z]+ spacechar+ [^>\x00]* ">";
+  declaration = [A-Z]+ spacechar+ [^>\x00]*;
 
-  cdata = "![CDATA[" ([^\]\x00]+ | "]" [^\]\x00] | "]]" [^>\x00])* "]]>";
+  cdata = "CDATA[" ([^\]\x00]+ | "]" [^\]\x00] | "]]" [^>\x00])*;
 
-  htmltag = opentag | closetag | htmlcomment | processinginstruction |
-            declaration | cdata;
+  htmltag = opentag | closetag;
 
   in_parens_nosp   = [(] (reg_char|escaped_char|[\\])* [)];
 
@@ -133,6 +132,46 @@ bufsize_t _scan_liberal_html_tag(const unsigned char *p)
 */
 }
 
+bufsize_t _scan_html_comment(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  htmlcomment { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}
+
+bufsize_t _scan_html_pi(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  processinginstruction { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}
+
+bufsize_t _scan_html_declaration(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  declaration { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}
+
+bufsize_t _scan_html_cdata(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  cdata { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}
+
 // Try to match an HTML block tag start line, returning
 // an integer code for the type of block (1-6, matching the spec).
 // #7 is handled by a separate function, below.
@@ -140,7 +179,7 @@ bufsize_t _scan_html_block_start(const unsigned char *p)
 {
   const unsigned char *marker = NULL;
 /*!re2c
-  [<] ('script'|'pre'|'style') (spacechar | [>]) { return 1; }
+  [<] ('script'|'pre'|'textarea'|'style') (spacechar | [>]) { return 1; }
   '<!--' { return 2; }
   '<?' { return 3; }
   '<!' [A-Z] { return 4; }
@@ -167,7 +206,7 @@ bufsize_t _scan_html_block_end_1(const unsigned char *p)
   const unsigned char *marker = NULL;
   const unsigned char *start = p;
 /*!re2c
-  [^\n\x00]* [<] [/] ('script'|'pre'|'style') [>] { return (bufsize_t)(p - start); }
+  [^\n\x00]* [<] [/] ('script'|'pre'|'textarea'|'style') [>] { return (bufsize_t)(p - start); }
   * { return 0; }
 */
 }

data/ext/markly/strikethrough.c

@@ -67,6 +67,7 @@ static delimiter *insert(cmark_syntax_extension *self, cmark_parser *parser,
   strikethrough->end_column = closer->inl_text->start_column + closer->inl_text->as.literal.len - 1;
   cmark_node_free(closer->inl_text);
 
+done:
   delim = closer;
   while (delim != NULL && delim != opener) {
     tmp_delim = delim->previous;
@@ -76,7 +77,6 @@ static delimiter *insert(cmark_syntax_extension *self, cmark_parser *parser,
 
   cmark_inline_parser_remove_delimiter(inline_parser, opener);
 
-done:
   return res;
 }
 

data/ext/markly/table.c

@@ -11,13 +11,21 @@
 #include "table.h"
 #include "cmark-gfm-core-extensions.h"
 
+// Custom node flag, initialized in `create_table_extension`.
+static cmark_node_internal_flags CMARK_NODE__TABLE_VISITED;
+
 cmark_node_type CMARK_NODE_TABLE, CMARK_NODE_TABLE_ROW,
     CMARK_NODE_TABLE_CELL;
 
+typedef struct {
+  cmark_strbuf *buf;
+  int start_offset, end_offset, internal_offset;
+} node_cell;
+
 typedef struct {
   uint16_t n_columns;
   int paragraph_offset;
-  cmark_llist *cells;
+  node_cell *cells;
 } table_row;
 
 typedef struct {
@@ -29,24 +37,24 @@ typedef struct {
   bool is_header;
 } node_table_row;
 
-typedef struct {
-  cmark_strbuf *buf;
-  int start_offset, end_offset, internal_offset;
-} node_cell;
-
-static void free_table_cell(cmark_mem *mem, void *data) {
-  node_cell *cell = (node_cell *)data;
+static void free_table_cell(cmark_mem *mem, node_cell *cell) {
   cmark_strbuf_free((cmark_strbuf *)cell->buf);
   mem->free(cell->buf);
-  mem->free(cell);
+}
+
+static void free_row_cells(cmark_mem *mem, table_row *row) {
+  while (row->n_columns > 0) {
+    free_table_cell(mem, &row->cells[--row->n_columns]);
+  }
+  mem->free(row->cells);
+  row->cells = NULL;
 }
 
 static void free_table_row(cmark_mem *mem, table_row *row) {
   if (!row)
     return;
 
-  cmark_llist_free_full(mem, row->cells, (cmark_free_func)free_table_cell);
-
+  free_row_cells(mem, row);
   mem->free(row);
 }
 
@@ -111,6 +119,24 @@ static cmark_strbuf *unescape_pipes(cmark_mem *mem, unsigned char *string, bufsi
   return res;
 }
 
+// Adds a new cell to the end of the row. A pointer to the new cell is returned
+// for the caller to initialize.
+static node_cell* append_row_cell(cmark_mem *mem, table_row *row) {
+  const uint32_t n_columns = row->n_columns + 1;
+  // realloc when n_columns is a power of 2
+  if ((n_columns & (n_columns-1)) == 0) {
+    // make sure we never wrap row->n_columns
+    // offset will != len and our exit will clean up as intended
+    if (n_columns > UINT16_MAX) {
+      return NULL;
+    }
+    // Use realloc to double the size of the buffer.
+    row->cells = (node_cell *)mem->realloc(row->cells, (2 * n_columns - 1) * sizeof(node_cell));
+  }
+  row->n_columns = (uint16_t)n_columns;
+  return &row->cells[n_columns-1];
+}
+
 static table_row *row_from_string(cmark_syntax_extension *self,
                                   cmark_parser *parser, unsigned char *string,
                                   int len) {
@@ -129,6 +155,7 @@ static table_row *row_from_string(cmark_syntax_extension *self,
   bufsize_t cell_matched = 1, pipe_matched = 1, offset;
   int expect_more_cells = 1;
   int row_end_offset = 0;
+  int int_overflow_abort = 0;
 
   row = (table_row *)parser->mem->calloc(1, sizeof(table_row));
   row->n_columns = 0;
@@ -151,18 +178,22 @@ static table_row *row_from_string(cmark_syntax_extension *self,
           cell_matched);
       cmark_strbuf_trim(cell_buf);
 
-      node_cell *cell = (node_cell *)parser->mem->calloc(1, sizeof(*cell));
+      node_cell *cell = append_row_cell(parser->mem, row);
+      if (!cell) {
+        int_overflow_abort = 1;
+        cmark_strbuf_free(cell_buf);
+        parser->mem->free(cell_buf);
+        break;
+      }
       cell->buf = cell_buf;
       cell->start_offset = offset;
       cell->end_offset = offset + cell_matched - 1;
+      cell->internal_offset = 0;
 
-      while (cell->start_offset > 0 && string[cell->start_offset - 1] != '|') {
+      while (cell->start_offset > row->paragraph_offset && string[cell->start_offset - 1] != '|') {
         --cell->start_offset;
         ++cell->internal_offset;
       }
-
-      row->n_columns += 1;
-      row->cells = cmark_llist_append(parser->mem, row->cells, cell);
     }
 
     offset += cell_matched + pipe_matched;
@@ -180,9 +211,7 @@ static table_row *row_from_string(cmark_syntax_extension *self,
       if (row_end_offset && offset != len) {
         row->paragraph_offset = offset;
 
-        cmark_llist_free_full(parser->mem, row->cells, (cmark_free_func)free_table_cell);
-        row->cells = NULL;
-        row->n_columns = 0;
+        free_row_cells(parser->mem, row);
 
         // Scan past the (optional) leading pipe.
         offset += scan_table_cell_end(string, len, offset);
@@ -194,7 +223,7 @@ static table_row *row_from_string(cmark_syntax_extension *self,
     }
   }
 
-  if (offset != len || row->n_columns == 0) {
+  if (offset != len || row->n_columns == 0 || int_overflow_abort) {
     free_table_row(parser->mem, row);
     row = NULL;
   }
@@ -233,6 +262,10 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
   const char *parent_string;
   uint16_t i;
 
+  if (parent_container->flags & CMARK_NODE__TABLE_VISITED) {
+    return parent_container;
+  }
+
   if (!scan_table_start(input, len, cmark_parser_get_first_nonspace(parser))) {
     return parent_container;
   }
@@ -241,6 +274,11 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
   marker_row = row_from_string(self, parser,
                                input + cmark_parser_get_first_nonspace(parser),
                                len - cmark_parser_get_first_nonspace(parser));
+  // assert may be optimized out, don't rely on it for security boundaries
+  if (!marker_row) {
+      return parent_container;
+  }
+  
   assert(marker_row);
 
   cmark_arena_push();
@@ -255,6 +293,7 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
     free_table_row(parser->mem, marker_row);
     free_table_row(parser->mem, header_row);
     cmark_arena_pop();
+    parent_container->flags |= CMARK_NODE__TABLE_VISITED;
     return parent_container;
   }
 
@@ -264,6 +303,12 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
         len - cmark_parser_get_first_nonspace(parser));
     header_row = row_from_string(self, parser, (unsigned char *)parent_string,
                                  (int)strlen(parent_string));
+    // row_from_string can return NULL, add additional check to ensure n_columns match
+    if (!marker_row || !header_row || header_row->n_columns != marker_row->n_columns) {
+        free_table_row(parser->mem, marker_row);
+        free_table_row(parser->mem, header_row);
+        return parent_container;
+    }
   }
 
   if (!cmark_node_set_type(parent_container, CMARK_NODE_TABLE)) {
@@ -281,11 +326,12 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
   parent_container->as.opaque = parser->mem->calloc(1, sizeof(node_table));
   set_n_table_columns(parent_container, header_row->n_columns);
 
+  // allocate alignments based on marker_row->n_columns
+  // since we populate the alignments array based on marker_row->cells
   uint8_t *alignments =
-      (uint8_t *)parser->mem->calloc(header_row->n_columns, sizeof(uint8_t));
-  cmark_llist *it = marker_row->cells;
-  for (i = 0; it; it = it->next, ++i) {
-    node_cell *node = (node_cell *)it->data;
+      (uint8_t *)parser->mem->calloc(marker_row->n_columns, sizeof(uint8_t));
+  for (i = 0; i < marker_row->n_columns; ++i) {
+    node_cell *node = &marker_row->cells[i];
     bool left = node->buf->ptr[0] == ':', right = node->buf->ptr[node->buf->size - 1] == ':';
 
     if (left && right)
@@ -308,10 +354,8 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
   ntr->is_header = true;
 
   {
-    cmark_llist *tmp;
-
-    for (tmp = header_row->cells; tmp; tmp = tmp->next) {
-      node_cell *cell = (node_cell *) tmp->data;
+    for (i = 0; i < header_row->n_columns; ++i) {
+      node_cell *cell = &header_row->cells[i];
       cmark_node *header_cell = cmark_parser_add_child(parser, table_header,
           CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset);
       header_cell->start_line = header_cell->end_line = parent_container->start_line;
@@ -351,12 +395,17 @@ static cmark_node *try_opening_table_row(cmark_syntax_extension *self,
   row = row_from_string(self, parser, input + cmark_parser_get_first_nonspace(parser),
       len - cmark_parser_get_first_nonspace(parser));
 
+  if (!row) {
+      // clean up the dangling node
+      cmark_node_free(table_row_block);
+      return NULL;
+  }
+
   {
-    cmark_llist *tmp;
     int i, table_columns = get_n_table_columns(parent_container);
 
-    for (tmp = row->cells, i = 0; tmp && i < table_columns; tmp = tmp->next, ++i) {
-      node_cell *cell = (node_cell *) tmp->data;
+    for (i = 0; i < row->n_columns && i < table_columns; ++i) {
+      node_cell *cell = &row->cells[i];
       cmark_node *node = cmark_parser_add_child(parser, table_row_block,
           CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset);
       node->internal_offset = cell->internal_offset;
@@ -759,6 +808,7 @@ static int escape(cmark_syntax_extension *self, cmark_node *node, int c) {
 cmark_syntax_extension *create_table_extension(void) {
   cmark_syntax_extension *self = cmark_syntax_extension_new("table");
 
+  cmark_register_node_flag(&CMARK_NODE__TABLE_VISITED);
   cmark_syntax_extension_set_match_block_func(self, matches);
   cmark_syntax_extension_set_open_block_func(self, try_opening_table_block);
   cmark_syntax_extension_set_get_type_string_func(self, get_type_string);

data/ext/markly/xml.c

@@ -11,6 +11,7 @@
 #include "syntax_extension.h"
 
 #define BUFFER_SIZE 100
+#define MAX_INDENT 40
 
 // Functions to convert cmark_nodes to XML strings.
 
@@ -26,7 +27,7 @@ struct render_state {
 
 static CMARK_INLINE void indent(struct render_state *state) {
   int i;
-  for (i = 0; i < state->indent; i++) {
+  for (i = 0; i < state->indent && i < MAX_INDENT; i++) {
     cmark_strbuf_putc(state->xml, ' ');
   }
 }

data/lib/markly/flags.rb

@@ -1,12 +1,22 @@
 # frozen_string_literal: true
 
+# Released under the MIT License.
+# Copyright, 2020-2023, by Samuel Williams.
+
 module Markly
+	# The default parsing system.
 	DEFAULT = 0
+	# Replace illegal sequences with the replacement character `U+FFFD`.
 	VALIDATE_UTF8 = 1 << 9
+	# Use smart punctuation (curly quotes, etc.).
 	SMART = 1 << 10
+	# Support liberal parsing of inline HTML tags.
 	LIBERAL_HTML_TAG = 1 << 12
+	# Parse footnotes.
 	FOOTNOTES = 1 << 13
+	# Support strikethrough using double tildes.
 	STRIKETHROUGH_DOUBLE_TILDE = 1 << 14
+	# Allow raw/custom HTML and unsafe links.
 	UNSAFE = 1 << 17
 	
 	PARSE_FLAGS = {
@@ -18,11 +28,17 @@ module Markly
 		unsafe: UNSAFE,
 	}
 	
+	# Include source position in rendered HTML.
 	SOURCE_POSITION = 1 << 1
+	# Treat `\n` as hardbreaks (by adding `<br/>`).
 	HARD_BREAKS = 1 << 2
+	# Translate `\n` in the source to a single whitespace.
 	NO_BREAKS = 1 << 4
+	# Use GitHub-style `<pre lang>` for fenced code blocks.
 	GITHUB_PRE_LANG = 1 << 11
+	# Use `style` insted of `align` for table cells.
 	TABLE_PREFER_STYLE_ATTRIBUTES = 1 << 15
+	# Include full info strings of code blocks in separate attribute.
 	FULL_INFO_STRING = 1 << 16
 	
 	RENDER_FLAGS = {

data/lib/markly/node/inspect.rb

@@ -1,59 +1,65 @@
 # frozen_string_literal: true
 
+# Released under the MIT License.
+# Copyright, 2017, by Goro Fuji.
+# Copyright, 2017-2019, by Garen Torikian.
+# Copyright, 2020, by Olle Jonsson.
+# Copyright, 2020-2023, by Samuel Williams.
+
 require 'pp'
 
 module Markly
-  class Node
-    module Inspect
-      PP_INDENT_SIZE = 2
-
-      def inspect
-        PP.pp(self, +'', Float::INFINITY)
-      end
-
-      # @param printer [PrettyPrint] pp
-      def pretty_print(printer)
-        printer.group(PP_INDENT_SIZE, "#<#{self.class}(#{type}):", '>') do
-          printer.breakable
-
-          attrs = %i[
-            source_position
-            string_content
-            url
-            title
-            header_level
-            list_type
-            list_start
-            list_tight
-            fence_info
-          ].map do |name|
-            begin
-              [name, __send__(name)]
-            rescue Error
-              nil
-            end
-          end.compact
-
-          printer.seplist(attrs) do |name, value|
-            printer.text "#{name}="
-            printer.pp value
-          end
-
-          if first_child
-            printer.breakable
-            printer.group(PP_INDENT_SIZE) do
-              children = []
-              node = first_child
-              while node
-                children << node
-                node = node.next
-              end
-              printer.text 'children='
-              printer.pp children
-            end
-          end
-        end
-      end
-    end
-  end
+	class Node
+		module Inspect
+			PP_INDENT_SIZE = 2
+
+			def inspect
+				PP.pp(self, +'', Float::INFINITY)
+			end
+
+			# @param printer [PrettyPrint] pp
+			def pretty_print(printer)
+				printer.group(PP_INDENT_SIZE, "#<#{self.class}(#{type}):", '>') do
+					printer.breakable
+
+					attrs = %i[
+						source_position
+						string_content
+						url
+						title
+						header_level
+						list_type
+						list_start
+						list_tight
+						fence_info
+					].map do |name|
+						begin
+							[name, __send__(name)]
+						rescue Error
+							nil
+						end
+					end.compact
+
+					printer.seplist(attrs) do |name, value|
+						printer.text "#{name}="
+						printer.pp value
+					end
+
+					if first_child
+						printer.breakable
+						printer.group(PP_INDENT_SIZE) do
+							children = []
+							node = first_child
+							while node
+								children << node
+								node = node.next
+							end
+							printer.text 'children='
+							printer.pp children
+						end
+					end
+				end
+			end
+		end
+	end
 end