commonmarker 0.23.6 → 0.23.8

data/ext/commonmarker/commonmarker.c

@@ -1304,4 +1304,5 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   rb_define_method(rb_cNode, "html_escape_html", rb_html_escape_html, 1);
 
   cmark_gfm_core_extensions_ensure_registered();
+  cmark_init_standard_node_flags();
 }

data/ext/commonmarker/html.c

@@ -63,10 +63,16 @@ static bool S_put_footnote_backref(cmark_html_renderer *renderer, cmark_strbuf *
   if (renderer->written_footnote_ix >= renderer->footnote_ix)
     return false;
   renderer->written_footnote_ix = renderer->footnote_ix;
+  char m[32];
+  snprintf(m, sizeof(m), "%d", renderer->written_footnote_ix);
 
   cmark_strbuf_puts(html, "<a href=\"#fnref-");
   houdini_escape_href(html, node->as.literal.data, node->as.literal.len);
-  cmark_strbuf_puts(html, "\" class=\"footnote-backref\" data-footnote-backref aria-label=\"Back to content\">↩</a>");
+  cmark_strbuf_puts(html, "\" class=\"footnote-backref\" data-footnote-backref data-footnote-backref-idx=\"");
+  cmark_strbuf_puts(html, m);
+  cmark_strbuf_puts(html, "\" aria-label=\"Back to reference ");
+  cmark_strbuf_puts(html, m);
+  cmark_strbuf_puts(html, "\">↩</a>");
 
   if (node->footnote.def_count > 1)
   {
@@ -78,7 +84,15 @@ static bool S_put_footnote_backref(cmark_html_renderer *renderer, cmark_strbuf *
       houdini_escape_href(html, node->as.literal.data, node->as.literal.len);
       cmark_strbuf_puts(html, "-");
       cmark_strbuf_puts(html, n);
-      cmark_strbuf_puts(html, "\" class=\"footnote-backref\" data-footnote-backref aria-label=\"Back to content\">↩<sup class=\"footnote-ref\">");
+      cmark_strbuf_puts(html, "\" class=\"footnote-backref\" data-footnote-backref data-footnote-backref-idx=\"");
+      cmark_strbuf_puts(html, m);
+      cmark_strbuf_puts(html, "-");
+      cmark_strbuf_puts(html, n);
+      cmark_strbuf_puts(html, "\" aria-label=\"Back to reference ");
+      cmark_strbuf_puts(html, m);
+      cmark_strbuf_puts(html, "-");
+      cmark_strbuf_puts(html, n);
+      cmark_strbuf_puts(html, "\">↩<sup class=\"footnote-ref\">");
       cmark_strbuf_puts(html, n);
       cmark_strbuf_puts(html, "</sup></a>");
     }

data/ext/commonmarker/inlines.c

@@ -35,7 +35,6 @@ static const char *RIGHTSINGLEQUOTE = "\xE2\x80\x99";
 
 typedef struct bracket {
   struct bracket *previous;
-  struct delimiter *previous_delimiter;
   cmark_node *inl_text;
   bufsize_t position;
   bool image;
@@ -45,9 +44,15 @@ typedef struct bracket {
   bool in_bracket_image1;
 } bracket;
 
+#define FLAG_SKIP_HTML_CDATA        (1u << 0)
+#define FLAG_SKIP_HTML_DECLARATION  (1u << 1)
+#define FLAG_SKIP_HTML_PI           (1u << 2)
+#define FLAG_SKIP_HTML_COMMENT      (1u << 3)
+
 typedef struct subject{
   cmark_mem *mem;
   cmark_chunk input;
+  unsigned flags;
   int line;
   bufsize_t pos;
   int block_offset;
@@ -57,6 +62,7 @@ typedef struct subject{
   bracket *last_bracket;
   bufsize_t backticks[MAXBACKTICKS + 1];
   bool scanned_for_backticks;
+  bool no_link_openers;
 } subject;
 
 // Extensions may populate this.
@@ -111,6 +117,24 @@ static cmark_node *make_str_with_entities(subject *subj,
   }
 }
 
+// Like cmark_node_append_child but without costly sanity checks.
+// Assumes that child was newly created.
+static void append_child(cmark_node *node, cmark_node *child) {
+  cmark_node *old_last_child = node->last_child;
+
+  child->next = NULL;
+  child->prev = old_last_child;
+  child->parent = node;
+  node->last_child = child;
+
+  if (old_last_child) {
+    old_last_child->next = child;
+  } else {
+    // Also set first_child if node previously had no children.
+    node->first_child = child;
+  }
+}
+
 // Duplicate a chunk by creating a copy of the buffer not by reusing the
 // buffer like cmark_chunk_dup does.
 static cmark_chunk chunk_clone(cmark_mem *mem, cmark_chunk *src) {
@@ -154,7 +178,7 @@ static CMARK_INLINE cmark_node *make_autolink(subject *subj,
   link->start_line = link->end_line = subj->line;
   link->start_column = start_column + 1;
   link->end_column = end_column + 1;
-  cmark_node_append_child(link, make_str_with_entities(subj, start_column + 1, end_column - 1, &url));
+  append_child(link, make_str_with_entities(subj, start_column + 1, end_column - 1, &url));
   return link;
 }
 
@@ -163,6 +187,7 @@ static void subject_from_buf(cmark_mem *mem, int line_number, int block_offset,
   int i;
   e->mem = mem;
   e->input = *chunk;
+  e->flags = 0;
   e->line = line_number;
   e->pos = 0;
   e->block_offset = block_offset;
@@ -174,6 +199,7 @@ static void subject_from_buf(cmark_mem *mem, int line_number, int block_offset,
     e->backticks[i] = 0;
   }
   e->scanned_for_backticks = false;
+  e->no_link_openers = true;
 }
 
 static CMARK_INLINE int isbacktick(int c) { return (c == '`'); }
@@ -505,6 +531,7 @@ static void push_delimiter(subject *subj, unsigned char c, bool can_open,
   delim->can_open = can_open;
   delim->can_close = can_close;
   delim->inl_text = inl_text;
+  delim->position = subj->pos;
   delim->length = inl_text->as.literal.len;
   delim->previous = subj->last_delim;
   delim->next = NULL;
@@ -525,7 +552,6 @@ static void push_bracket(subject *subj, bool image, cmark_node *inl_text) {
   b->active = true;
   b->inl_text = inl_text;
   b->previous = subj->last_bracket;
-  b->previous_delimiter = subj->last_delim;
   b->position = subj->pos;
   b->bracket_after = false;
   if (image) {
@@ -534,6 +560,9 @@ static void push_bracket(subject *subj, bool image, cmark_node *inl_text) {
     b->in_bracket_image0 = true;
   }
   subj->last_bracket = b;
+  if (!image) {
+    subj->no_link_openers = false;
+  }
 }
 
 // Assumes the subject has a c at the current position.
@@ -640,12 +669,13 @@ static cmark_syntax_extension *get_extension_for_special_char(cmark_parser *pars
   return NULL;
 }
 
-static void process_emphasis(cmark_parser *parser, subject *subj, delimiter *stack_bottom) {
-  delimiter *closer = subj->last_delim;
+static void process_emphasis(cmark_parser *parser, subject *subj, bufsize_t stack_bottom) {
+  delimiter *candidate;
+  delimiter *closer = NULL;
   delimiter *opener;
   delimiter *old_closer;
   bool opener_found;
-  delimiter *openers_bottom[3][128];
+  bufsize_t openers_bottom[3][128];
   int i;
 
   // initialize openers_bottom:
@@ -658,8 +688,10 @@ static void process_emphasis(cmark_parser *parser, subject *subj, delimiter *sta
   }
 
   // move back to first relevant delim.
-  while (closer != NULL && closer->previous != stack_bottom) {
-    closer = closer->previous;
+  candidate = subj->last_delim;
+  while (candidate != NULL && candidate->position >= stack_bottom) {
+    closer = candidate;
+    candidate = candidate->previous;
   }
 
   // now move forward, looking for closers, and handling each
@@ -669,8 +701,8 @@ static void process_emphasis(cmark_parser *parser, subject *subj, delimiter *sta
       // Now look backwards for first matching opener:
       opener = closer->previous;
       opener_found = false;
-      while (opener != NULL && opener != stack_bottom &&
-             opener != openers_bottom[closer->length % 3][closer->delim_char]) {
+      while (opener != NULL && opener->position >= stack_bottom &&
+             opener->position >= openers_bottom[closer->length % 3][closer->delim_char]) {
         if (opener->can_open && opener->delim_char == closer->delim_char) {
           // interior closer of size 2 can't match opener of size 1
           // or of size 1 can't match 2
@@ -696,27 +728,29 @@ static void process_emphasis(cmark_parser *parser, subject *subj, delimiter *sta
         } else {
           closer = closer->next;
         }
-      } else if (closer->delim_char == '\'') {
+      } else if (closer->delim_char == '\'' || closer->delim_char == '"') {
         cmark_chunk_free(subj->mem, &closer->inl_text->as.literal);
-        closer->inl_text->as.literal = cmark_chunk_literal(RIGHTSINGLEQUOTE);
-        if (opener_found) {
-          cmark_chunk_free(subj->mem, &opener->inl_text->as.literal);
-          opener->inl_text->as.literal = cmark_chunk_literal(LEFTSINGLEQUOTE);
+        if (closer->delim_char == '\'') {
+          closer->inl_text->as.literal = cmark_chunk_literal(RIGHTSINGLEQUOTE);
+        } else {
+          closer->inl_text->as.literal = cmark_chunk_literal(RIGHTDOUBLEQUOTE);
         }
         closer = closer->next;
-      } else if (closer->delim_char == '"') {
-        cmark_chunk_free(subj->mem, &closer->inl_text->as.literal);
-        closer->inl_text->as.literal = cmark_chunk_literal(RIGHTDOUBLEQUOTE);
         if (opener_found) {
           cmark_chunk_free(subj->mem, &opener->inl_text->as.literal);
-          opener->inl_text->as.literal = cmark_chunk_literal(LEFTDOUBLEQUOTE);
+          if (old_closer->delim_char == '\'') {
+            opener->inl_text->as.literal = cmark_chunk_literal(LEFTSINGLEQUOTE);
+          } else {
+            opener->inl_text->as.literal = cmark_chunk_literal(LEFTDOUBLEQUOTE);
+          }
+          remove_delimiter(subj, opener);
+          remove_delimiter(subj, old_closer);
         }
-        closer = closer->next;
       }
       if (!opener_found) {
         // set lower bound for future searches for openers
         openers_bottom[old_closer->length % 3][old_closer->delim_char] =
-		old_closer->previous;
+		old_closer->position;
         if (!old_closer->can_open) {
           // we can remove a closer that can't be an
           // opener, once we've seen there's no
@@ -729,7 +763,8 @@ static void process_emphasis(cmark_parser *parser, subject *subj, delimiter *sta
     }
   }
   // free all delimiters in list until stack_bottom:
-  while (subj->last_delim != NULL && subj->last_delim != stack_bottom) {
+  while (subj->last_delim != NULL &&
+         subj->last_delim->position >= stack_bottom) {
     remove_delimiter(subj, subj->last_delim);
   }
 }
@@ -768,7 +803,8 @@ static delimiter *S_insert_emph(subject *subj, delimiter *opener,
   tmp = opener_inl->next;
   while (tmp && tmp != closer_inl) {
     tmpnext = tmp->next;
-    cmark_node_append_child(emph, tmp);
+    cmark_node_unlink(tmp);
+    append_child(emph, tmp);
     tmp = tmpnext;
   }
   cmark_node_insert_after(opener_inl, emph);
@@ -899,7 +935,63 @@ static cmark_node *handle_pointy_brace(subject *subj, int options) {
   }
 
   // finally, try to match an html tag
-  matchlen = scan_html_tag(&subj->input, subj->pos);
+  if (subj->pos + 2 <= subj->input.len) {
+    int c = subj->input.data[subj->pos];
+    if (c == '!' && (subj->flags & FLAG_SKIP_HTML_COMMENT) == 0) {
+      c = subj->input.data[subj->pos+1];
+      if (c == '-' && subj->input.data[subj->pos+2] == '-') {
+        if (subj->input.data[subj->pos+3] == '>') {
+          matchlen = 4;
+        } else if (subj->input.data[subj->pos+3] == '-' &&
+                   subj->input.data[subj->pos+4] == '>') {
+          matchlen = 5;
+        } else {
+          matchlen = scan_html_comment(&subj->input, subj->pos + 1);
+          if (matchlen > 0) {
+            matchlen += 1; // prefix "<"
+          } else { // no match through end of input: set a flag so
+                   // we don't reparse looking for -->:
+            subj->flags |= FLAG_SKIP_HTML_COMMENT;
+          }
+        }
+      } else if (c == '[') {
+        if ((subj->flags & FLAG_SKIP_HTML_CDATA) == 0) {
+          matchlen = scan_html_cdata(&subj->input, subj->pos + 2);
+          if (matchlen > 0) {
+            // The regex doesn't require the final "]]>". But if we're not at
+            // the end of input, it must come after the match. Otherwise,
+            // disable subsequent scans to avoid quadratic behavior.
+            matchlen += 5; // prefix "![", suffix "]]>"
+            if (subj->pos + matchlen > subj->input.len) {
+              subj->flags |= FLAG_SKIP_HTML_CDATA;
+              matchlen = 0;
+            }
+          }
+        }
+      } else if ((subj->flags & FLAG_SKIP_HTML_DECLARATION) == 0) {
+        matchlen = scan_html_declaration(&subj->input, subj->pos + 1);
+        if (matchlen > 0) {
+          matchlen += 2; // prefix "!", suffix ">"
+          if (subj->pos + matchlen > subj->input.len) {
+            subj->flags |= FLAG_SKIP_HTML_DECLARATION;
+            matchlen = 0;
+          }
+        }
+      }
+    } else if (c == '?') {
+      if ((subj->flags & FLAG_SKIP_HTML_PI) == 0) {
+        // Note that we allow an empty match.
+        matchlen = scan_html_pi(&subj->input, subj->pos + 1);
+        matchlen += 3; // prefix "?", suffix "?>"
+        if (subj->pos + matchlen > subj->input.len) {
+          subj->flags |= FLAG_SKIP_HTML_PI;
+          matchlen = 0;
+        }
+      }
+    } else {
+      matchlen = scan_html_tag(&subj->input, subj->pos);
+    }
+  }
   if (matchlen > 0) {
     contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1);
     subj->pos += matchlen;
@@ -1065,16 +1157,16 @@ static cmark_node *handle_close_bracket(cmark_parser *parser, subject *subj) {
     return make_str(subj, subj->pos - 1, subj->pos - 1, cmark_chunk_literal("]"));
   }
 
-  if (!opener->active) {
+  // If we got here, we matched a potential link/image text.
+  // Now we check to see if it's a link/image.
+  is_image = opener->image;
+
+  if (!is_image && subj->no_link_openers) {
     // take delimiter off stack
     pop_bracket(subj);
     return make_str(subj, subj->pos - 1, subj->pos - 1, cmark_chunk_literal("]"));
   }
 
-  // If we got here, we matched a potential link/image text.
-  // Now we check to see if it's a link/image.
-  is_image = opener->image;
-
   after_link_text_pos = subj->pos;
 
   // First, look for an inline link.
@@ -1193,7 +1285,7 @@ noMatch:
       // being replacing the opening '[' text node with a `^footnote-ref]` node.
       cmark_node_insert_before(opener->inl_text, fnref);
 
-      process_emphasis(parser, subj, opener->previous_delimiter);
+      process_emphasis(parser, subj, opener->position);
       // sometimes, the footnote reference text gets parsed into multiple nodes
       // i.e. '[^example]' parsed into '[', '^exam', 'ple]'.
       // this happens for ex with the autolink extension. when the autolinker
@@ -1238,42 +1330,22 @@ match:
   tmp = opener->inl_text->next;
   while (tmp) {
     tmpnext = tmp->next;
-    cmark_node_append_child(inl, tmp);
+    cmark_node_unlink(tmp);
+    append_child(inl, tmp);
     tmp = tmpnext;
   }
 
   // Free the bracket [:
   cmark_node_free(opener->inl_text);
 
-  process_emphasis(parser, subj, opener->previous_delimiter);
+  process_emphasis(parser, subj, opener->position);
   pop_bracket(subj);
 
-  // Now, if we have a link, we also want to deactivate earlier link
-  // delimiters. (This code can be removed if we decide to allow links
+  // Now, if we have a link, we also want to deactivate links until
+  // we get a new opener. (This code can be removed if we decide to allow links
   // inside links.)
   if (!is_image) {
-    opener = subj->last_bracket;
-    while (opener != NULL) {
-      if (!opener->image) {
-        if (!opener->active) {
-          break;
-        } else {
-          opener->active = false;
-        }
-      }
-      opener = opener->previous;
-    }
-    bool in_bracket_image1 = false;
-    if (opener) {
-      in_bracket_image1 = opener->in_bracket_image1;
-    }
-    bracket *opener2 = subj->last_bracket;
-    while (opener2 != opener) {
-      if (opener2->image) {
-        opener2->in_bracket_image1 = in_bracket_image1;
-      }
-      opener2 = opener2->previous;
-    }
+    subj->no_link_openers = true;
   }
 
   return NULL;
@@ -1451,7 +1523,7 @@ static int parse_inline(cmark_parser *parser, subject *subj, cmark_node *parent,
     new_inl = make_str(subj, startpos, endpos - 1, contents);
   }
   if (new_inl != NULL) {
-    cmark_node_append_child(parent, new_inl);
+    append_child(parent, new_inl);
   }
 
   return 1;
@@ -1470,7 +1542,7 @@ void cmark_parse_inlines(cmark_parser *parser,
   while (!is_eof(&subj) && parse_inline(parser, &subj, parent, options))
     ;
 
-  process_emphasis(parser, &subj, NULL);
+  process_emphasis(parser, &subj, 0);
   // free bracket and delim stack
   while (subj.last_delim) {
     remove_delimiter(&subj, subj.last_delim);

data/ext/commonmarker/map.c

@@ -51,7 +51,7 @@ refsearch(const void *label, const void *p2) {
 }
 
 static void sort_map(cmark_map *map) {
-  unsigned int i = 0, last = 0, size = map->size;
+  size_t i = 0, last = 0, size = map->size;
   cmark_map_entry *r = map->refs, **sorted = NULL;
 
   sorted = (cmark_map_entry **)map->mem->calloc(size, sizeof(cmark_map_entry *));
@@ -73,6 +73,7 @@ static void sort_map(cmark_map *map) {
 
 cmark_map_entry *cmark_map_lookup(cmark_map *map, cmark_chunk *label) {
   cmark_map_entry **ref = NULL;
+  cmark_map_entry *r = NULL;
   unsigned char *norm;
 
   if (label->len < 1 || label->len > MAX_LINK_LABEL_LENGTH)
@@ -91,10 +92,15 @@ cmark_map_entry *cmark_map_lookup(cmark_map *map, cmark_chunk *label) {
   ref = (cmark_map_entry **)bsearch(norm, map->sorted, map->size, sizeof(cmark_map_entry *), refsearch);
   map->mem->free(norm);
 
-  if (!ref)
-    return NULL;
+  if (ref != NULL) {
+    r = ref[0];
+    /* Check for expansion limit */
+    if (r->size > map->max_ref_size - map->ref_size)
+      return NULL;
+    map->ref_size += r->size;
+  }
 
-  return ref[0];
+  return r;
 }
 
 void cmark_map_free(cmark_map *map) {
@@ -118,5 +124,6 @@ cmark_map *cmark_map_new(cmark_mem *mem, cmark_map_free_f free) {
   cmark_map *map = (cmark_map *)mem->calloc(1, sizeof(cmark_map));
   map->mem = mem;
   map->free = free;
+  map->max_ref_size = UINT_MAX;
   return map;
 }

data/ext/commonmarker/map.h

@@ -10,7 +10,8 @@ extern "C" {
 struct cmark_map_entry {
   struct cmark_map_entry *next;
   unsigned char *label;
-  unsigned int age;
+  size_t age;
+  size_t size;
 };
 
 typedef struct cmark_map_entry cmark_map_entry;
@@ -23,7 +24,9 @@ struct cmark_map {
   cmark_mem *mem;
   cmark_map_entry *refs;
   cmark_map_entry **sorted;
-  unsigned int size;
+  size_t size;
+  size_t ref_size;
+  size_t max_ref_size;
   cmark_map_free_f free;
 };
 

data/ext/commonmarker/node.c

@@ -9,6 +9,28 @@ static void S_node_unlink(cmark_node *node);
 
 #define NODE_MEM(node) cmark_node_mem(node)
 
+void cmark_register_node_flag(cmark_node_internal_flags *flags) {
+  static cmark_node_internal_flags nextflag = CMARK_NODE__REGISTER_FIRST;
+
+  // flags should be a pointer to a global variable and this function
+  // should only be called once to initialize its value.
+  if (*flags) {
+    fprintf(stderr, "flag initialization error in cmark_register_node_flag\n");
+    abort();
+  }
+
+  // Check that we haven't run out of bits.
+  if (nextflag == 0) {
+    fprintf(stderr, "too many flags in cmark_register_node_flag\n");
+    abort();
+  }
+
+  *flags = nextflag;
+  nextflag <<= 1;
+}
+
+void cmark_init_standard_node_flags() {}
+
 bool cmark_node_can_contain_type(cmark_node *node, cmark_node_type child_type) {
   if (child_type == CMARK_NODE_DOCUMENT) {
       return false;
@@ -301,6 +323,14 @@ cmark_node *cmark_node_last_child(cmark_node *node) {
   }
 }
 
+cmark_node *cmark_node_parent_footnote_def(cmark_node *node) {
+  if (node == NULL) {
+    return NULL;
+  } else {
+    return node->parent_footnote_def;
+  }
+}
+
 void *cmark_node_get_user_data(cmark_node *node) {
   if (node == NULL) {
     return NULL;

data/ext/commonmarker/node.h

@@ -52,8 +52,14 @@ enum cmark_node__internal_flags {
   CMARK_NODE__OPEN = (1 << 0),
   CMARK_NODE__LAST_LINE_BLANK = (1 << 1),
   CMARK_NODE__LAST_LINE_CHECKED = (1 << 2),
+
+  // Extensions can register custom flags by calling `cmark_register_node_flag`.
+  // This is the starting value for the custom flags.
+  CMARK_NODE__REGISTER_FIRST = (1 << 3),
 };
 
+typedef uint16_t cmark_node_internal_flags;
+
 struct cmark_node {
   cmark_strbuf content;
 
@@ -72,7 +78,7 @@ struct cmark_node {
   int end_column;
   int internal_offset;
   uint16_t type;
-  uint16_t flags;
+  cmark_node_internal_flags flags;
 
   cmark_syntax_extension *extension;
 
@@ -95,6 +101,26 @@ struct cmark_node {
   } as;
 };
 
+/**
+ * Syntax extensions can use this function to register a custom node
+ * flag. The flags are stored in the `flags` field of the `cmark_node`
+ * struct. The `flags` parameter should be the address of a global variable
+ * which will store the flag value.
+ */
+CMARK_GFM_EXPORT
+void cmark_register_node_flag(cmark_node_internal_flags *flags);
+
+/**
+ * DEPRECATED.
+ *
+ * This function was added in cmark-gfm version 0.29.0.gfm.7, and was
+ * required to be called at program start time, which caused
+ * backwards-compatibility issues in applications that use cmark-gfm as a
+ * library. It is now a no-op.
+ */
+CMARK_GFM_EXPORT
+void cmark_init_standard_node_flags();
+
 static CMARK_INLINE cmark_mem *cmark_node_mem(cmark_node *node) {
   return node->content.mem;
 }

data/ext/commonmarker/parser.h

@@ -46,6 +46,7 @@ struct cmark_parser {
   /* Options set by the user, see the Options section in cmark.h */
   int options;
   bool last_buffer_ended_with_cr;
+  size_t total_size;
   cmark_llist *syntax_extensions;
   cmark_llist *inline_syntax_extensions;
   cmark_ispunct_func backslash_ispunct;

data/ext/commonmarker/references.c

@@ -32,6 +32,7 @@ void cmark_reference_create(cmark_map *map, cmark_chunk *label,
   ref->title = cmark_clean_title(map->mem, title);
   ref->entry.age = map->size;
   ref->entry.next = map->refs;
+  ref->entry.size = ref->url.len + ref->title.len;
 
   map->refs = (cmark_map_entry *)ref;
   map->size++;