commonmarker 0.17.5 → 0.17.6

data/ext/commonmarker/cmark/src/references.h

@@ -1,38 +1,23 @@
 #ifndef CMARK_REFERENCES_H
 #define CMARK_REFERENCES_H
 
-#include "memory.h"
-#include "chunk.h"
+#include "map.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct cmark_reference {
-  struct cmark_reference *next;
-  unsigned char *label;
+  cmark_map_entry entry;
   cmark_chunk url;
   cmark_chunk title;
-  unsigned int age;
 };
 
 typedef struct cmark_reference cmark_reference;
 
-struct cmark_reference_map {
-  cmark_mem *mem;
-  cmark_reference *refs;
-  cmark_reference **sorted;
-  unsigned int size;
-};
-
-typedef struct cmark_reference_map cmark_reference_map;
-
-cmark_reference_map *cmark_reference_map_new(cmark_mem *mem);
-void cmark_reference_map_free(cmark_reference_map *map);
-cmark_reference *cmark_reference_lookup(cmark_reference_map *map,
-                                        cmark_chunk *label);
-extern void cmark_reference_create(cmark_reference_map *map, cmark_chunk *label,
-                                   cmark_chunk *url, cmark_chunk *title);
+void cmark_reference_create(cmark_map *map, cmark_chunk *label,
+                            cmark_chunk *url, cmark_chunk *title);
+cmark_map *cmark_reference_map_new(cmark_mem *mem);
 
 #ifdef __cplusplus
 }

data/ext/commonmarker/cmark/src/render.c

@@ -174,7 +174,8 @@ char *cmark_render(cmark_mem *mem, cmark_node *root, int options, int width,
 
   cmark_renderer renderer = {mem,   &buf, &pref, 0,           width,
                              0,     0,    true,  true,        false,
-                             false, outc, S_cr,  S_blankline, S_out};
+                             false, outc, S_cr,  S_blankline, S_out,
+                             0};
 
   while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
     cur = cmark_iter_get_node(iter);

data/ext/commonmarker/cmark/src/render.h

@@ -28,6 +28,7 @@ struct cmark_renderer {
   void (*cr)(struct cmark_renderer *);
   void (*blankline)(struct cmark_renderer *);
   void (*out)(struct cmark_renderer *, cmark_node *, const char *, bool, cmark_escaping);
+  unsigned int footnote_ix;
 };
 
 typedef struct cmark_renderer cmark_renderer;
@@ -36,6 +37,8 @@ struct cmark_html_renderer {
   cmark_strbuf *html;
   cmark_node *plain;
   cmark_llist *filter_extensions;
+  unsigned int footnote_ix;
+  unsigned int written_footnote_ix;
   void *opaque;
 };
 

data/ext/commonmarker/cmark/src/scanners.c

@@ -7887,45 +7887,35 @@ bufsize_t _scan_html_tag(const unsigned char *p) {
     unsigned char yych;
     static const unsigned char yybm[] = {
         /* table 1 .. 8: 0 */
-        0,   239, 239, 239, 239, 239, 239, 239, 239, 238, 238, 238, 238, 238,
+        0, 239, 239, 239, 239, 239, 239, 239, 239, 238, 238, 238, 238, 238, 239,
         239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239,
-        239, 239, 239, 239, 238, 239, 234, 239, 239, 239, 239, 236, 239, 239,
-        239, 239, 239, 207, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239,
-        239, 239, 239, 239, 238, 238, 174, 231, 239, 255, 255, 255, 255, 255,
+        239, 239, 239, 238, 239, 234, 239, 239, 239, 239, 236, 239, 239, 239,
+        239, 239, 207, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239,
+        239, 239, 239, 238, 238, 174, 231, 239, 255, 255, 255, 255, 255, 255,
         255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
-        255, 255, 255, 255, 255, 255, 255, 239, 239, 111, 239, 239, 238, 239,
+        255, 255, 255, 255, 255, 255, 239, 239, 111, 239, 239, 238, 239, 239,
         239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239,
         239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, 239,
-        239, 239, 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,
+        239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         /* table 9 .. 11: 256 */
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   64,  64,  64,  64,  64,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   64,  0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   160, 128, 0,   160, 160, 160, 160, 160, 160, 160, 160,
-        160, 160, 128, 0,   0,   0,   0,   0,   0,   160, 160, 160, 160, 160,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 64, 64, 64, 64, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        160, 128, 0, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 128, 0,
+        0, 0, 0, 0, 0, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
-        160, 160, 160, 160, 160, 160, 160, 0,   0,   0,   0,   128, 0,   160,
+        160, 0, 0, 0, 0, 128, 0, 160, 160, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
-        160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
-        0,   0,   0,   0,
+        160, 160, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     };
     yych = *p;
     if (yych <= '>') {
@@ -18057,3 +18047,149 @@ bufsize_t _scan_dangerous_url(const unsigned char *p) {
     goto yy1188;
   }
 }
+
+// Scans a footnote definition opening.
+bufsize_t _scan_footnote_definition(const unsigned char *p) {
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+
+  {
+    unsigned char yych;
+    static const unsigned char yybm[] = {
+        0,   64, 64, 64, 64, 64, 64, 64, 64, 128, 0,  64, 64, 0,  64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        128, 64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 0,  64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        64,  64, 64, 64, 64, 64, 64, 64, 64, 64,  64, 64, 64, 64, 64, 64,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+        0,   0,  0,  0,  0,  0,  0,  0,  0,  0,   0,  0,  0,  0,  0,  0,
+    };
+    yych = *p;
+    if (yych == '[')
+      goto yy1235;
+    ++p;
+  yy1234 : { return 0; }
+  yy1235:
+    yych = *(marker = ++p);
+    if (yych != '^')
+      goto yy1234;
+    yych = *++p;
+    if (yych != ']')
+      goto yy1239;
+  yy1237:
+    p = marker;
+    goto yy1234;
+  yy1238:
+    ++p;
+    yych = *p;
+  yy1239:
+    if (yybm[0 + yych] & 64) {
+      goto yy1238;
+    }
+    if (yych <= 0xEC) {
+      if (yych <= 0xC1) {
+        if (yych <= ' ')
+          goto yy1237;
+        if (yych <= ']')
+          goto yy1247;
+        goto yy1237;
+      } else {
+        if (yych <= 0xDF)
+          goto yy1240;
+        if (yych <= 0xE0)
+          goto yy1241;
+        goto yy1242;
+      }
+    } else {
+      if (yych <= 0xF0) {
+        if (yych <= 0xED)
+          goto yy1246;
+        if (yych <= 0xEF)
+          goto yy1242;
+        goto yy1243;
+      } else {
+        if (yych <= 0xF3)
+          goto yy1244;
+        if (yych <= 0xF4)
+          goto yy1245;
+        goto yy1237;
+      }
+    }
+  yy1240:
+    ++p;
+    yych = *p;
+    if (yych <= 0x7F)
+      goto yy1237;
+    if (yych <= 0xBF)
+      goto yy1238;
+    goto yy1237;
+  yy1241:
+    ++p;
+    yych = *p;
+    if (yych <= 0x9F)
+      goto yy1237;
+    if (yych <= 0xBF)
+      goto yy1240;
+    goto yy1237;
+  yy1242:
+    ++p;
+    yych = *p;
+    if (yych <= 0x7F)
+      goto yy1237;
+    if (yych <= 0xBF)
+      goto yy1240;
+    goto yy1237;
+  yy1243:
+    ++p;
+    yych = *p;
+    if (yych <= 0x8F)
+      goto yy1237;
+    if (yych <= 0xBF)
+      goto yy1242;
+    goto yy1237;
+  yy1244:
+    ++p;
+    yych = *p;
+    if (yych <= 0x7F)
+      goto yy1237;
+    if (yych <= 0xBF)
+      goto yy1242;
+    goto yy1237;
+  yy1245:
+    ++p;
+    yych = *p;
+    if (yych <= 0x7F)
+      goto yy1237;
+    if (yych <= 0x8F)
+      goto yy1242;
+    goto yy1237;
+  yy1246:
+    ++p;
+    yych = *p;
+    if (yych <= 0x7F)
+      goto yy1237;
+    if (yych <= 0x9F)
+      goto yy1240;
+    goto yy1237;
+  yy1247:
+    yych = *++p;
+    if (yych != ':')
+      goto yy1237;
+  yy1248:
+    ++p;
+    yych = *p;
+    if (yybm[0 + yych] & 128) {
+      goto yy1248;
+    }
+    { return (bufsize_t)(p - start); }
+  }
+}

data/ext/commonmarker/cmark/src/scanners.h

@@ -31,6 +31,7 @@ bufsize_t _scan_open_code_fence(const unsigned char *p);
 bufsize_t _scan_close_code_fence(const unsigned char *p);
 bufsize_t _scan_entity(const unsigned char *p);
 bufsize_t _scan_dangerous_url(const unsigned char *p);
+bufsize_t _scan_footnote_definition(const unsigned char *p);
 
 #define scan_scheme(c, n) _scan_at(&_scan_scheme, c, n)
 #define scan_autolink_uri(c, n) _scan_at(&_scan_autolink_uri, c, n)
@@ -54,6 +55,7 @@ bufsize_t _scan_dangerous_url(const unsigned char *p);
 #define scan_close_code_fence(c, n) _scan_at(&_scan_close_code_fence, c, n)
 #define scan_entity(c, n) _scan_at(&_scan_entity, c, n)
 #define scan_dangerous_url(c, n) _scan_at(&_scan_dangerous_url, c, n)
+#define scan_footnote_definition(c, n) _scan_at(&_scan_footnote_definition, c, n)
 
 #ifdef __cplusplus
 }

data/ext/commonmarker/cmark/src/scanners.re

@@ -329,3 +329,13 @@ bufsize_t _scan_dangerous_url(const unsigned char *p)
 */
 }
 
+// Scans a footnote definition opening.
+bufsize_t _scan_footnote_definition(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  '[^' ([^\] \r\n\x00\t]+) ']:' [ \t]* { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}

data/ext/commonmarker/cmark/src/syntax_extension.c

@@ -1,4 +1,5 @@
 #include <stdlib.h>
+#include <assert.h>
 
 #include "cmark.h"
 #include "syntax_extension.h"
@@ -37,8 +38,8 @@ cmark_node_type cmark_syntax_extension_add_node(int is_inline) {
 }
 
 void cmark_syntax_extension_set_emphasis(cmark_syntax_extension *extension,
-                                         bool emphasis) {
-  extension->emphasis = emphasis;
+                                         int emphasis) {
+  extension->emphasis = emphasis == 1;
 }
 
 void cmark_syntax_extension_set_open_block_func(cmark_syntax_extension *extension,

data/ext/commonmarker/cmark/src/syntax_extension.h

@@ -3,6 +3,7 @@
 
 #include "cmark.h"
 #include "cmark_extension_api.h"
+#include "config.h"
 
 struct cmark_syntax_extension {
   cmark_match_block_func          last_block_matches;

data/ext/commonmarker/cmark/test/CMakeLists.txt

@@ -64,14 +64,19 @@ IF (PYTHONINTERP_FOUND)
     )
 
   add_test(extensions_executable
-    ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/spec_tests.py" "--no-normalize" "--spec" "${CMAKE_CURRENT_SOURCE_DIR}/extensions.txt" "--program" "${CMAKE_CURRENT_BINARY_DIR}/../src/cmark-gfm" "--extensions" "table strikethrough autolink tagfilter"
+    ${PYTHON_EXECUTABLE}
+    "${CMAKE_CURRENT_SOURCE_DIR}/spec_tests.py"
+    "--no-normalize"
+    "--spec" "${CMAKE_CURRENT_SOURCE_DIR}/extensions.txt"
+    "--program" "${CMAKE_CURRENT_BINARY_DIR}/../src/cmark-gfm --footnotes"
+    "--extensions" "table strikethrough autolink tagfilter"
     )
 
   add_test(roundtrip_extensions_executable
     ${PYTHON_EXECUTABLE}
     "${CMAKE_CURRENT_SOURCE_DIR}/roundtrip_tests.py"
     "--spec" "${CMAKE_CURRENT_SOURCE_DIR}/extensions.txt"
-    "--program" "${CMAKE_CURRENT_BINARY_DIR}/../src/cmark-gfm"
+    "--program" "${CMAKE_CURRENT_BINARY_DIR}/../src/cmark-gfm --footnotes"
     "--extensions" "table strikethrough autolink tagfilter"
     )
 

data/ext/commonmarker/cmark/test/extensions.txt

@@ -579,6 +579,65 @@ Even with {"x":"y"} or 1 > 2 or whatever. Even **markdown**.
 <!--thistoo-->
 ````````````````````````````````
 
+## Footnotes
+
+```````````````````````````````` example
+This is some text.[^1]. Other text.[^footnote].
+
+Here's a thing[^other-note].
+
+And another thing[^codeblock-note].
+
+This doesn't have a referent[^nope].
+
+
+[^other-note]:       no code block here (spaces are stripped away)
+
+[^codeblock-note]:
+        this is now a code block (8 spaces indentation)
+
+[^1]: Some *bolded* footnote definition.
+
+Hi!
+
+[^footnote]:
+    > Blockquotes can be in a footnote.
+
+        as well as code blocks
+
+    or, naturally, simple paragraphs.
+
+[^unused]: This is unused.
+.
+<p>This is some text.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Other text.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p>
+<p>Here's a thing<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p>
+<p>And another thing<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p>
+<p>This doesn't have a referent[^nope].</p>
+<p>Hi!</p>
+<section class="footnotes">
+<ol>
+<li id="fn1">
+<p>Some <em>bolded</em> footnote definition. <a href="#fnref1" class="footnote-backref">↩</a></p>
+</li>
+<li id="fn2">
+<blockquote>
+<p>Blockquotes can be in a footnote.</p>
+</blockquote>
+<pre><code>as well as code blocks
+</code></pre>
+<p>or, naturally, simple paragraphs. <a href="#fnref2" class="footnote-backref">↩</a></p>
+</li>
+<li id="fn3">
+<p>no code block here (spaces are stripped away) <a href="#fnref3" class="footnote-backref">↩</a></p>
+</li>
+<li id="fn4">
+<pre><code>this is now a code block (8 spaces indentation)
+</code></pre>
+<a href="#fnref4" class="footnote-backref">↩</a>
+</li>
+</ol>
+</section>
+````````````````````````````````
 
 ## Interop
 

data/ext/commonmarker/cmark/test/pathological_tests.py

@@ -31,6 +31,8 @@ def hash_collisions():
 
     return document, re.compile("(<p>\[%s\]</p>\n){%d}" % (bad_key, COUNT-1))
 
+allowed_failures = {"many references": True}
+
 # list of pairs consisting of input and a regex that must match the output.
 pathological = {
     # note - some pythons have limit of 65535 for {num-matches} in re.
@@ -73,12 +75,22 @@ pathological = {
     "backticks":
                  ("".join(map(lambda x: ("e" + "`" * x), range(1,10000))),
                   re.compile("^<p>[e`]*</p>\n$")),
+    "unclosed links A":
+                 ("[a](<b" * 50000,
+                  re.compile("(\[a\]\(&lt;b){50000}")),
+    "unclosed links B":
+                 ("[a](b" * 50000,
+                  re.compile("(\[a\]\(b){50000}")),
+    "many references":
+                 ("".join(map(lambda x: ("[" + str(x) + "]: u\n"), range(1,50000 * 16))) + "[0] " * 50000,
+                  re.compile("(\[0\] ){49999}")),
     "reference collisions": hash_collisions()
     }
 
 whitespace_re = re.compile('/s+/')
 passed = 0
 errored = 0
+ignored = 0
 TIMEOUT = 5
 
 def run_test(inp, regex):
@@ -117,11 +129,17 @@ if __name__ == '__main__':
             p.terminate()
             p.join()
             print('[TIMED OUT]')
-            errored += 1
+            if allowed_failures[description]:
+                ignored += 1
+            else:
+                errored += 1
         elif p.exitcode != 0:
-            errored += 1
+            if allowed_failures[description]:
+                ignored += 1
+            else:
+                errored += 1
         else:
             passed += 1
 
-    print("%d passed, %d errored" % (passed, errored))
+    print("%d passed, %d errored, %d ignored" % (passed, errored, ignored))
     exit(errored)