commonmarker 0.16.4 → 0.16.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b55efd0b945c8781f23517de410731b3d2949968
-  data.tar.gz: 0646b61988c1e2a3945a1dfb013dcc9fd36f3063
+  metadata.gz: 095c88f0b8335861ecd1f2fd3036eb38e3ba15e5
+  data.tar.gz: 523409323276c59e1da4efda9c08d846b512f599
 SHA512:
-  metadata.gz: 29cca5037bf400c570a4d95163c0f448169be0672ffe8dc9d309dc4a3687ba7793e412e9d31cb0e378f8bf5c83cdcc5649d61682159892777f021f7f3ef30d94
-  data.tar.gz: 99707c291528f130ec808985930c7bf82e3d6884209b31782205dc02ac8f46bb7f4c261a5abb076ab0b3001652e3fff4299acebb59adbad3c9a7ced09a7f740d
+  metadata.gz: 0bc5a2a5d1193c8beb136e392db2fdb1f323e0b87f73aa4e2a54eb0ebc8240518a0151852e82b64ed2e8469e4786f41b8b166162cd0b929e38e6e385c5b6c8b2
+  data.tar.gz: 7bd29b88abaaacc24e05fa03bbfccffb9e703d0f57ec0efe903aa502bd7db3efdb6ef8e4682cdb36f58f04828e763f5f988a82f42f02c125fcd6e0f9e8ec01ee

data/ext/commonmarker/cmark/CMakeLists.txt

@@ -19,12 +19,13 @@ set(PROJECT_NAME "cmark")
 set(PROJECT_VERSION_MAJOR 0)
 set(PROJECT_VERSION_MINOR 27)
 set(PROJECT_VERSION_PATCH 1)
-set(PROJECT_VERSION_GFM 1)
+set(PROJECT_VERSION_GFM 2)
 set(PROJECT_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}.gfm.${PROJECT_VERSION_GFM} )
 
 option(CMARK_TESTS "Build cmark tests and enable testing" ON)
 option(CMARK_STATIC "Build static libcmark-gfm library" ON)
 option(CMARK_SHARED "Build shared libcmark-gfm library" ON)
+option(CMARK_LIB_FUZZER "Build libFuzzer fuzzing harness" OFF)
 
 add_subdirectory(src)
 add_subdirectory(extensions)

data/ext/commonmarker/cmark/Makefile

@@ -16,6 +16,7 @@ BENCHFILE=$(BENCHDIR)/benchinput.md
 ALLTESTS=alltests.md
 NUMRUNS?=20
 CMARK=$(BUILDDIR)/src/cmark-gfm
+CMARK_FUZZ=$(BUILDDIR)/src/cmark-fuzz
 PROG?=$(CMARK)
 VERSION?=$(SPECVERSION)
 RELEASE?=CommonMark-$(VERSION)
@@ -24,7 +25,7 @@ CLANG_CHECK?=clang-check
 CLANG_FORMAT=clang-format-3.5 -style llvm -sort-includes=0 -i
 AFL_PATH?=/usr/local/bin
 
-.PHONY: all cmake_build leakcheck clean fuzztest test debug ubsan asan mingw archive newbench bench format update-spec afl clang-check docker
+.PHONY: all cmake_build leakcheck clean fuzztest test debug ubsan asan mingw archive newbench bench format update-spec afl clang-check docker libFuzzer
 
 all: cmake_build man/man3/cmark-gfm.3
 
@@ -79,11 +80,18 @@ afl:
 	$(AFL_PATH)/afl-fuzz \
 	    -i test/afl_test_cases \
 	    -o test/afl_results \
-	    -x test/afl_dictionary \
+	    -x test/fuzzing_dictionary \
 	    $(AFL_OPTIONS) \
 	    -t 100 \
 	    $(CMARK) -e table -e strikethrough -e autolink -e tagfilter $(CMARK_OPTS)
 
+libFuzzer:
+	@[ -n "$(LIB_FUZZER_PATH)" ] || { echo '$$LIB_FUZZER_PATH not set'; false; }
+	mkdir -p $(BUILDDIR)
+	cd $(BUILDDIR) && cmake -DCMAKE_BUILD_TYPE=Asan -DCMARK_LIB_FUZZER=ON -DCMAKE_LIB_FUZZER_PATH=$(LIB_FUZZER_PATH) ..
+	$(MAKE) -j2 -C $(BUILDDIR) cmark-fuzz
+	test/run-cmark-fuzz $(CMARK_FUZZ)
+
 clang-check: all
 	${CLANG_CHECK} -p build -analyze src/*.c
 

data/ext/commonmarker/cmark/README.md

@@ -59,12 +59,14 @@ It is easy to use `libcmark` in python, lua, ruby, and other dynamic
 languages: see the `wrappers/` subdirectory for some simple examples.
 
 There are also libraries that wrap `libcmark` for
-[go](https://github.com/rhinoman/go-commonmark),
-[Haskell](http://hackage.haskell.org/package/cmark),
-[ruby](https://github.com/gjtorikian/commonmarker),
-[lua](https://github.com/jgm/cmark-lua),
-[Perl](https://metacpan.org/release/CommonMark), and
-[R](http://cran.r-project.org/package=commonmark).
+[Go](https://github.com/rhinoman/go-commonmark),
+[Haskell](https://hackage.haskell.org/package/cmark),
+[Ruby](https://github.com/gjtorikian/commonmarker),
+[Lua](https://github.com/jgm/cmark-lua),
+[Perl](https://metacpan.org/release/CommonMark),
+[Python](https://pypi.python.org/pypi/paka.cmark),
+[R](https://cran.r-project.org/package=commonmark) and
+[Scala](https://github.com/sparsetech/cmark-scala).
 
 Installing
 ----------
@@ -126,6 +128,13 @@ To do a more systematic fuzz test with [american fuzzy lop]:
 
     AFL_PATH=/path/to/afl_directory make afl
 
+Fuzzing with [libFuzzer] is also supported but, because libFuzzer is still
+under active development, may not work with your system-installed version of
+clang. Assuming LLVM has been built in `$HOME/src/llvm/build` the fuzzer can be
+run with:
+
+    CC="$HOME/src/llvm/build/bin/clang" LIB_FUZZER_PATH="$HOME/src/llvm/lib/Fuzzer/libFuzzer.a" make libFuzzer
+
 To make a release tarball and zip archive:
 
     make archive
@@ -193,3 +202,4 @@ most of the C library's API and its test harness.
 [Build Status]: https://img.shields.io/travis/github/cmark/master.svg?style=flat
 [Windows Build Status]: https://ci.appveyor.com/api/projects/status/wv7ifhqhv5itm3d5?svg=true
 [american fuzzy lop]: http://lcamtuf.coredump.cx/afl/
+[libFuzzer]: http://llvm.org/docs/LibFuzzer.html

data/ext/commonmarker/cmark/api_test/main.c

@@ -59,8 +59,8 @@ static void constructor(test_batch_runner *runner) {
              "default is list type is bullet");
       INT_EQ(runner, cmark_node_get_list_delim(node), CMARK_NO_DELIM,
              "default is list delim is NO_DELIM");
-      INT_EQ(runner, cmark_node_get_list_start(node), 1,
-             "default is list start is 1");
+      INT_EQ(runner, cmark_node_get_list_start(node), 0,
+             "default is list start is 0");
       INT_EQ(runner, cmark_node_get_list_tight(node), 0,
              "default is list is loose");
       break;

data/ext/commonmarker/cmark/changelog.txt

@@ -1,3 +1,8 @@
+[0.27.1.gfm.2]
+
+  * Fix a number of bugs (reading past end of buffer, undefined behavior.
+  * Add `cmark_syntax_extension_get_private()`. (Jonathan Müller)
+
 [0.27.1.gfm.1]
 
   * Add plaintext renderer.

data/ext/commonmarker/cmark/man/man3/cmark-gfm.3

@@ -1,4 +1,4 @@
-.TH cmark-gfm 3 "May 05, 2017" "LOCAL" "Library Functions Manual"
+.TH cmark-gfm 3 "April 03, 2017" "LOCAL" "Library Functions Manual"
 .SH
 NAME
 .PP
@@ -397,7 +397,8 @@ of the type, or \f[C]"<unknown>"\f[].
 
 .PP
 Returns the string contents of \f[I]node\f[], or an empty string if none
-is set.
+is set. Returns NULL if called on a node that does not have string
+content.
 
 .PP
 \fIint\f[] \fBcmark_node_set_literal\f[](\fIcmark_node *node\f[], \fIconst char *content\f[])
@@ -504,7 +505,8 @@ Returns code blocks fencing details
 
 .PP
 Returns the URL of a link or image \f[I]node\f[], or an empty string if
-no URL is set.
+no URL is set. Returns NULL if called on a node that is not a link or
+image.
 
 .PP
 \fIint\f[] \fBcmark_node_set_url\f[](\fIcmark_node *node\f[], \fIconst char *url\f[])
@@ -518,7 +520,8 @@ on failure.
 
 .PP
 Returns the title of a link or image \f[I]node\f[], or an empty string
-if no title is set.
+if no title is set. Returns NULL if called on a node that is not a link
+or image.
 
 .PP
 \fIint\f[] \fBcmark_node_set_title\f[](\fIcmark_node *node\f[], \fIconst char *title\f[])
@@ -532,7 +535,8 @@ on failure.
 
 .PP
 Returns the literal "on enter" text for a custom \f[I]node\f[], or an
-empty string if no on_enter is set.
+empty string if no on_enter is set. Returns NULL if called on a
+non\-custom node.
 
 .PP
 \fIint\f[] \fBcmark_node_set_on_enter\f[](\fIcmark_node *node\f[], \fIconst char *on_enter\f[])
@@ -547,7 +551,8 @@ success 0 on failure.
 
 .PP
 Returns the literal "on exit" text for a custom \f[I]node\f[], or an
-empty string if no on_exit is set.
+empty string if no on_exit is set. Returns NULL if called on a
+non\-custom node.
 
 .PP
 \fIint\f[] \fBcmark_node_set_on_exit\f[](\fIcmark_node *node\f[], \fIconst char *on_exit\f[])
@@ -862,6 +867,18 @@ Render \f[C]softbreak\f[] elements as spaces.
 .SS
 Options affecting parsing
 
+.PP
+.nf
+\fC
+.RS 0n
+#define CMARK_OPT_NORMALIZE (1 << 8)
+.RE
+\f[]
+.fi
+
+.PP
+Legacy option (no effect).
+
 .PP
 .nf
 \fC

data/ext/commonmarker/cmark/src/CMakeLists.txt

@@ -203,3 +203,14 @@ endif()
 if(CMAKE_BUILD_TYPE STREQUAL "Ubsan")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
 endif()
+
+if(CMARK_LIB_FUZZER)
+  set(FUZZ_HARNESS "cmark-fuzz")
+  add_executable(${FUZZ_HARNESS} ../test/cmark-fuzz.c ${LIBRARY_SOURCES})
+  target_link_libraries(${FUZZ_HARNESS} "${CMAKE_LIB_FUZZER_PATH}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize-coverage=trace-pc-guard")
+
+  # cmark is written in C but the libFuzzer runtime is written in C++ which
+  # needs to link against the C++ runtime. Explicitly link it into cmark-fuzz
+  set_target_properties(${FUZZ_HARNESS} PROPERTIES LINK_FLAGS "-lstdc++")
+endif()

data/ext/commonmarker/cmark/src/arena.c

@@ -71,7 +71,8 @@ static void *arena_calloc(size_t nmem, size_t size) {
   }
   void *ptr = (uint8_t *) A->ptr + A->used;
   A->used += sz;
-  *((size_t *) ptr) = nmem * size;
+  size_t new_sz = nmem * size;
+  memcpy(ptr, &new_sz, sizeof(new_sz));
   return (uint8_t *) ptr + sizeof(size_t);
 }
 

data/ext/commonmarker/cmark/src/blocks.c

@@ -460,8 +460,8 @@ static bufsize_t parse_list_marker(cmark_mem *mem, cmark_chunk *input,
     data->marker_offset = 0; // will be adjusted later
     data->list_type = CMARK_BULLET_LIST;
     data->bullet_char = c;
-    data->start = 1;
-    data->delimiter = CMARK_PERIOD_DELIM;
+    data->start = 0;
+    data->delimiter = CMARK_NO_DELIM;
     data->tight = false;
   } else if (cmark_isdigit(c)) {
     int start = 0;
@@ -1247,6 +1247,8 @@ static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
   else
     cmark_strbuf_put(&parser->curline, buffer, bytes);
 
+  bytes = parser->curline.size;
+
   // ensure line ends with a newline:
   if (bytes == 0 || !S_is_line_end_char(parser->curline.ptr[bytes - 1]))
     cmark_strbuf_putc(&parser->curline, '\n');

data/ext/commonmarker/cmark/src/cmark.h

@@ -337,7 +337,8 @@ CMARK_EXPORT
 const char *cmark_node_get_type_string(cmark_node *node);
 
 /** Returns the string contents of 'node', or an empty
-    string if none is set.
+    string if none is set.  Returns NULL if called on a
+    node that does not have string content.
  */
 CMARK_EXPORT const char *cmark_node_get_literal(cmark_node *node);
 
@@ -415,7 +416,8 @@ CMARK_EXPORT int cmark_node_set_fenced(cmark_node * node, int fenced,
 CMARK_EXPORT int cmark_node_get_fenced(cmark_node *node, int *length, int *offset, char *character);
 
 /** Returns the URL of a link or image 'node', or an empty string
-    if no URL is set.
+    if no URL is set.  Returns NULL if called on a node that is
+    not a link or image.
  */
 CMARK_EXPORT const char *cmark_node_get_url(cmark_node *node);
 
@@ -425,7 +427,8 @@ CMARK_EXPORT const char *cmark_node_get_url(cmark_node *node);
 CMARK_EXPORT int cmark_node_set_url(cmark_node *node, const char *url);
 
 /** Returns the title of a link or image 'node', or an empty
-    string if no title is set.
+    string if no title is set.  Returns NULL if called on a node
+    that is not a link or image.
  */
 CMARK_EXPORT const char *cmark_node_get_title(cmark_node *node);
 
@@ -435,7 +438,8 @@ CMARK_EXPORT const char *cmark_node_get_title(cmark_node *node);
 CMARK_EXPORT int cmark_node_set_title(cmark_node *node, const char *title);
 
 /** Returns the literal "on enter" text for a custom 'node', or
-    an empty string if no on_enter is set.
+    an empty string if no on_enter is set.  Returns NULL if called
+    on a non-custom node.
  */
 CMARK_EXPORT const char *cmark_node_get_on_enter(cmark_node *node);
 
@@ -447,7 +451,8 @@ CMARK_EXPORT int cmark_node_set_on_enter(cmark_node *node,
                                          const char *on_enter);
 
 /** Returns the literal "on exit" text for a custom 'node', or
-    an empty string if no on_exit is set.
+    an empty string if no on_exit is set.  Returns NULL if
+    called on a non-custom node.
  */
 CMARK_EXPORT const char *cmark_node_get_on_exit(cmark_node *node);
 
@@ -690,6 +695,10 @@ char *cmark_render_latex_with_mem(cmark_node *root, int options, int width, cmar
  * ### Options affecting parsing
  */
 
+/** Legacy option (no effect).
+ */
+#define CMARK_OPT_NORMALIZE (1 << 8)
+
 /** Validate UTF-8 in the input before parsing, replacing illegal
  * sequences with the replacement character U+FFFD.
  */

data/ext/commonmarker/cmark/src/commonmark.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <stdint.h>
 #include <assert.h>
 
 #include "config.h"
@@ -83,7 +84,9 @@ static int longest_backtick_sequence(const char *code) {
 }
 
 static int shortest_unused_backtick_sequence(const char *code) {
-  int32_t used = 1;
+  // note: if the shortest sequence is >= 32, this returns 32
+  // so as not to overflow the bit array.
+  uint32_t used = 1;
   int current = 0;
   size_t i = 0;
   size_t code_len = strlen(code);
@@ -91,7 +94,7 @@ static int shortest_unused_backtick_sequence(const char *code) {
     if (code[i] == '`') {
       current++;
     } else {
-      if (current) {
+      if (current > 0 && current < 32) {
         used |= (1 << current);
       }
       current = 0;
@@ -100,7 +103,7 @@ static int shortest_unused_backtick_sequence(const char *code) {
   }
   // return number of first bit that is 0:
   i = 0;
-  while (used & 1) {
+  while (i < 32 && used & 1) {
     used = used >> 1;
     i++;
   }

data/ext/commonmarker/cmark/src/inlines.c

@@ -322,13 +322,13 @@ static int scan_delims(subject *subj, unsigned char c, bool *can_open,
     after_char = 10;
   }
   left_flanking = numdelims > 0 && !cmark_utf8proc_is_space(after_char) &&
-                  !(cmark_utf8proc_is_punctuation(after_char) &&
-                    !cmark_utf8proc_is_space(before_char) &&
-                    !cmark_utf8proc_is_punctuation(before_char));
+                  (!cmark_utf8proc_is_punctuation(after_char) ||
+                   cmark_utf8proc_is_space(before_char) ||
+                   cmark_utf8proc_is_punctuation(before_char));
   right_flanking = numdelims > 0 && !cmark_utf8proc_is_space(before_char) &&
-                   !(cmark_utf8proc_is_punctuation(before_char) &&
-                     !cmark_utf8proc_is_space(after_char) &&
-                     !cmark_utf8proc_is_punctuation(after_char));
+                   (!cmark_utf8proc_is_punctuation(before_char) ||
+                    cmark_utf8proc_is_space(after_char) ||
+                    cmark_utf8proc_is_punctuation(after_char));
   if (c == '_') {
     *can_open = left_flanking &&
                 (!right_flanking || cmark_utf8proc_is_punctuation(before_char));
@@ -863,7 +863,9 @@ static bufsize_t manual_scan_link_url(cmark_chunk *input, bufsize_t offset) {
     }
   } else {
     while (i < input->len) {
-      if (input->data[i] == '\\')
+      if (input->data[i] == '\\' &&
+	  i + 1 < input-> len &&
+          cmark_ispunct(input->data[i+1]))
         i += 2;
       else if (input->data[i] == '(') {
         ++nb_p;

data/ext/commonmarker/cmark/src/latex.c

@@ -181,6 +181,10 @@ static link_type get_link_type(cmark_node *node) {
 
     link_text = node->first_child;
     cmark_consolidate_text_nodes(link_text);
+
+    if (!link_text)
+      return NO_LINK;
+
     realurl = (char *)url;
     realurllen = (int)url_len;
     if (strncmp(realurl, "mailto:", 7) == 0) {
@@ -218,11 +222,10 @@ static int S_get_enumlevel(cmark_node *node) {
 static int S_render_node(cmark_renderer *renderer, cmark_node *node,
                          cmark_event_type ev_type, int options) {
   int list_number;
+  int enumlevel;
   char list_number_string[LIST_NUMBER_STRING_SIZE];
   bool entering = (ev_type == CMARK_EVENT_ENTER);
   cmark_list_type list_type;
-  const char *roman_numerals[] = {"",   "i",   "ii",   "iii", "iv", "v",
-                                  "vi", "vii", "viii", "ix",  "x"};
   bool allow_wrap = renderer->width > 0 && !(CMARK_OPT_NOBREAKS & options);
 
   if (node->extension && node->extension->latex_render_func) {
@@ -253,13 +256,24 @@ static int S_render_node(cmark_renderer *renderer, cmark_node *node,
       CR();
       list_number = cmark_node_get_list_start(node);
       if (list_number > 1) {
-        snprintf(list_number_string, LIST_NUMBER_STRING_SIZE, "%d",
-                 list_number);
-        LIT("\\setcounter{enum");
-        LIT((char *)roman_numerals[S_get_enumlevel(node)]);
-        LIT("}{");
-        OUT(list_number_string, false, NORMAL);
-        LIT("}");
+	enumlevel = S_get_enumlevel(node);
+	// latex normally supports only five levels
+	if (enumlevel >= 1 && enumlevel <= 5) {
+          snprintf(list_number_string, LIST_NUMBER_STRING_SIZE, "%d",
+                   list_number);
+          LIT("\\setcounter{enum");
+          switch(enumlevel) {
+	  case 1: LIT("i"); break;
+	  case 2: LIT("ii"); break;
+	  case 3: LIT("iii"); break;
+	  case 4: LIT("iv"); break;
+	  case 5: LIT("v"); break;
+	  default: LIT("i"); break;
+	  }
+          LIT("}{");
+          OUT(list_number_string, false, NORMAL);
+          LIT("}");
+	}
         CR();
       }
     } else {

data/ext/commonmarker/cmark/src/node.c

@@ -81,7 +81,7 @@ cmark_node *cmark_node_new_with_mem(cmark_node_type type, cmark_mem *mem) {
   case CMARK_NODE_LIST: {
     cmark_list *list = &node->as.list;
     list->list_type = CMARK_BULLET_LIST;
-    list->start = 1;
+    list->start = 0;
     list->tight = false;
     break;
   }

data/ext/commonmarker/cmark/src/scanners.c

@@ -752,7 +752,7 @@ bufsize_t _scan_autolink_uri(const unsigned char *p) {
         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
         0,   0,   0,   0,   0,   128, 128, 128, 128, 128, 128, 128, 128, 128,
         128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
-        128, 128, 128, 128, 0,   128, 0,   128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 0,   128, 128, 128, 128, 128, 128, 128,
         128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
         128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
         128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
@@ -839,7 +839,7 @@ bufsize_t _scan_autolink_uri(const unsigned char *p) {
     }
     if (yych <= 0xEC) {
       if (yych <= 0xC1) {
-        if (yych <= '<')
+        if (yych <= ' ')
           goto yy45;
         if (yych <= '>')
           goto yy85;

data/ext/commonmarker/cmark/src/scanners.re

@@ -91,7 +91,7 @@ bufsize_t _scan_autolink_uri(const unsigned char *p)
   const unsigned char *marker = NULL;
   const unsigned char *start = p;
 /*!re2c
-  scheme [:][^\x00-\x20<>]*[>]  { return (bufsize_t)(p - start); }
+  scheme [:][^\x00-\x20>]*[>]  { return (bufsize_t)(p - start); }
   * { return 0; }
 */
 }

data/ext/commonmarker/cmark/test/cmark-fuzz.c

@@ -0,0 +1,28 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include "cmark.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  int options = 0;
+  if (size > sizeof(options)) {
+    /* First 4 bytes of input are treated as options */
+    int options = *(const int *)data;
+
+    /* Mask off valid option bits */
+    options = options & (CMARK_OPT_SOURCEPOS | CMARK_OPT_HARDBREAKS | CMARK_OPT_SAFE | CMARK_OPT_NOBREAKS | CMARK_OPT_NORMALIZE | CMARK_OPT_VALIDATE_UTF8 | CMARK_OPT_SMART);
+
+    /* Remainder of input is the markdown */
+    const char *markdown = (const char *)(data + sizeof(options));
+    const size_t markdown_size = size - sizeof(options);
+    cmark_node *doc = cmark_parse_document(markdown, markdown_size, options);
+
+    free(cmark_render_commonmark(doc, options, 80));
+    free(cmark_render_html(doc, options, NULL));
+    free(cmark_render_latex(doc, options, 80));
+    free(cmark_render_man(doc, options, 80));
+    free(cmark_render_xml(doc, options));
+
+    cmark_node_free(doc);
+  }
+  return 0;
+}

data/ext/commonmarker/cmark/test/fuzzing_dictionary

@@ -0,0 +1,49 @@
+asterisk="*"
+attr_generic=" a=\"1\""
+attr_href=" href=\"1\""
+attr_xml_lang=" xml:lang=\"1\""
+attr_xmlns=" xmlns=\"1\""
+backslash="\\"
+backtick="`"
+colon=":"
+dashes="---"
+double_quote="\""
+entity_builtin="<"
+entity_decimal=""
+entity_external="&a;"
+entity_hex=""
+equals="==="
+exclamation="!"
+greater_than=">"
+hash="#"
+hyphen="-"
+indent="  "
+left_bracket="["
+left_paren="("
+less_than="<"
+plus="+"
+right_bracket="]"
+right_paren=")"
+single_quote="'"
+string_any="ANY"
+string_brackets="[]"
+string_cdata="CDATA"
+string_dashes="--"
+string_empty_dblquotes="\"\""
+string_empty_quotes="''"
+string_idrefs="IDREFS"
+string_parentheses="()"
+string_pcdata="#PCDATA"
+tag_cdata="<![CDATA["
+tag_close="</a>"
+tag_doctype="<!DOCTYPE"
+tag_element="<!ELEMENT"
+tag_entity="<!ENTITY"
+tag_notation="<!NOTATION"
+tag_open="<a>"
+tag_open_close="<a />"
+tag_open_exclamation="<!"
+tag_open_q="<?"
+tag_sq2_close="]]>"
+tag_xml_q="<?xml?>"
+underscore="_"

data/ext/commonmarker/cmark/test/regression.txt

@@ -74,3 +74,21 @@ a***b* c*
 <p>a*<em><em>b</em> c</em></p>
 ````````````````````````````````
 
+Issue #193 - unescaped left angle brackets in link destination
+
+```````````````````````````````` example
+[a]
+
+[a]: <te<st>
+.
+<p><a href="te%3Cst">a</a></p>
+````````````````````````````````
+
+Issue #192 - escaped spaces in link destination
+
+
+```````````````````````````````` example
+[a](te\ st)
+.
+<p>[a](te\ st)</p>
+````````````````````````````````

data/ext/commonmarker/cmark/test/run-cmark-fuzz

@@ -0,0 +1,4 @@
+#!/bin/bash -eu
+CMARK_FUZZ="$1"
+shift
+ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1" "${CMARK_FUZZ}" -max_len=256 -timeout=1 -dict=test/fuzzing_dictionary "$@"

data/ext/commonmarker/cmark/test/spec.txt

@@ -23,10 +23,12 @@ are hence known as **extensions**, and highlighted as such.
 ## What is Markdown?
 
 Markdown is a plain text format for writing structured documents,
-based on conventions used for indicating formatting in email and
-usenet posts.  It was developed in 2004 by John Gruber, who wrote
-the first Markdown-to-HTML converter in Perl, and it soon became
-ubiquitous.  In the next decade, dozens of implementations were
+based on conventions for indicating formatting in email
+and usenet posts.  It was developed by John Gruber (with
+help from Aaron Swartz) and released in 2004 in the form of a
+[syntax description](http://daringfireball.net/projects/markdown/syntax)
+and a Perl script (`Markdown.pl`) for converting Markdown to
+HTML.  In the next decade, dozens of implementations were
 developed in many languages.  Some extended the original
 Markdown syntax with conventions for footnotes, tables, and
 other document elements.  Some allowed Markdown documents to be
@@ -324,7 +326,7 @@ form feed (`U+000C`), or carriage return (`U+000D`).
 characters].
 
 A [Unicode whitespace character](@) is
-any code point in the Unicode `Zs` class, or a tab (`U+0009`),
+any code point in the Unicode `Zs` general category, or a tab (`U+0009`),
 carriage return (`U+000D`), newline (`U+000A`), or form feed
 (`U+000C`).
 
@@ -343,7 +345,7 @@ is `!`, `"`, `#`, `$`, `%`, `&`, `'`, `(`, `)`,
 
 A [punctuation character](@) is an [ASCII
 punctuation character] or anything in
-the Unicode classes `Pc`, `Pd`, `Pe`, `Pf`, `Pi`, `Po`, or `Ps`.
+the general Unicode categories  `Pc`, `Pd`, `Pe`, `Pf`, `Pi`, `Po`, or `Ps`.
 
 ## Tabs
 
@@ -414,8 +416,8 @@ as indentation with four spaces would:
 Normally the `>` that begins a block quote may be followed
 optionally by a space, which is not considered part of the
 content.  In the following case `>` is followed by a tab,
-which is treated as if it were expanded into spaces.
-Since one of theses spaces is considered part of the
+which is treated as if it were expanded into three spaces.
+Since one of these spaces is considered part of the
 delimiter, `foo` is considered to be indented six spaces
 inside the block quote context, so we get an indented
 code block starting with two spaces.
@@ -493,7 +495,7 @@ We can think of a document as a sequence of
 quotations, lists, headings, rules, and code blocks.  Some blocks (like
 block quotes and list items) contain other blocks; others (like
 headings and paragraphs) contain [inline](@) content---text,
-links, emphasized text, images, code, and so on.
+links, emphasized text, images, code spans, and so on.
 
 ## Precedence
 
@@ -6045,6 +6047,15 @@ we just have literal backticks:
 <p>`foo</p>
 ````````````````````````````````
 
+The following case also illustrates the need for opening and
+closing backtick strings to be equal in length:
+
+```````````````````````````````` example
+`foo``bar``
+.
+<p>`foo<code>bar</code></p>
+````````````````````````````````
+
 
 ## Emphasis and strong emphasis
 
@@ -6099,14 +6110,14 @@ characters that is not preceded or followed by a `_` character.
 
 A [left-flanking delimiter run](@) is
 a [delimiter run] that is (a) not followed by [Unicode whitespace],
-and (b) either not followed by a [punctuation character], or
+and (b) not followed by a [punctuation character], or
 preceded by [Unicode whitespace] or a [punctuation character].
 For purposes of this definition, the beginning and the end of
 the line count as Unicode whitespace.
 
 A [right-flanking delimiter run](@) is
 a [delimiter run] that is (a) not preceded by [Unicode whitespace],
-and (b) either not preceded by a [punctuation character], or
+and (b) not preceded by a [punctuation character], or
 followed by [Unicode whitespace] or a [punctuation character].
 For purposes of this definition, the beginning and the end of
 the line count as Unicode whitespace.
@@ -6185,7 +6196,7 @@ The following rules define emphasis and strong emphasis:
 7.  A double `**` [can close strong emphasis](@)
     iff it is part of a [right-flanking delimiter run].
 
-8.  A double `__` [can close strong emphasis]
+8.  A double `__` [can close strong emphasis] iff
     it is part of a [right-flanking delimiter run]
     and either (a) not part of a [left-flanking delimiter run]
     or (b) part of a [left-flanking delimiter run]
@@ -6226,7 +6237,7 @@ the following principles resolve ambiguity:
     `<em><em>...</em></em>`.
 
 14. An interpretation `<em><strong>...</strong></em>` is always
-    preferred to `<strong><em>..</em></strong>`.
+    preferred to `<strong><em>...</em></strong>`.
 
 15. When two potential emphasis or strong emphasis spans overlap,
     so that the second begins before the first ends and ends after
@@ -8605,11 +8616,11 @@ The link labels are case-insensitive:
 ````````````````````````````````
 
 
-If you just want bracketed text, you can backslash-escape the
-opening `!` and `[`:
+If you just want a literal `!` followed by bracketed text, you can
+backslash-escape the opening `[`:
 
 ```````````````````````````````` example
-\!\[foo]
+!\[foo]
 
 [foo]: /url "title"
 .

data/ext/commonmarker/cmark/tools/Dockerfile

@@ -1,20 +1,19 @@
 FROM debian:jessie
 
 RUN apt-get update && apt-get install -y \
-  build-essential autoconf libtool \
+  build-essential \
+  autoconf \
+  libtool \
   git \
   pkg-config \
-  && apt-get clean
-
-RUN apt-get install -y \
-  cmake \
   gdb \
   valgrind \
-  python3
-
-RUN apt-get install -y \
+  python3 \
   wget \
-  clang
+  clang \
+  man \
+  clang-format-3.5 \
+  && apt-get clean
 
 RUN wget http://lcamtuf.coredump.cx/afl/releases/afl-latest.tgz && \
 	tar xf afl-latest.tgz && \
@@ -23,8 +22,6 @@ RUN wget http://lcamtuf.coredump.cx/afl/releases/afl-latest.tgz && \
 	cd .. && \
 	rm -rf afl-*
 
-RUN apt-get install -y man
-
 RUN wget https://github.com/skvadrik/re2c/releases/download/0.15.3/re2c-0.15.3.tar.gz && \
 	tar xf re2c-0.15.3.tar.gz && \
 	cd re2c-* && \
@@ -33,4 +30,10 @@ RUN wget https://github.com/skvadrik/re2c/releases/download/0.15.3/re2c-0.15.3.t
 	cd .. && \
 	rm -rf re2c-*
 
-RUN apt-get install -y clang-format-3.5
+RUN wget https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz && \
+	tar xf cmake-3.8.2.tar.gz && \
+	cd cmake-* && \
+	./bootstrap && \
+	make install && \
+	cd .. && \
+	rm -rf cmake-*

data/lib/commonmarker/version.rb

@@ -1,3 +1,3 @@
 module CommonMarker
-  VERSION = '0.16.4'.freeze
+  VERSION = '0.16.5'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: commonmarker
 version: !ruby/object:Gem::Version
-  version: 0.16.4
+  version: 0.16.5
 platform: ruby
 authors:
 - Garen Torikian
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-27 00:00:00.000000000 Z
+date: 2017-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-enum
@@ -238,63 +238,17 @@ files:
 - ext/commonmarker/cmark/src/xml.c
 - ext/commonmarker/cmark/suppressions
 - ext/commonmarker/cmark/test/CMakeLists.txt
-- ext/commonmarker/cmark/test/afl_dictionary/asterisk
-- ext/commonmarker/cmark/test/afl_dictionary/attr_generic
-- ext/commonmarker/cmark/test/afl_dictionary/attr_href
-- ext/commonmarker/cmark/test/afl_dictionary/attr_xml_lang
-- ext/commonmarker/cmark/test/afl_dictionary/attr_xmlns
-- ext/commonmarker/cmark/test/afl_dictionary/backslash
-- ext/commonmarker/cmark/test/afl_dictionary/backtick
-- ext/commonmarker/cmark/test/afl_dictionary/colon
-- ext/commonmarker/cmark/test/afl_dictionary/dashes
-- ext/commonmarker/cmark/test/afl_dictionary/double_quote
-- ext/commonmarker/cmark/test/afl_dictionary/entity_builtin
-- ext/commonmarker/cmark/test/afl_dictionary/entity_decimal
-- ext/commonmarker/cmark/test/afl_dictionary/entity_external
-- ext/commonmarker/cmark/test/afl_dictionary/entity_hex
-- ext/commonmarker/cmark/test/afl_dictionary/equals
-- ext/commonmarker/cmark/test/afl_dictionary/exclamation
-- ext/commonmarker/cmark/test/afl_dictionary/greater_than
-- ext/commonmarker/cmark/test/afl_dictionary/hash
-- ext/commonmarker/cmark/test/afl_dictionary/hyphen
-- ext/commonmarker/cmark/test/afl_dictionary/indent
-- ext/commonmarker/cmark/test/afl_dictionary/left_bracket
-- ext/commonmarker/cmark/test/afl_dictionary/left_paren
-- ext/commonmarker/cmark/test/afl_dictionary/less_than
-- ext/commonmarker/cmark/test/afl_dictionary/plus
-- ext/commonmarker/cmark/test/afl_dictionary/right_bracket
-- ext/commonmarker/cmark/test/afl_dictionary/right_paren
-- ext/commonmarker/cmark/test/afl_dictionary/single_quote
-- ext/commonmarker/cmark/test/afl_dictionary/string_any
-- ext/commonmarker/cmark/test/afl_dictionary/string_brackets
-- ext/commonmarker/cmark/test/afl_dictionary/string_cdata
-- ext/commonmarker/cmark/test/afl_dictionary/string_dashes
-- ext/commonmarker/cmark/test/afl_dictionary/string_empty_dblquotes
-- ext/commonmarker/cmark/test/afl_dictionary/string_empty_quotes
-- ext/commonmarker/cmark/test/afl_dictionary/string_idrefs
-- ext/commonmarker/cmark/test/afl_dictionary/string_parentheses
-- ext/commonmarker/cmark/test/afl_dictionary/string_pcdata
-- ext/commonmarker/cmark/test/afl_dictionary/tag_cdata
-- ext/commonmarker/cmark/test/afl_dictionary/tag_close
-- ext/commonmarker/cmark/test/afl_dictionary/tag_doctype
-- ext/commonmarker/cmark/test/afl_dictionary/tag_element
-- ext/commonmarker/cmark/test/afl_dictionary/tag_entity
-- ext/commonmarker/cmark/test/afl_dictionary/tag_notation
-- ext/commonmarker/cmark/test/afl_dictionary/tag_open
-- ext/commonmarker/cmark/test/afl_dictionary/tag_open_close
-- ext/commonmarker/cmark/test/afl_dictionary/tag_open_exclamation
-- ext/commonmarker/cmark/test/afl_dictionary/tag_open_q
-- ext/commonmarker/cmark/test/afl_dictionary/tag_sq2_close
-- ext/commonmarker/cmark/test/afl_dictionary/tag_xml_q
-- ext/commonmarker/cmark/test/afl_dictionary/underscore
 - ext/commonmarker/cmark/test/afl_test_cases/test.md
+- ext/commonmarker/cmark/test/cmark-fuzz.c
 - ext/commonmarker/cmark/test/cmark.py
 - ext/commonmarker/cmark/test/entity_tests.py
 - ext/commonmarker/cmark/test/extensions.txt
+- ext/commonmarker/cmark/test/fuzzing_dictionary
 - ext/commonmarker/cmark/test/normalize.py
 - ext/commonmarker/cmark/test/pathological_tests.py
 - ext/commonmarker/cmark/test/regression.txt
 - ext/commonmarker/cmark/test/roundtrip_tests.py
+- ext/commonmarker/cmark/test/run-cmark-fuzz
 - ext/commonmarker/cmark/test/smart_punct.txt
 - ext/commonmarker/cmark/test/spec.txt
 - ext/commonmarker/cmark/test/spec_tests.py

data/ext/commonmarker/cmark/test/afl_dictionary/asterisk

@@ -1 +0,0 @@
-*

data/ext/commonmarker/cmark/test/afl_dictionary/attr_generic

@@ -1 +0,0 @@
- a="1"

data/ext/commonmarker/cmark/test/afl_dictionary/attr_href

@@ -1 +0,0 @@
- href="1"

data/ext/commonmarker/cmark/test/afl_dictionary/attr_xml_lang

@@ -1 +0,0 @@
- xml:lang="1"

data/ext/commonmarker/cmark/test/afl_dictionary/attr_xmlns

@@ -1 +0,0 @@
- xmlns="1"

data/ext/commonmarker/cmark/test/afl_dictionary/backslash

@@ -1 +0,0 @@
-\

data/ext/commonmarker/cmark/test/afl_dictionary/backtick

@@ -1 +0,0 @@
-`

data/ext/commonmarker/cmark/test/afl_dictionary/colon

@@ -1 +0,0 @@
-:

data/ext/commonmarker/cmark/test/afl_dictionary/dashes

@@ -1 +0,0 @@
----

data/ext/commonmarker/cmark/test/afl_dictionary/double_quote

@@ -1 +0,0 @@
-"

data/ext/commonmarker/cmark/test/afl_dictionary/entity_builtin

@@ -1 +0,0 @@
-<

data/ext/commonmarker/cmark/test/afl_dictionary/entity_decimal

@@ -1 +0,0 @@
-

data/ext/commonmarker/cmark/test/afl_dictionary/entity_external

@@ -1 +0,0 @@
-&a;

data/ext/commonmarker/cmark/test/afl_dictionary/entity_hex

@@ -1 +0,0 @@
-

data/ext/commonmarker/cmark/test/afl_dictionary/equals

@@ -1 +0,0 @@
-===

data/ext/commonmarker/cmark/test/afl_dictionary/exclamation

@@ -1 +0,0 @@
-!

data/ext/commonmarker/cmark/test/afl_dictionary/greater_than

@@ -1 +0,0 @@
->

data/ext/commonmarker/cmark/test/afl_dictionary/hash

@@ -1 +0,0 @@
-#

data/ext/commonmarker/cmark/test/afl_dictionary/indent

@@ -1 +0,0 @@
-  

data/ext/commonmarker/cmark/test/afl_dictionary/left_bracket

@@ -1 +0,0 @@
-[

data/ext/commonmarker/cmark/test/afl_dictionary/left_paren

@@ -1 +0,0 @@
-(

data/ext/commonmarker/cmark/test/afl_dictionary/less_than

@@ -1 +0,0 @@
-<

data/ext/commonmarker/cmark/test/afl_dictionary/plus

@@ -1 +0,0 @@
-+

data/ext/commonmarker/cmark/test/afl_dictionary/right_bracket

@@ -1 +0,0 @@
-]

data/ext/commonmarker/cmark/test/afl_dictionary/right_paren

@@ -1 +0,0 @@
-)

data/ext/commonmarker/cmark/test/afl_dictionary/single_quote

@@ -1 +0,0 @@
-'

data/ext/commonmarker/cmark/test/afl_dictionary/string_any

@@ -1 +0,0 @@
-ANY

data/ext/commonmarker/cmark/test/afl_dictionary/string_brackets

@@ -1 +0,0 @@
-[]

data/ext/commonmarker/cmark/test/afl_dictionary/string_cdata

@@ -1 +0,0 @@
-CDATA

data/ext/commonmarker/cmark/test/afl_dictionary/string_dashes

@@ -1 +0,0 @@
---

data/ext/commonmarker/cmark/test/afl_dictionary/string_empty_dblquotes

@@ -1 +0,0 @@
-""

data/ext/commonmarker/cmark/test/afl_dictionary/string_empty_quotes

@@ -1 +0,0 @@
-''

data/ext/commonmarker/cmark/test/afl_dictionary/string_idrefs

@@ -1 +0,0 @@
-IDREFS

data/ext/commonmarker/cmark/test/afl_dictionary/string_parentheses

@@ -1 +0,0 @@
-()

data/ext/commonmarker/cmark/test/afl_dictionary/string_pcdata

@@ -1 +0,0 @@
-#PCDATA

data/ext/commonmarker/cmark/test/afl_dictionary/tag_cdata

@@ -1 +0,0 @@
-<![CDATA[

data/ext/commonmarker/cmark/test/afl_dictionary/tag_close

@@ -1 +0,0 @@
-</a>

data/ext/commonmarker/cmark/test/afl_dictionary/tag_doctype

@@ -1 +0,0 @@
-<!DOCTYPE

data/ext/commonmarker/cmark/test/afl_dictionary/tag_element

@@ -1 +0,0 @@
-<!ELEMENT

data/ext/commonmarker/cmark/test/afl_dictionary/tag_entity

@@ -1 +0,0 @@
-<!ENTITY

data/ext/commonmarker/cmark/test/afl_dictionary/tag_notation

@@ -1 +0,0 @@
-<!NOTATION

data/ext/commonmarker/cmark/test/afl_dictionary/tag_open

@@ -1 +0,0 @@
-<a>

data/ext/commonmarker/cmark/test/afl_dictionary/tag_open_close

@@ -1 +0,0 @@
-<a />

data/ext/commonmarker/cmark/test/afl_dictionary/tag_open_exclamation

@@ -1 +0,0 @@
-<!

data/ext/commonmarker/cmark/test/afl_dictionary/tag_open_q

@@ -1 +0,0 @@
-<?

data/ext/commonmarker/cmark/test/afl_dictionary/tag_sq2_close

@@ -1 +0,0 @@
-]]>

data/ext/commonmarker/cmark/test/afl_dictionary/tag_xml_q

@@ -1 +0,0 @@
-<?xml?>

data/ext/commonmarker/cmark/test/afl_dictionary/underscore

@@ -1 +0,0 @@
-_