commonmarker 0.16.8 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c967d121b146595ca1a31130b381e8f323879cf3
-  data.tar.gz: 4fd79c41fd8520743ccbba1039681a2dc41c6762
+  metadata.gz: 78ee91668a78aaedd870bbaa8bf8e6cd551cadc9
+  data.tar.gz: 2391e06fd9d4e7dff10c71b9750e5141e19c0516
 SHA512:
-  metadata.gz: be9d93dc746d42505c8d10007c379d252d04c72efbfe2236dcac606d6df13d7ec4465b251129cecaaa19e3827ef32a6fea445fcdcc0c8836caa1488d7186b134
-  data.tar.gz: a1086e64de21020597e4b66cc941295a7be660fa5aafed3a50be342d93acae175fd4f9b309264db010dd6ec71d1655cc0cc57e129449d3dd9e56c191fb8ada4a
+  metadata.gz: 6df4d33c89c1a28d2d26f187ac1e54038dcfb0ecabb947d1437b48aeb49943e5896efce5adba324dc30d2e8b9d063be96a7c866d02f9849672e29b25b23c656a
+  data.tar.gz: ea01b53f7f78cd0d4c0489beaee09cf9aa6dafb38d932aaae7211aa82100d717ae67580837c27721b9aa23134462b2b1f4feaac67b3d02db5ca880f4cbb40471

data/README.md

@@ -160,7 +160,7 @@ To have multiple options applied, pass in an array of symbols:
 CommonMarker.render_html("\"'Shelob' is my name.\"", [:HARDBREAKS, :SOURCEPOS])
 ```
 
-For more information on these options, see [the CMark documentation](http://git.io/vlQii).
+For more information on these options, see [the CMark documentation](https://git.io/v7nh1).
 
 ## Hacking
 

data/ext/commonmarker/cmark/CMakeLists.txt

@@ -17,9 +17,9 @@ endif()
 set(PROJECT_NAME "cmark")
 
 set(PROJECT_VERSION_MAJOR 0)
-set(PROJECT_VERSION_MINOR 27)
-set(PROJECT_VERSION_PATCH 1)
-set(PROJECT_VERSION_GFM 4)
+set(PROJECT_VERSION_MINOR 28)
+set(PROJECT_VERSION_PATCH 0)
+set(PROJECT_VERSION_GFM 8)
 set(PROJECT_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}.gfm.${PROJECT_VERSION_GFM} )
 
 option(CMARK_TESTS "Build cmark tests and enable testing" ON)

data/ext/commonmarker/cmark/README.md

@@ -116,9 +116,9 @@ To run a test for memory leaks using `valgrind`:
 
     make leakcheck
 
-To reformat source code using `astyle`:
+To reformat source code using `clang-format`:
 
-    make astyle
+    make format
 
 To run a "fuzz test" against ten long randomly generated inputs:
 

data/ext/commonmarker/cmark/api_test/CMakeLists.txt

@@ -7,8 +7,9 @@ add_executable(api_test
 include_directories(
   ${PROJECT_SOURCE_DIR}/src
   ${PROJECT_BINARY_DIR}/src
+  ${PROJECT_BINARY_DIR}/extensions
 )
-target_link_libraries(api_test libcmark-gfm)
+target_link_libraries(api_test libcmark-gfmextensions libcmark-gfm)
 
 # Compiler flags
 if(MSVC)

data/ext/commonmarker/cmark/api_test/harness.c

@@ -50,6 +50,21 @@ void INT_EQ(test_batch_runner *runner, int got, int expected, const char *msg,
   }
 }
 
+#ifndef _WIN32
+#include <unistd.h>
+
+static char *write_tmp(char const *header, char const *data) {
+  char *name = strdup("/tmp/fileXXXXXX");
+  int fd = mkstemp(name);
+  FILE *f = fdopen(fd, "w+");
+  fputs(header, f);
+  fwrite(data, 1, strlen(data), f);
+  fclose(f);
+  return name;
+}
+
+#endif
+
 void STR_EQ(test_batch_runner *runner, const char *got, const char *expected,
             const char *msg, ...) {
   int cond = strcmp(got, expected) == 0;
@@ -60,8 +75,20 @@ void STR_EQ(test_batch_runner *runner, const char *got, const char *expected,
   va_end(ap);
 
   if (!cond) {
+#ifndef _WIN32
+    char *got_fn = write_tmp("actual\n", got);
+    char *expected_fn = write_tmp("expected\n", expected);
+    char buf[1024];
+    snprintf(buf, sizeof(buf), "git diff --no-index %s %s", expected_fn, got_fn);
+    system(buf);
+    remove(got_fn);
+    remove(expected_fn);
+    free(got_fn);
+    free(expected_fn);
+#else
     fprintf(stderr, "  Got:      \"%s\"\n", got);
     fprintf(stderr, "  Expected: \"%s\"\n", expected);
+#endif
   }
 }
 

data/ext/commonmarker/cmark/api_test/main.c

@@ -5,6 +5,7 @@
 #define CMARK_NO_SHORT_NAMES
 #include "cmark.h"
 #include "node.h"
+#include "../extensions/core-extensions.h"
 
 #include "harness.h"
 #include "cplusplus.h"
@@ -551,9 +552,9 @@ static void render_xml(test_batch_runner *runner) {
   STR_EQ(runner, xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                       "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n"
                       "<paragraph sourcepos=\"1:1-1:9\">\n"
-                      "  <text>foo </text>\n"
-                      "  <emph>\n"
-                      "    <text>bar</text>\n"
+                      "  <text sourcepos=\"1:1-1:4\">foo </text>\n"
+                      "  <emph sourcepos=\"1:5-1:9\">\n"
+                      "    <text sourcepos=\"1:6-1:8\">bar</text>\n"
                       "  </emph>\n"
                       "</paragraph>\n",
          "render first paragraph with source pos");
@@ -933,6 +934,178 @@ static void test_feed_across_line_ending(test_batch_runner *runner) {
   cmark_node_free(document);
 }
 
+static void source_pos(test_batch_runner *runner) {
+  static const char markdown[] =
+    "# Hi *there*.\n"
+    "\n"
+    "Hello &ldquo; <http://www.google.com>\n"
+    "there `hi` -- [okay](www.google.com (ok)).\n"
+    "\n"
+    "> 1. Okay.\n"
+    ">    Sure.\n"
+    ">\n"
+    "> 2. Yes, okay.\n"
+    ">    ![ok](hi \"yes\")\n";
+
+  cmark_node *doc = cmark_parse_document(markdown, sizeof(markdown) - 1, CMARK_OPT_DEFAULT);
+  char *xml = cmark_render_xml(doc, CMARK_OPT_DEFAULT | CMARK_OPT_SOURCEPOS);
+  STR_EQ(runner, xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                      "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n"
+                      "<document sourcepos=\"1:1-10:20\" xmlns=\"http://commonmark.org/xml/1.0\">\n"
+                      "  <heading sourcepos=\"1:1-1:13\" level=\"1\">\n"
+                      "    <text sourcepos=\"1:3-1:5\">Hi </text>\n"
+                      "    <emph sourcepos=\"1:6-1:12\">\n"
+                      "      <text sourcepos=\"1:7-1:11\">there</text>\n"
+                      "    </emph>\n"
+                      "    <text sourcepos=\"1:13-1:13\">.</text>\n"
+                      "  </heading>\n"
+                      "  <paragraph sourcepos=\"3:1-4:42\">\n"
+                      "    <text sourcepos=\"3:1-3:14\">Hello “ </text>\n"
+                      "    <link sourcepos=\"3:15-3:37\" destination=\"http://www.google.com\" title=\"\">\n"
+                      "      <text sourcepos=\"3:16-3:36\">http://www.google.com</text>\n"
+                      "    </link>\n"
+                      "    <softbreak />\n"
+                      "    <text sourcepos=\"4:1-4:6\">there </text>\n"
+                      "    <code sourcepos=\"4:8-4:9\">hi</code>\n"
+                      "    <text sourcepos=\"4:11-4:14\"> -- </text>\n"
+                      "    <link sourcepos=\"4:15-4:41\" destination=\"www.google.com\" title=\"ok\">\n"
+                      "      <text sourcepos=\"4:16-4:19\">okay</text>\n"
+                      "    </link>\n"
+                      "    <text sourcepos=\"4:42-4:42\">.</text>\n"
+                      "  </paragraph>\n"
+                      "  <block_quote sourcepos=\"6:1-10:20\">\n"
+                      "    <list sourcepos=\"6:3-10:20\" type=\"ordered\" start=\"1\" delim=\"period\" tight=\"false\">\n"
+                      "      <item sourcepos=\"6:3-8:1\">\n"
+                      "        <paragraph sourcepos=\"6:6-7:10\">\n"
+                      "          <text sourcepos=\"6:6-6:10\">Okay.</text>\n"
+                      "          <softbreak />\n"
+                      "          <text sourcepos=\"7:6-7:10\">Sure.</text>\n"
+                      "        </paragraph>\n"
+                      "      </item>\n"
+                      "      <item sourcepos=\"9:3-10:20\">\n"
+                      "        <paragraph sourcepos=\"9:6-10:20\">\n"
+                      "          <text sourcepos=\"9:6-9:15\">Yes, okay.</text>\n"
+                      "          <softbreak />\n"
+                      "          <image sourcepos=\"10:6-10:20\" destination=\"hi\" title=\"yes\">\n"
+                      "            <text sourcepos=\"10:8-10:9\">ok</text>\n"
+                      "          </image>\n"
+                      "        </paragraph>\n"
+                      "      </item>\n"
+                      "    </list>\n"
+                      "  </block_quote>\n"
+                      "</document>\n",
+         "sourcepos are as expected");
+  free(xml);
+  cmark_node_free(doc);
+}
+
+static void ext_source_pos(test_batch_runner *runner) {
+  static const char *extensions[3] = {
+    "strikethrough",
+    "table",
+    "autolink",
+  };
+
+  static const char markdown[] =
+    "Hi ~~friend~~.\n"
+    "\n"
+    "> www.github.com\n"
+    "\n"
+    "1. | a | b | *c* |\n"
+    "   | - | - | --: |\n"
+    "   | 1 | 2 | ~3~ |\n";
+
+  cmark_parser *parser = cmark_parser_new(CMARK_OPT_DEFAULT);
+  core_extensions_ensure_registered();
+
+  for (int i = 0; i < (int)(sizeof(extensions) / sizeof(*extensions)); ++i) {
+    cmark_syntax_extension *ext = cmark_find_syntax_extension(extensions[i]);
+    cmark_parser_attach_syntax_extension(parser, ext);
+  }
+
+  cmark_parser_feed(parser, markdown, sizeof(markdown) - 1);
+
+  cmark_node *doc = cmark_parser_finish(parser);
+  char *xml = cmark_render_xml(doc, CMARK_OPT_DEFAULT | CMARK_OPT_SOURCEPOS);
+  STR_EQ(runner, xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                      "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n"
+                      "<document sourcepos=\"1:1-7:18\" xmlns=\"http://commonmark.org/xml/1.0\">\n"
+                      "  <paragraph sourcepos=\"1:1-1:14\">\n"
+                      "    <text sourcepos=\"1:1-1:3\">Hi </text>\n"
+                      "    <strikethrough sourcepos=\"1:4-1:13\">\n"
+                      "      <text sourcepos=\"1:6-1:11\">friend</text>\n"
+                      "    </strikethrough>\n"
+                      "    <text sourcepos=\"1:14-1:14\">.</text>\n"
+                      "  </paragraph>\n"
+                      "  <block_quote sourcepos=\"3:1-3:16\">\n"
+                      "    <paragraph sourcepos=\"3:3-3:16\">\n"
+                      "      <link sourcepos=\"3:2-3:16\" destination=\"http://www.github.com\" title=\"\">\n"
+                      "        <text sourcepos=\"3:2-3:16\">www.github.com</text>\n"
+                      "      </link>\n"
+                      "    </paragraph>\n"
+                      "  </block_quote>\n"
+		      "  <list sourcepos=\"5:1-7:18\" type=\"ordered\" start=\"1\" delim=\"period\" tight=\"true\">\n"
+		      "    <item sourcepos=\"5:1-7:18\">\n"
+                      "      <table sourcepos=\"5:4-7:18\">\n"
+                      "        <table_header sourcepos=\"5:4-5:18\">\n"
+                      "          <table_cell sourcepos=\"5:5-5:7\">\n"
+                      "            <text sourcepos=\"5:6-5:6\">a</text>\n"
+                      "          </table_cell>\n"
+                      "          <table_cell sourcepos=\"5:9-5:11\">\n"
+                      "            <text sourcepos=\"5:10-5:10\">b</text>\n"
+                      "          </table_cell>\n"
+                      "          <table_cell sourcepos=\"5:13-5:17\">\n"
+                      "            <emph sourcepos=\"5:14-5:16\">\n"
+                      "              <text sourcepos=\"5:15-5:15\">c</text>\n"
+                      "            </emph>\n"
+                      "          </table_cell>\n"
+                      "        </table_header>\n"
+                      "        <table_row sourcepos=\"7:4-7:18\">\n"
+                      "          <table_cell sourcepos=\"7:5-7:7\">\n"
+                      "            <text sourcepos=\"7:6-7:6\">1</text>\n"
+                      "          </table_cell>\n"
+                      "          <table_cell sourcepos=\"7:9-7:11\">\n"
+                      "            <text sourcepos=\"7:10-7:10\">2</text>\n"
+                      "          </table_cell>\n"
+                      "          <table_cell sourcepos=\"7:13-7:17\">\n"
+                      "            <strikethrough sourcepos=\"7:14-7:16\">\n"
+                      "              <text sourcepos=\"7:15-7:15\">3</text>\n"
+                      "            </strikethrough>\n"
+                      "          </table_cell>\n"
+                      "        </table_row>\n"
+                      "      </table>\n"
+		      "    </item>\n"
+		      "  </list>\n"
+                      "</document>\n",
+         "sourcepos are as expected");
+  free(xml);
+  cmark_node_free(doc);
+}
+
+static void ref_source_pos(test_batch_runner *runner) {
+  static const char markdown[] =
+    "Let's try [reference] links.\n"
+    "\n"
+    "[reference]: https://github.com (GitHub)\n";
+
+  cmark_node *doc = cmark_parse_document(markdown, sizeof(markdown) - 1, CMARK_OPT_DEFAULT);
+  char *xml = cmark_render_xml(doc, CMARK_OPT_DEFAULT | CMARK_OPT_SOURCEPOS);
+  STR_EQ(runner, xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+                      "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n"
+                      "<document sourcepos=\"1:1-3:40\" xmlns=\"http://commonmark.org/xml/1.0\">\n"
+                      "  <paragraph sourcepos=\"1:1-1:28\">\n"
+                      "    <text sourcepos=\"1:1-1:10\">Let's try </text>\n"
+                      "    <link sourcepos=\"1:11-1:21\" destination=\"https://github.com\" title=\"GitHub\">\n"
+                      "      <text sourcepos=\"1:12-1:20\">reference</text>\n"
+                      "    </link>\n"
+                      "    <text sourcepos=\"1:22-1:28\"> links.</text>\n"
+                      "  </paragraph>\n"
+                      "</document>\n",
+         "sourcepos are as expected");
+  free(xml);
+  cmark_node_free(doc);
+}
+
 int main() {
   int retval;
   test_batch_runner *runner = test_batch_runner_new();
@@ -959,6 +1132,9 @@ int main() {
   test_cplusplus(runner);
   test_safe(runner);
   test_feed_across_line_ending(runner);
+  source_pos(runner);
+  ext_source_pos(runner);
+  ref_source_pos(runner);
 
   test_print_summary(runner);
   retval = test_ok(runner) ? 0 : 1;

data/ext/commonmarker/cmark/changelog.txt

@@ -1,3 +1,151 @@
+[0.28.0.gfm.8]
+
+  * Fix bug where autolink would cause `:` to be skipped in emphasis
+    processing.
+
+[0.28.0.gfm.7]
+
+  * Strikethrough characters do not disturb regular emphasis processing.
+
+[0.28.0.gfm.6]
+
+  * Fix inline sourcepos info when inlines span multiple lines, and in
+    ATX headings.
+
+[0.28.0.gfm.5]
+
+  * Latest spec.
+  * Fix a typo in the spec (John Gardner).
+  * Fix quadratic behavior in reference lookups.
+  * Add `core_extensions_ensure_registered`.
+  * Add sourcepos information for inlines.
+
+[0.28]
+
+  * Update spec.
+  * Use unsigned integer when shifting (Phil Turnbull).
+    Avoids a UBSAN warning which can be triggered when handling a
+    long sequence of backticks.
+  * Avoid memcpy'ing NULL pointers (Phil Turnbull).
+    Avoids a UBSAN warning when link title is empty string.
+    The length of the memcpy is zero so the NULL pointer is not
+    dereferenced but it is still undefined behaviour.
+  * DeMorgan simplification of some tests in emphasis parser.
+    This also brings the code into closer alignment with the wording
+    of the spec (see jgm/CommonMark#467).
+  * Fixed undefined shift in commonmark writer (#211).
+    Found by google/oss-fuzz:
+    <https://oss-fuzz.com/v2/testcase-detail/4686992824598528>.
+  * latex writer:  fix memory overflow (#210).
+    We got an array overflow in enumerated lists nested more than
+    10 deep with start number =/= 1.
+    This commit also ensures that we don't try to set `enum_` counters
+    that aren't defined by LaTeX (generally up to enumv).
+    Found by google/oss-fuzz:
+    <https://oss-fuzz.com/v2/testcase-detail/5546760854306816>.
+  * Check for NULL pointer in get_link_type (Phil Turnbull).
+    `echo '[](xx:)' | ./build/src/cmark -t latex` gave a
+    segfault.
+  * Move fuzzing dictionary into single file (Phil Turnbull).
+    This allows AFL and libFuzzer to use the same dictionary
+  * Reset bytes after UTF8 proc (Yuki Izumi, #206).
+  * Don't scan past an EOL (Yuki Izumi).
+    The existing negated character classes (`[^…]`) are careful to
+    always include` \x00` in the characters excluded, but these `.`
+    catch-alls can scan right past the terminating NUL placed
+    at the end of the buffer by `_scan_at`.  As such, buffer
+    overruns can occur.  Also, don't scan past a newline in HTML
+    block end scanners.
+  * Document cases where `get_` functions return `NULL` (#155).
+    E.g. `cmark_node_get_url` on a non-link or image.
+  * Properly handle backslashes in link destinations (#192).
+    Only ascii punctuation characters are escapable, per the spec.
+  * Fixed `cmark_node_get_list_start` to return 0 for bullet lists,
+    as documented (#202).
+  * Use `CMARK_NO_DELIM` for bullet lists (#201).
+  * Fixed code for freeing delimiter stack (#189).
+  * Removed abort outside of conditional (typo).
+  * Removed coercion in error message when aborting from buffer.
+  * Print message to stderr when we abort due to memory demands (#188).
+  * `libcmark.pc`: use `CMAKE_INSTALL_LIBDIR` (#185, Jens Petersen).
+    Needed for multilib distros like Fedora.
+  * Fixed buffer overflow error in `S_parser_feed` (#184).
+    The overflow could occur in the following condition:
+    the buffer ends with `\r` and the next memory address
+    contains `\n`.
+  * Update emphasis parsing for spec change.
+    Strong now goes inside Emph rather than the reverse,
+    when both scopes are possible.  The code is much simpler.
+    This also avoids a spec inconsistency that cmark had previously:
+    `***hi***` became Strong (Emph "hi")) but
+    `***hi****` became Emph (Strong "hi")) "*"
+  * Fixes for the LaTeX renderer (#182, Doeme)
+    + Don't double-output the link in latex-rendering.
+    + Prevent ligatures in dashes sensibly when rendering latex.
+      `\-` is a hyphenation, so it doesn't get displayed at all.
+  * Added a test for NULL when freeing `subj->last_delim`.
+  * Cleaned up setting of lower bounds for openers.
+    We now use a much smaller array.
+  * Fix #178, quadratic parsing bug.  Add pathological test.
+  * Slight improvement of clarity of logic in emph matching.
+  * Fix "multiple of 3" determination in emph/strong parsing.
+    We need to store the length of the original delimiter run,
+    instead of using the length of the remaining delimiters
+    after some have been subtracted.  Test case:
+    `a***b* c*`.  Thanks to Raph Levin for reporting.
+  * Correctly initialize chunk in S_process_line (Nick Wellnhofer, #170).
+    The `alloc` member wasn't initialized.  This also allows to add an
+    assertion in `chunk_rtrim` which doesn't work for alloced chunks.
+  * Added 'make newbench'.
+  * `scanners.c` generated with re2c 0.16 (68K smaller!).
+  * `scanners.re` - fixed warnings; use `*` for fallback.
+  * Fixed some warnings in `scanners.re`.
+  * Update CaseFolding to latest (Kevin Wojniak, #168).
+  * Allow balanced nested parens in link destinations (Yuki Izumi, #166)
+  * Allocate enough bytes for backticks array.
+  * Inlines: Ensure that the delimiter stack is freed in subject.
+  * Fixed pathological cases with backtick code spans:
+
+    - Removed recursion in scan_to_closing_backticks
+    - Added an array of pointers to potential backtick closers
+      to subject
+    - This array is used to avoid traversing the subject again
+      when we've already seen all the potential backtick closers.
+    - Added a max bound of 1000 for backtick code span delimiters.
+    - This helps with pathological cases like:
+
+            x
+            x `
+            x ``
+            x ```
+            x ````
+            ...
+
+    - Added pathological test case.
+
+    Thanks to Martin Mitáš for identifying the problem and for
+    discussion of solutions.
+  * Remove redundant cmake_minimum_required (#163, @kainjow).
+  * Make shared and static libraries optional (Azamat H. Hackimov).
+    Now you can enable/disable compilation and installation targets for
+    shared and static libraries via `-DCMARK_SHARED=ON/OFF` and
+    `-DCMARK_STATIC=ON/OFF`.
+  * Added support for built-in `${LIB_SUFFIX}` feature (Azamat H.
+    Hackimov).  Replaced `${LIB_INSTALL_DIR}` option with built-in
+    `${LIB_SUFFIX}` for installing for 32/64-bit systems. Normally,
+    CMake will set `${LIB_SUFFIX}` automatically for required enviroment.
+    If you have any issues with it, you can override this option with
+    `-DLIB_SUFFIX=64` or `-DLIB_SUFFIX=""` during configuration.
+  * Add Makefile target and harness to fuzz with libFuzzer (Phil Turnbull).
+    This can be run locally with `make libFuzzer` but the harness will be
+    integrated into oss-fuzz for large-scale fuzzing.
+  * Advertise `--validate-utf8` in usage information
+    (Nguyễn Thái Ngọc Duy).
+  * Makefile: use warnings with re2c.
+  * README: Add link to Python wrapper, prettify languages list
+    (Pavlo Kapyshin).
+  * README: Add link to cmark-scala (Tim Nieradzik, #196)
+
 [0.27.1.gfm.4]
 
   * Fix regression with nested parentheses in link targets (#48).