commonmarker 0.9.1 → 0.9.2

data/ext/commonmarker/cmark/src/parser.h

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include "node.h"
 #include "buffer.h"
+#include "memory.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -12,6 +13,7 @@ extern "C" {
 #define MAX_LINK_LABEL_LENGTH 1000
 
 struct cmark_parser {
+  struct cmark_mem *mem;
   struct cmark_reference_map *refmap;
   struct cmark_node *root;
   struct cmark_node *current;
@@ -23,9 +25,9 @@ struct cmark_parser {
   int indent;
   bool blank;
   bool partially_consumed_tab;
-  cmark_strbuf *curline;
+  cmark_strbuf curline;
   bufsize_t last_line_length;
-  cmark_strbuf *linebuf;
+  cmark_strbuf linebuf;
   int options;
   bool last_buffer_ended_with_cr;
 };

data/ext/commonmarker/cmark/src/references.c

@@ -14,12 +14,13 @@ static unsigned int refhash(const unsigned char *link_ref) {
   return hash;
 }
 
-static void reference_free(cmark_reference *ref) {
+static void reference_free(cmark_reference_map *map, cmark_reference *ref) {
+  cmark_mem *mem = map->mem;
   if (ref != NULL) {
-    free(ref->label);
-    cmark_chunk_free(&ref->url);
-    cmark_chunk_free(&ref->title);
-    free(ref);
+    mem->free(ref->label);
+    cmark_chunk_free(mem, &ref->url);
+    cmark_chunk_free(mem, &ref->title);
+    mem->free(ref);
   }
 }
 
@@ -27,8 +28,8 @@ static void reference_free(cmark_reference *ref) {
 // remove leading/trailing whitespace, case fold
 // Return NULL if the reference name is actually empty (i.e. composed
 // solely from whitespace)
-static unsigned char *normalize_reference(cmark_chunk *ref) {
-  cmark_strbuf normalized = GH_BUF_INIT;
+static unsigned char *normalize_reference(cmark_mem *mem, cmark_chunk *ref) {
+  cmark_strbuf normalized = CMARK_BUF_INIT(mem);
   unsigned char *result;
 
   if (ref == NULL)
@@ -57,7 +58,7 @@ static void add_reference(cmark_reference_map *map, cmark_reference *ref) {
 
   while (t) {
     if (t->hash == ref->hash && !strcmp((char *)t->label, (char *)ref->label)) {
-      reference_free(ref);
+      reference_free(map, ref);
       return;
     }
 
@@ -70,22 +71,20 @@ static void add_reference(cmark_reference_map *map, cmark_reference *ref) {
 void cmark_reference_create(cmark_reference_map *map, cmark_chunk *label,
                             cmark_chunk *url, cmark_chunk *title) {
   cmark_reference *ref;
-  unsigned char *reflabel = normalize_reference(label);
+  unsigned char *reflabel = normalize_reference(map->mem, label);
 
   /* empty reference name, or composed from only whitespace */
   if (reflabel == NULL)
     return;
 
-  ref = (cmark_reference *)calloc(1, sizeof(*ref));
-  if (ref != NULL) {
-    ref->label = reflabel;
-    ref->hash = refhash(ref->label);
-    ref->url = cmark_clean_url(url);
-    ref->title = cmark_clean_title(title);
-    ref->next = NULL;
+  ref = (cmark_reference *)map->mem->calloc(1, sizeof(*ref));
+  ref->label = reflabel;
+  ref->hash = refhash(ref->label);
+  ref->url = cmark_clean_url(map->mem, url);
+  ref->title = cmark_clean_title(map->mem, title);
+  ref->next = NULL;
 
-    add_reference(map, ref);
-  }
+  add_reference(map, ref);
 }
 
 // Returns reference if refmap contains a reference with matching
@@ -102,7 +101,7 @@ cmark_reference *cmark_reference_lookup(cmark_reference_map *map,
   if (map == NULL)
     return NULL;
 
-  norm = normalize_reference(label);
+  norm = normalize_reference(map->mem, label);
   if (norm == NULL)
     return NULL;
 
@@ -131,7 +130,7 @@ void cmark_reference_map_free(cmark_reference_map *map) {
 
     while (ref) {
       next = ref->next;
-      reference_free(ref);
+      reference_free(map, ref);
       ref = next;
     }
   }
@@ -139,6 +138,8 @@ void cmark_reference_map_free(cmark_reference_map *map) {
   free(map);
 }
 
-cmark_reference_map *cmark_reference_map_new(void) {
-  return (cmark_reference_map *)calloc(1, sizeof(cmark_reference_map));
+cmark_reference_map *cmark_reference_map_new(cmark_mem *mem) {
+  cmark_reference_map *map = (cmark_reference_map *)mem->calloc(1, sizeof(cmark_reference_map));
+  map->mem = mem;
+  return map;
 }

data/ext/commonmarker/cmark/src/references.h

@@ -1,6 +1,7 @@
 #ifndef CMARK_REFERENCES_H
 #define CMARK_REFERENCES_H
 
+#include "memory.h"
 #include "chunk.h"
 
 #ifdef __cplusplus
@@ -20,12 +21,13 @@ struct cmark_reference {
 typedef struct cmark_reference cmark_reference;
 
 struct cmark_reference_map {
+  cmark_mem *mem;
   cmark_reference *table[REFMAP_SIZE];
 };
 
 typedef struct cmark_reference_map cmark_reference_map;
 
-cmark_reference_map *cmark_reference_map_new(void);
+cmark_reference_map *cmark_reference_map_new(cmark_mem *mem);
 void cmark_reference_map_free(cmark_reference_map *map);
 cmark_reference *cmark_reference_lookup(cmark_reference_map *map,
                                         cmark_chunk *label);

data/ext/commonmarker/cmark/src/render.c

@@ -4,6 +4,7 @@
 #include "cmark.h"
 #include "utf8.h"
 #include "render.h"
+#include "node.h"
 
 static CMARK_INLINE void S_cr(cmark_renderer *renderer) {
   if (renderer->need_cr < 1) {
@@ -19,7 +20,7 @@ static CMARK_INLINE void S_blankline(cmark_renderer *renderer) {
 
 static void S_out(cmark_renderer *renderer, const char *source, bool wrap,
                   cmark_escaping escape) {
-  int length = cmark_strbuf_safe_strlen(source);
+  int length = strlen(source);
   unsigned char nextc;
   int32_t c;
   int i = 0;
@@ -28,7 +29,7 @@ static void S_out(cmark_renderer *renderer, const char *source, bool wrap,
   cmark_chunk remainder = cmark_chunk_literal("");
   int k = renderer->buffer->size - 1;
 
-  wrap = wrap && !renderer->no_wrap;
+  wrap = wrap && !renderer->no_linebreaks;
 
   if (renderer->in_tight_list_item && renderer->need_cr > 1) {
     renderer->need_cr = 1;
@@ -108,7 +109,7 @@ static void S_out(cmark_renderer *renderer, const char *source, bool wrap,
         !renderer->begin_line && renderer->last_breakable > 0) {
 
       // copy from last_breakable to remainder
-      cmark_chunk_set_cstr(&remainder, (char *)renderer->buffer->ptr +
+      cmark_chunk_set_cstr(renderer->mem, &remainder, (char *)renderer->buffer->ptr +
                                            renderer->last_breakable + 1);
       // truncate at last_breakable
       cmark_strbuf_truncate(renderer->buffer, renderer->last_breakable);
@@ -118,7 +119,7 @@ static void S_out(cmark_renderer *renderer, const char *source, bool wrap,
                        renderer->prefix->size);
       cmark_strbuf_put(renderer->buffer, remainder.data, remainder.len);
       renderer->column = renderer->prefix->size + remainder.len;
-      cmark_chunk_free(&remainder);
+      cmark_chunk_free(renderer->mem, &remainder);
       renderer->last_breakable = 0;
       renderer->begin_line = false;
       renderer->begin_content = false;
@@ -146,14 +147,15 @@ char *cmark_render(cmark_node *root, int options, int width,
                    int (*render_node)(cmark_renderer *renderer,
                                       cmark_node *node,
                                       cmark_event_type ev_type, int options)) {
-  cmark_strbuf pref = GH_BUF_INIT;
-  cmark_strbuf buf = GH_BUF_INIT;
+  cmark_mem *mem = cmark_node_mem(root);
+  cmark_strbuf pref = CMARK_BUF_INIT(mem);
+  cmark_strbuf buf = CMARK_BUF_INIT(mem);
   cmark_node *cur;
   cmark_event_type ev_type;
   char *result;
   cmark_iter *iter = cmark_iter_new(root);
 
-  cmark_renderer renderer = {&buf, &pref, 0,           width, 0,
+  cmark_renderer renderer = {mem, &buf, &pref, 0, width, 0,
                              0,    true,  true,        false, false,
                              outc, S_cr,  S_blankline, S_out};
 

data/ext/commonmarker/cmark/src/render.h

@@ -8,10 +8,12 @@ extern "C" {
 #include <stdlib.h>
 #include "buffer.h"
 #include "chunk.h"
+#include "memory.h"
 
 typedef enum { LITERAL, NORMAL, TITLE, URL } cmark_escaping;
 
 struct cmark_renderer {
+  cmark_mem *mem;
   cmark_strbuf *buffer;
   cmark_strbuf *prefix;
   int column;
@@ -20,7 +22,7 @@ struct cmark_renderer {
   bufsize_t last_breakable;
   bool begin_line;
   bool begin_content;
-  bool no_wrap;
+  bool no_linebreaks;
   bool in_tight_list_item;
   void (*outc)(struct cmark_renderer *, cmark_escaping, int32_t, unsigned char);
   void (*cr)(struct cmark_renderer *);

data/ext/commonmarker/cmark/src/scanners.c

@@ -8258,37 +8258,45 @@ bufsize_t _scan_html_tag(const unsigned char *p) {
     unsigned char yych;
     static const unsigned char yybm[] = {
         /* table 1 .. 8: 0 */
-        0, 230, 230, 230, 230, 230, 230, 230, 230, 199, 199, 199, 199, 199, 230,
+        0,   230, 230, 230, 230, 230, 230, 230, 230, 199, 199, 199, 199, 199,
         230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230,
-        230, 230, 230, 199, 230, 70, 230, 230, 230, 230, 134, 230, 230, 230,
-        230, 230, 254, 246, 230, 254, 254, 254, 254, 254, 254, 254, 254, 254,
-        254, 246, 230, 198, 198, 196, 230, 230, 254, 254, 254, 254, 254, 254,
+        230, 230, 230, 230, 199, 230, 70,  230, 230, 230, 230, 134, 230, 230,
+        230, 230, 230, 254, 246, 230, 254, 254, 254, 254, 254, 254, 254, 254,
+        254, 254, 246, 230, 198, 198, 196, 230, 230, 254, 254, 254, 254, 254,
         254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
-        254, 254, 254, 254, 254, 254, 230, 230, 226, 230, 246, 198, 254, 254,
+        254, 254, 254, 254, 254, 254, 254, 230, 230, 226, 230, 246, 198, 254,
         254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
-        254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 230, 230, 230, 230,
-        230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 230, 230, 230,
+        230, 230, 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,
         /* table 9 .. 11: 256 */
-        0, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
+        0,   160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
-        160, 160, 32, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
-        160, 160, 160, 160, 160, 160, 128, 160, 224, 224, 224, 224, 224, 224,
+        160, 160, 160, 32,  160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
+        160, 160, 160, 160, 160, 160, 160, 128, 160, 224, 224, 224, 224, 224,
         224, 224, 224, 224, 224, 224, 224, 224, 224, 224, 224, 224, 224, 224,
-        224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 160, 160, 160, 160,
+        224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
         160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, 160,
-        160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        160, 160, 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,
     };
     yych = *(marker = p);
     if (yych <= '`') {

data/ext/commonmarker/cmark/src/xml.c

@@ -149,7 +149,7 @@ static int S_render_node(cmark_node *node, cmark_event_type ev_type,
 
 char *cmark_render_xml(cmark_node *root, int options) {
   char *result;
-  cmark_strbuf xml = GH_BUF_INIT;
+  cmark_strbuf xml = CMARK_BUF_INIT(cmark_node_mem(root));
   cmark_event_type ev_type;
   cmark_node *cur;
   struct render_state state = {&xml, 0};

data/ext/commonmarker/cmark/test/CMakeLists.txt

@@ -13,12 +13,12 @@ add_test(NAME api_test COMMAND api_test)
 
 if (WIN32)
   file(TO_NATIVE_PATH ${CMAKE_BINARY_DIR}/src WIN_DLL_DIR)
-  set_tests_properties(api_test PROPERTIES
-    ENVIRONMENT "PATH=${WIN_DLL_DIR};$ENV{PATH}"
-    )
-  set(ROUNDTRIP,"${CMAKE_CURRENT_SOURCE_DIR}/roundtrip.bat")
+  set(NEWPATH "${WIN_DLL_DIR};$ENV{PATH}")
+  string(REPLACE ";" "\\;" NEWPATH "${NEWPATH}")
+  set_tests_properties(api_test PROPERTIES ENVIRONMENT "PATH=${NEWPATH}")
+  set(ROUNDTRIP "${CMAKE_CURRENT_SOURCE_DIR}/roundtrip.bat")
 else(WIN32)
-  set(ROUNDTRIP,"${CMAKE_CURRENT_SOURCE_DIR}/roundtrip.sh")
+  set(ROUNDTRIP "${CMAKE_CURRENT_SOURCE_DIR}/roundtrip.sh")
 endif(WIN32)
 
 IF (PYTHONINTERP_FOUND)
@@ -48,9 +48,9 @@ IF (PYTHONINTERP_FOUND)
 
   add_test(roundtriptest_executable
     ${PYTHON_EXECUTABLE}
-    "${CMAKE_CURRENT_SOURCE_DIR}/spec_tests.py" "--no-normalize"
-    "--spec" "${CMAKE_CURRENT_SOURCE_DIR}/spec.txt" "--program"
-    "${ROUNDTRIP} ${CMAKE_CURRENT_BINARY_DIR}/../src/cmark"
+    "${CMAKE_CURRENT_SOURCE_DIR}/roundtrip_tests.py"
+    "--spec" "${CMAKE_CURRENT_SOURCE_DIR}/spec.txt"
+    "--library-dir" "${CMAKE_CURRENT_BINARY_DIR}/../src"
     )
 
   add_test(regressiontest_executable

data/ext/commonmarker/cmark/test/cmark.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from ctypes import CDLL, c_char_p, c_long
+from ctypes import CDLL, c_char_p, c_size_t, c_int, c_void_p
 from subprocess import *
 import platform
 import os
@@ -11,30 +11,50 @@ def pipe_through_prog(prog, text):
     [result, err] = p1.communicate(input=text.encode('utf-8'))
     return [p1.returncode, result.decode('utf-8'), err]
 
-def use_library(lib, text):
+def to_html(lib, text):
+    markdown = lib.cmark_markdown_to_html
+    markdown.restype = c_char_p
+    markdown.argtypes = [c_char_p, c_size_t, c_int]
     textbytes = text.encode('utf-8')
     textlen = len(textbytes)
-    return [0, lib(textbytes, textlen, 0).decode('utf-8'), '']
+    result = markdown(textbytes, textlen, 0).decode('utf-8')
+    return [0, result, '']
+
+def to_commonmark(lib, text):
+    textbytes = text.encode('utf-8')
+    textlen = len(textbytes)
+    parse_document = lib.cmark_parse_document
+    parse_document.restype = c_void_p
+    parse_document.argtypes = [c_char_p, c_size_t, c_int]
+    render_commonmark = lib.cmark_render_commonmark
+    render_commonmark.restype = c_char_p
+    render_commonmark.argtypes = [c_void_p, c_int, c_int]
+    node = parse_document(textbytes, textlen, 0)
+    result = render_commonmark(node, 0, 0).decode('utf-8')
+    return [0, result, '']
 
 class CMark:
     def __init__(self, prog=None, library_dir=None):
         self.prog = prog
         if prog:
             self.to_html = lambda x: pipe_through_prog(prog, x)
+            self.to_commonmark = lambda x: pipe_through_prog(prog + ' -t commonmark', x)
         else:
             sysname = platform.system()
             if sysname == 'Darwin':
-                libname = "libcmark.dylib"
+                libnames = [ "libcmark.dylib" ]
             elif sysname == 'Windows':
-                libname = "cmark.dll"
+                libnames = [ "cmark.dll", "libcmark.dll" ]
             else:
-                libname = "libcmark.so"
-            if library_dir:
-                libpath = os.path.join(library_dir, libname)
-            else:
-                libpath = os.path.join("build", "src", libname)
+                libnames = [ "libcmark.so" ]
+            if not library_dir:
+                library_dir = os.path.join("build", "src")
+            for libname in libnames:
+                candidate = os.path.join(library_dir, libname)
+                if os.path.isfile(candidate):
+                    libpath = candidate
+                    break
             cmark = CDLL(libpath)
-            markdown = cmark.cmark_markdown_to_html
-            markdown.restype = c_char_p
-            markdown.argtypes = [c_char_p, c_long]
-            self.to_html = lambda x: use_library(markdown, x)
+            self.to_html = lambda x: to_html(cmark, x)
+            self.to_commonmark = lambda x: to_commonmark(cmark, x)
+

data/ext/commonmarker/cmark/test/roundtrip_tests.py

@@ -0,0 +1,47 @@
+import re
+import sys
+from spec_tests import get_tests, do_test
+from cmark import CMark
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Run cmark roundtrip tests.')
+    parser.add_argument('-p', '--program', dest='program', nargs='?', default=None,
+            help='program to test')
+    parser.add_argument('-s', '--spec', dest='spec', nargs='?', default='spec.txt',
+            help='path to spec')
+    parser.add_argument('-P', '--pattern', dest='pattern', nargs='?',
+            default=None, help='limit to sections matching regex pattern')
+    parser.add_argument('--library-dir', dest='library_dir', nargs='?',
+            default=None, help='directory containing dynamic library')
+    parser.add_argument('--no-normalize', dest='normalize',
+            action='store_const', const=False, default=True,
+            help='do not normalize HTML')
+    parser.add_argument('-n', '--number', type=int, default=None,
+            help='only consider the test with the given number')
+    args = parser.parse_args(sys.argv[1:])
+
+spec = sys.argv[1]
+
+def converter(md):
+  cmark = CMark(prog=args.program, library_dir=args.library_dir)
+  [ec, result, err] = cmark.to_commonmark(md)
+  if ec == 0:
+    [ec, html, err] = cmark.to_html(result)
+    if ec == 0:
+        # In the commonmark writer we insert dummy HTML
+        # comments between lists, and between lists and code
+        # blocks.  Strip these out, since the spec uses
+        # two blank lines instead:
+        return [ec, re.sub('<!-- end list -->\n', '', html), '']
+    else:
+        return [ec, html, err]
+  else:
+    return [ec, result, err]
+
+tests = get_tests(args.spec)
+result_counts = {'pass': 0, 'fail': 0, 'error': 0, 'skip': 0}
+for test in tests:
+    do_test(converter, test, args.normalize, result_counts)
+
+exit(result_counts['fail'] + result_counts['error'])