commonmarker 0.0.1

data/ext/commonmarker/cmark/Makefile

@@ -0,0 +1,170 @@
+SRCDIR=src
+DATADIR=data
+BUILDDIR?=build
+GENERATOR?=Unix Makefiles
+MINGW_BUILDDIR?=build-mingw
+MINGW_INSTALLDIR?=windows
+SPEC=test/spec.txt
+SITE=_site
+SPECVERSION=$(shell perl -ne 'print $$1 if /^version: *([0-9.]+)/' $(SPEC))
+FUZZCHARS?=2000000  # for fuzztest
+BENCHDIR=bench
+BENCHFILE=$(BENCHDIR)/benchinput.md
+ALLTESTS=alltests.md
+NUMRUNS?=10
+CMARK=$(BUILDDIR)/src/cmark
+PROG?=$(CMARK)
+BENCHINP?=README.md
+VERSION?=$(SPECVERSION)
+RELEASE?=CommonMark-$(VERSION)
+INSTALL_PREFIX?=/usr/local
+
+.PHONY: all cmake_build spec leakcheck clean fuzztest dingus upload test update-site upload-site debug asan mingw archive bench astyle update-spec afl
+
+all: cmake_build man/man3/cmark.3
+
+$(CMARK): cmake_build
+
+cmake_build: $(BUILDDIR)
+	@make -j2 -C $(BUILDDIR)
+	@echo "Binaries can be found in $(BUILDDIR)/src"
+
+$(BUILDDIR):
+	@cmake --version > /dev/null || (echo "You need cmake to build this program: http://www.cmake.org/download/" && exit 1)
+	mkdir -p $(BUILDDIR); \
+	cd $(BUILDDIR); \
+	cmake .. \
+		-G "$(GENERATOR)" \
+		-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
+		-DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX)
+
+install: $(BUILDDIR)
+	make -C $(BUILDDIR) install
+
+debug:
+	mkdir -p $(BUILDDIR); \
+	cd $(BUILDDIR); \
+	cmake .. -DCMAKE_BUILD_TYPE=Debug; \
+	make
+
+asan:
+	mkdir -p $(BUILDDIR); \
+	cd $(BUILDDIR); \
+	cmake .. -DCMAKE_BUILD_TYPE=Asan; \
+	make
+
+prof:
+	mkdir -p $(BUILDDIR); \
+	cd $(BUILDDIR); \
+	cmake .. -DCMAKE_BUILD_TYPE=Profile; \
+	make
+
+afl:
+	@[ -n "$(AFL_PATH)" ] || { echo '$$AFL_PATH not set'; false; }
+	mkdir -p $(BUILDDIR)
+	cd $(BUILDDIR) && cmake .. -DCMAKE_C_COMPILER=$(AFL_PATH)/afl-gcc
+	make
+	$(AFL_PATH)/afl-fuzz \
+	    -i test/afl_test_cases \
+	    -o test/afl_results \
+	    -x test/afl_dictionary \
+	    -t 100 \
+	    -m none \
+	    $(CMARK)
+
+mingw:
+	mkdir -p $(MINGW_BUILDDIR); \
+	cd $(MINGW_BUILDDIR); \
+	cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain-mingw32.cmake -DCMAKE_INSTALL_PREFIX=$(MINGW_INSTALLDIR) ;\
+	make && make install
+
+man/man3/cmark.3: src/cmark.h | $(CMARK)
+	python man/make_man_page.py $< > $@ \
+
+archive:
+	git archive --prefix=$(RELEASE)/ -o $(RELEASE).tar.gz HEAD
+	git archive --prefix=$(RELEASE)/ -o $(RELEASE).zip HEAD
+
+clean:
+	rm -rf $(BUILDDIR) $(MINGW_BUILDDIR) $(MINGW_INSTALLDIR)
+
+# We include html_unescape.h in the repository, so this shouldn't
+# normally need to be generated.
+$(SRCDIR)/html_unescape.h: $(SRCDIR)/html_unescape.gperf
+	gperf -L ANSI-C -I -t -N find_entity -H hash_entity -K entity -C -l \
+		-F ',{0}' --null-strings -m5 -P -Q entity_pool $< > $@
+
+# We include case_fold_switch.inc in the repository, so this shouldn't
+# normally need to be generated.
+$(SRCDIR)/case_fold_switch.inc: $(DATADIR)/CaseFolding-3.2.0.txt
+	perl tools/mkcasefold.pl < $< > $@
+
+# We include scanners.c in the repository, so this shouldn't
+# normally need to be generated.
+$(SRCDIR)/scanners.c: $(SRCDIR)/scanners.re
+	@case "$$(re2c -v)" in \
+	    *\ 0.13.7*|*\ 0.14|*\ 0.14.1) \
+		echo "re2c versions 0.13.7 through 0.14.1 are known to produce buggy code."; \
+		echo "Try a version >= 0.14.2 or <= 0.13.6."; \
+		false; \
+		;; \
+	esac
+	re2c --case-insensitive -b -i --no-generation-date -o $@ $<
+
+update-spec:
+	curl 'https://raw.githubusercontent.com/jgm/CommonMark/master/spec.txt'\
+ > $(SPEC)
+
+test: $(SPEC) cmake_build
+	make -C $(BUILDDIR) test || (cat $(BUILDDIR)/Testing/Temporary/LastTest.log && exit 1)
+
+roundtrip_test: $(SPEC) cmake_build
+	python3 test/spec_tests.py --spec $< --prog test/roundtrip.sh
+
+$(ALLTESTS): $(SPEC)
+	python3 test/spec_tests.py --spec $< --dump-tests | python3 -c 'import json; import sys; tests = json.loads(sys.stdin.read()); print("\n".join([test["markdown"] for test in tests]))' > $@
+
+leakcheck: $(ALLTESTS)
+	rc=0; \
+	for format in html man xml commonmark; do \
+	  for opts in "" "--smart" "--normalize"; do \
+	     echo "cmark -t $$format $$opts" ; \
+	     cat $< | valgrind -q --leak-check=full --dsymutil=yes --error-exitcode=1 $(PROG) -t $$format $$opts >/dev/null || rc=1; \
+          done; \
+	done; \
+	exit $$rc
+
+fuzztest:
+	{ for i in `seq 1 10`; do \
+	  cat /dev/urandom | head -c $(FUZZCHARS) | iconv -f latin1 -t utf-8 | tee fuzz-$$i.txt | \
+		/usr/bin/env time -p $(PROG) >/dev/null && rm fuzz-$$i.txt ; \
+	done } 2>&1 | grep 'user\|abnormally'
+
+progit:
+	git clone https://github.com/progit/progit.git
+
+$(BENCHFILE): progit
+	echo "" > $@
+	for lang in ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw; do \
+		cat progit/$$lang/*/*.markdown >> $@; \
+	done
+
+bench: $(BENCHFILE)
+	{ sudo renice 99 $$$$; \
+	  for x in `seq 1 $(NUMRUNS)` ; do \
+	  /usr/bin/env time -p $(PROG) </dev/null >/dev/null ; \
+	  /usr/bin/env time -p $(PROG) $< >/dev/null ; \
+		  done \
+	} 2>&1  | grep 'real' | awk '{print $$2}' | python3 'bench/stats.py'
+
+astyle:
+	astyle --style=linux -t -p -r  'src/*.c' --exclude=scanners.c
+	astyle --style=linux -t -p -r  'src/*.h' --exclude=html_unescape.h
+
+operf: $(CMARK)
+	operf $< < $(BENCHINP) > /dev/null
+
+distclean: clean
+	-rm -rf *.dSYM
+	-rm -f README.html
+	-rm -rf $(BENCHFILE) $(ALLTESTS) progit

data/ext/commonmarker/cmark/Makefile.nmake

@@ -0,0 +1,36 @@
+SRCDIR=src
+DATADIR=data
+BUILDDIR=build
+INSTALLDIR=windows
+SPEC=test/spec.txt
+PROG=$(BUILDDIR)\src\cmark.exe
+GENERATOR=NMake Makefiles
+
+all: $(BUILDDIR)
+	@cd $(BUILDDIR) && $(MAKE) /nologo && cd ..
+
+$(BUILDDIR):
+	@-mkdir $(BUILDDIR) 2> nul
+	cd $(BUILDDIR) && \
+	cmake \
+	    -G "$(GENERATOR)" \
+	    -D CMAKE_BUILD_TYPE=$(BUILD_TYPE) \
+	    -D CMAKE_INSTALL_PREFIX=$(INSTALLDIR) \
+	    .. && \
+	cd ..
+
+install: all
+	@cd $(BUILDDIR) && $(MAKE) /nologo install && cd ..
+
+clean:
+	-rmdir /s /q $(BUILDDIR) $(MINGW_INSTALLDIR) 2> nul
+
+$(SRCDIR)\case_fold_switch.inc: $(DATADIR)\CaseFolding-3.2.0.txt
+	perl mkcasefold.pl < $? > $@
+
+test: $(SPEC) all
+	@cd $(BUILDDIR) && $(MAKE) /nologo test ARGS="-V" && cd ..
+
+distclean: clean
+	del /q src\scanners.c 2> nul
+	del /q spec.md spec.html 2> nul

data/ext/commonmarker/cmark/README.md

@@ -0,0 +1,169 @@
+cmark
+=====
+
+[![Build Status]](https://travis-ci.org/jgm/cmark)
+[![Windows Build Status]](https://ci.appveyor.com/project/jgm/cmark)
+
+`cmark` is the C reference implementation of [CommonMark], a
+rationalized version of Markdown syntax with a [spec][the spec].
+(For the JavaScript reference implementation, see
+[commonmark.js].)
+
+It provides a shared library (`libcmark`) with functions for parsing
+CommonMark documents to an abstract syntax tree (AST), manipulating
+the AST, and rendering the document to HTML, groff man,
+CommonMark, or an XML representation of the AST.  It also provides a
+command-line program (`cmark`) for parsing and rendering CommonMark
+documents.
+
+Advantages of this library:
+
+- **Portable.**  The library and program are written in standard
+  C99 and have no external dependencies.  It has been tested with
+  MSVC, gcc, tcc, and clang.
+
+- **Fast.** Performance is on par with the fastest existing
+  Markdown parser, [sundown]:  see the [benchmarks].
+
+- **Accurate.** The library passes all CommonMark conformance tests.
+
+- **Standardized.** The library can be expected to parse CommonMark
+  the same way as any other conforming parser.  So, for example,
+  you can use `commonmark.js` on the client to preview content that
+  will be rendered on the server using `cmark`.
+
+- **Robust.** The library has been extensively fuzz-tested using
+  american fuzzy lop.  The test suite includes pathological cases
+  that bring many other Markdown parsers to a crawl (for example,
+  thousands-deep nested bracketed text or block quotes).
+
+- **Flexible.** CommonMark input is parsed to an AST which can be
+  manipulated programatically prior to rendering.
+
+- **Multiple renderers.**  Output in HTML, groff man, CommonMark,
+  and a custom XML format is supported. And it is easy to write new
+  renderers to support other formats.
+
+- **Free.** BSD2-licensed.
+
+It is easy to use `libcmark` in python, lua, ruby, and other dynamic
+languages: see the `wrappers/` subdirectory for some simple examples.
+
+There are also libraries that wrap `libcmark` for
+[go](https://github.com/rhinoman/go-commonmark),
+[Haskell](http://hackage.haskell.org/package/cmark),
+and [Perl](https://metacpan.org/release/CommonMark).
+
+Installing
+----------
+
+Building the C program (`cmark`) and shared library (`libcmark`)
+requires [cmake].  If you modify `scanners.re`, then you will also
+need [re2c], which is used to generate `scanners.c` from
+`scanners.re`.  We have included a pre-generated `scanners.c` in
+the repository to reduce build dependencies.
+
+If you have GNU make, you can simply `make`, `make test`, and `make
+install`.  This calls [cmake] to create a `Makefile` in the `build`
+directory, then uses that `Makefile` to create the executable and
+library.  The binaries can be found in `build/src`.  The default
+installation prefix is `/usr/local`.  To change the installation
+prefix, pass the `INSTALL_PREFIX` variable if you run `make` for the
+first time: `make INSTALL_PREFIX=path`.
+
+For a more portable method, you can use [cmake] manually. [cmake] knows
+how to create build environments for many build systems.  For example,
+on FreeBSD:
+
+    mkdir build
+    cd build
+    cmake ..  # optionally: -DCMAKE_INSTALL_PREFIX=path
+    make      # executable will be created as build/src/cmark
+    make test
+    make install
+
+Or, to create Xcode project files on OSX:
+
+    mkdir build
+    cd build
+    cmake -G Xcode ..
+    open cmark.xcodeproj
+
+The GNU Makefile also provides a few other targets for developers.
+To run a benchmark:
+
+    make bench
+
+To run a "fuzz test" against ten long randomly generated inputs:
+
+    make fuzztest
+
+To run a test for memory leaks using `valgrind`:
+
+    make leakcheck
+
+To reformat source code using `astyle`:
+
+    make astyle
+
+To make a release tarball and zip archive:
+
+    make archive
+
+Installing (Windows)
+--------------------
+
+To compile with MSVC and NMAKE:
+
+    nmake
+
+You can cross-compile a Windows binary and dll on linux if you have the
+`mingw32` compiler:
+
+    make mingw
+
+The binaries will be in `build-mingw/windows/bin`.
+
+Usage
+-----
+
+Instructions for the use of the command line program and library can
+be found in the man pages in the `man` subdirectory.
+
+**A note on security:**
+This library does not attempt to sanitize link attributes or
+raw HTML.  If you use it in applications that accept
+untrusted user input, you must run the output through an HTML
+sanitizer to protect against
+[XSS attacks](http://en.wikipedia.org/wiki/Cross-site_scripting).
+
+Contributing
+------------
+
+There is a [forum for discussing
+CommonMark](http://talk.commonmark.org); you should use it instead of
+github issues for questions and possibly open-ended discussions.
+Use the [github issue tracker](http://github.com/jgm/CommonMark/issues)
+only for simple, clear, actionable issues.
+
+Authors
+-------
+
+John MacFarlane wrote the original library and program.
+The block parsing algorithm was worked out together with David
+Greenspan. Vicent Marti optimized the C implementation for
+performance, increasing its speed tenfold.  Kārlis Gaņģis helped
+work out a better parsing algorithm for links and emphasis,
+eliminating several worst-case performance issues.
+Nick Wellnhofer contributed many improvements, including
+most of the C library's API and its test harness.
+
+[sundown]: https://github.com/vmg/sundown
+[benchmarks]: benchmarks.md
+[the spec]: http://spec.commonmark.org
+[CommonMark]: http://commonmark.org
+[cmake]: http://www.cmake.org/download/
+[re2c]: http://re2c.org
+[commonmark.js]: https://github.com/jgm/commonmark.js
+[Build Status]: https://img.shields.io/travis/jgm/cmark/master.svg?style=flat
+[Windows Build Status]: https://ci.appveyor.com/api/projects/status/32r7s2skrgm9ubva?svg=true

data/ext/commonmarker/cmark/api_test/CMakeLists.txt

@@ -0,0 +1,25 @@
+add_executable(api_test
+  cplusplus.cpp
+  harness.c
+  harness.h
+  main.c
+)
+include_directories(
+  ${PROJECT_SOURCE_DIR}/src
+  ${PROJECT_BINARY_DIR}/src
+)
+target_link_libraries(api_test libcmark)
+
+# Compiler flags
+if(MSVC)
+  # Force to always compile with W4
+  if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
+    string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+  else()
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
+  endif()
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127 /wd4244 /wd4267 /wd4706 /wd4800 /D_CRT_SECURE_NO_WARNINGS")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /TP")
+elseif(CMAKE_COMPILER_IS_GNUCC OR "${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -std=c99 -pedantic")
+endif()

data/ext/commonmarker/cmark/api_test/cplusplus.cpp

@@ -0,0 +1,15 @@
+#include <cstdlib>
+
+#include "cmark.h"
+#include "cplusplus.h"
+#include "harness.h"
+
+void
+test_cplusplus(test_batch_runner *runner)
+{
+    static const char md[] = "paragraph\n";
+    char *html = cmark_markdown_to_html(md, sizeof(md) - 1, CMARK_OPT_DEFAULT);
+    STR_EQ(runner, html, "<p>paragraph</p>\n", "libcmark works with C++");
+    free(html);
+}
+

data/ext/commonmarker/cmark/api_test/cplusplus.h

@@ -0,0 +1,17 @@
+#ifndef CMARK_API_TEST_CPLUSPLUS_H
+#define CMARK_API_TEST_CPLUSPLUS_H
+
+#include "harness.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void
+test_cplusplus(test_batch_runner *runner);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/ext/commonmarker/cmark/api_test/harness.c

@@ -0,0 +1,102 @@
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "harness.h"
+
+test_batch_runner*
+test_batch_runner_new()
+{
+	return (test_batch_runner *)calloc(1, sizeof(test_batch_runner));
+}
+
+static void
+test_result(test_batch_runner *runner, int cond, const char *msg, va_list ap)
+{
+	++runner->test_num;
+
+	if (cond) {
+		++runner->num_passed;
+	}
+	else {
+		fprintf(stderr, "FAILED test %d: ", runner->test_num);
+		vfprintf(stderr, msg, ap);
+		fprintf(stderr, "\n");
+		++runner->num_failed;
+	}
+}
+
+void
+SKIP(test_batch_runner *runner, int num_tests)
+{
+	runner->test_num    += num_tests;
+	runner->num_skipped += num_tests;
+}
+
+void
+OK(test_batch_runner *runner, int cond, const char *msg, ...)
+{
+	va_list ap;
+	va_start(ap, msg);
+	test_result(runner, cond, msg, ap);
+	va_end(ap);
+}
+
+void
+INT_EQ(test_batch_runner *runner, int got, int expected, const char *msg, ...)
+{
+	int cond = got == expected;
+
+	va_list ap;
+	va_start(ap, msg);
+	test_result(runner, cond, msg, ap);
+	va_end(ap);
+
+	if (!cond) {
+		fprintf(stderr, "  Got:      %d\n", got);
+		fprintf(stderr, "  Expected: %d\n", expected);
+	}
+}
+
+void
+STR_EQ(test_batch_runner *runner, const char *got, const char *expected,
+       const char *msg, ...)
+{
+	int cond = strcmp(got, expected) == 0;
+
+	va_list ap;
+	va_start(ap, msg);
+	test_result(runner, cond, msg, ap);
+	va_end(ap);
+
+	if (!cond) {
+		fprintf(stderr, "  Got:      \"%s\"\n", got);
+		fprintf(stderr, "  Expected: \"%s\"\n", expected);
+	}
+}
+
+int
+test_ok(test_batch_runner *runner)
+{
+	return runner->num_failed == 0;
+}
+
+void
+test_print_summary(test_batch_runner *runner)
+{
+	int num_passed  = runner->num_passed;
+	int num_skipped = runner->num_skipped;
+	int num_failed  = runner->num_failed;
+
+	fprintf(stderr, "%d tests passed, %d failed, %d skipped\n",
+		num_passed, num_failed, num_skipped);
+
+	if (test_ok(runner)) {
+		fprintf(stderr, "PASS\n");
+	}
+	else {
+		fprintf(stderr, "FAIL\n");
+	}
+}
+

data/ext/commonmarker/cmark/api_test/harness.h

@@ -0,0 +1,42 @@
+#ifndef CMARK_API_TEST_HARNESS_H
+#define CMARK_API_TEST_HARNESS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+	int test_num;
+	int num_passed;
+	int num_failed;
+	int num_skipped;
+} test_batch_runner;
+
+test_batch_runner*
+test_batch_runner_new();
+
+void
+SKIP(test_batch_runner *runner, int num_tests);
+
+void
+OK(test_batch_runner *runner, int cond, const char *msg, ...);
+
+void
+INT_EQ(test_batch_runner *runner, int got, int expected, const char *msg, ...);
+
+void
+STR_EQ(test_batch_runner *runner, const char *got, const char *expected,
+       const char *msg, ...);
+
+int
+test_ok(test_batch_runner *runner);
+
+void
+test_print_summary(test_batch_runner *runner);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+