commonmarker 0.15.0 → 0.16.0

data/ext/commonmarker/cmark/src/scanners.h

@@ -14,6 +14,7 @@ bufsize_t _scan_scheme(const unsigned char *p);
 bufsize_t _scan_autolink_uri(const unsigned char *p);
 bufsize_t _scan_autolink_email(const unsigned char *p);
 bufsize_t _scan_html_tag(const unsigned char *p);
+bufsize_t _scan_liberal_html_tag(const unsigned char *p);
 bufsize_t _scan_html_block_start(const unsigned char *p);
 bufsize_t _scan_html_block_start_7(const unsigned char *p);
 bufsize_t _scan_html_block_end_1(const unsigned char *p);
@@ -35,6 +36,7 @@ bufsize_t _scan_dangerous_url(const unsigned char *p);
 #define scan_autolink_uri(c, n) _scan_at(&_scan_autolink_uri, c, n)
 #define scan_autolink_email(c, n) _scan_at(&_scan_autolink_email, c, n)
 #define scan_html_tag(c, n) _scan_at(&_scan_html_tag, c, n)
+#define scan_liberal_html_tag(c, n) _scan_at(&_scan_liberal_html_tag, c, n)
 #define scan_html_block_start(c, n) _scan_at(&_scan_html_block_start, c, n)
 #define scan_html_block_start_7(c, n) _scan_at(&_scan_html_block_start_7, c, n)
 #define scan_html_block_end_1(c, n) _scan_at(&_scan_html_block_end_1, c, n)

data/ext/commonmarker/cmark/src/scanners.re

@@ -122,6 +122,17 @@ bufsize_t _scan_html_tag(const unsigned char *p)
 */
 }
 
+// Try to (liberally) match an HTML tag after first <, returning num of chars matched.
+bufsize_t _scan_liberal_html_tag(const unsigned char *p)
+{
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+/*!re2c
+  .+ [>] { return (bufsize_t)(p - start); }
+  * { return 0; }
+*/
+}
+
 // Try to match an HTML block tag start line, returning
 // an integer code for the type of block (1-6, matching the spec).
 // #7 is handled by a separate function, below.

data/ext/commonmarker/cmark/src/syntax_extension.c

@@ -81,6 +81,11 @@ void cmark_syntax_extension_set_commonmark_render_func(cmark_syntax_extension *e
   extension->commonmark_render_func = func;
 }
 
+void cmark_syntax_extension_set_plaintext_render_func(cmark_syntax_extension *extension,
+                                                      cmark_common_render_func func) {
+  extension->plaintext_render_func = func;
+}
+
 void cmark_syntax_extension_set_latex_render_func(cmark_syntax_extension *extension,
                                                   cmark_common_render_func func) {
   extension->latex_render_func = func;

data/ext/commonmarker/cmark/src/syntax_extension.h

@@ -17,6 +17,7 @@ struct cmark_syntax_extension {
   cmark_can_contain_func          can_contain_func;
   cmark_contains_inlines_func     contains_inlines_func;
   cmark_common_render_func        commonmark_render_func;
+  cmark_common_render_func        plaintext_render_func;
   cmark_common_render_func        latex_render_func;
   cmark_common_render_func        man_render_func;
   cmark_html_render_func          html_render_func;

data/ext/commonmarker/commonmarker.c

@@ -570,6 +570,25 @@ static VALUE rb_render_commonmark(VALUE n, VALUE rb_options) {
   return ruby_cmark;
 }
 
+/* Internal: Convert the node to a plain textstring.
+ *
+ * Returns a {String}.
+ */
+static VALUE rb_render_plaintext(VALUE n, VALUE rb_options) {
+  int options;
+  cmark_node *node;
+  Check_Type(rb_options, T_FIXNUM);
+
+  options = FIX2INT(rb_options);
+  Data_Get_Struct(n, cmark_node, node);
+
+  char *text = cmark_render_plaintext(node, options, 120);
+  VALUE ruby_text = rb_str_new2(text);
+  free(text);
+
+  return ruby_text;
+}
+
 /*
  * Public: Inserts a node as a sibling after the current node.
  *
@@ -1080,6 +1099,7 @@ __attribute__((visibility("default"))) void Init_commonmarker() {
   rb_define_method(rb_mNode, "insert_before", rb_node_insert_before, 1);
   rb_define_method(rb_mNode, "_render_html", rb_render_html, 2);
   rb_define_method(rb_mNode, "_render_commonmark", rb_render_commonmark, 1);
+  rb_define_method(rb_mNode, "_render_plaintext", rb_render_plaintext, 1);
   rb_define_method(rb_mNode, "insert_after", rb_node_insert_after, 1);
   rb_define_method(rb_mNode, "prepend_child", rb_node_prepend_child, 1);
   rb_define_method(rb_mNode, "append_child", rb_node_append_child, 1);

data/lib/commonmarker/config.rb

@@ -6,9 +6,9 @@ module CommonMarker
       include Ruby::Enum
 
       define :DEFAULT, 0
-      define :NORMALIZE, (1 << 8)
       define :VALIDATE_UTF8, (1 << 9)
       define :SMART, (1 << 10)
+      define :LIBERAL_HTML_TAG, (1 << 12)
     end
 
     class Render

data/lib/commonmarker/node.rb

@@ -38,6 +38,16 @@ module CommonMarker
       _render_commonmark(opts).force_encoding('utf-8')
     end
 
+    # Public: Convert the node to a plain text string.
+    #
+    # options - A {Symbol} or {Array of Symbol}s indicating the render options
+    #
+    # Returns a {String}.
+    def to_plaintext(options = :DEFAULT)
+      opts = Config.process_options(options, :render)
+      _render_plaintext(opts).force_encoding('utf-8')
+    end
+
     # Public: Iterate over the children (if any) of the current pointer.
     def each(&block)
       return enum_for(:each) unless block_given?

data/lib/commonmarker/version.rb

@@ -1,3 +1,3 @@
 module CommonMarker
-  VERSION = '0.15.0'.freeze
+  VERSION = '0.16.0'.freeze
 end

data/test/test_node.rb

@@ -10,7 +10,7 @@ class TestNode < Minitest::Test
     @doc.walk do |node|
       nodes << node.type
     end
-    assert_equal [:document, :paragraph, :text, :emph, :text, :text, :text], nodes
+    assert_equal [:document, :paragraph, :text, :emph, :text, :text], nodes
   end
 
   def test_each
@@ -18,7 +18,7 @@ class TestNode < Minitest::Test
     @doc.first_child.each do |node|
       nodes << node.type
     end
-    assert_equal [:text, :emph, :text, :text], nodes
+    assert_equal [:text, :emph, :text], nodes
   end
 
   def test_deprecated_each_child
@@ -28,19 +28,19 @@ class TestNode < Minitest::Test
         nodes << node.type
       end
     end
-    assert_equal [:text, :emph, :text, :text], nodes
+    assert_equal [:text, :emph, :text], nodes
     assert_match /`each_child` is deprecated/, err
   end
 
   def test_select
     nodes = @doc.first_child.select { |node| node.type == :text }
     assert_equal CommonMarker::Node, nodes.first.class
-    assert_equal [:text, :text, :text], nodes.map(&:type)
+    assert_equal [:text, :text], nodes.map(&:type)
   end
 
   def test_map
     nodes = @doc.first_child.map(&:type)
-    assert_equal [:text, :emph, :text, :text], nodes
+    assert_equal [:text, :emph, :text], nodes
   end
 
   def test_insert_illegal

data/test/test_plaintext.rb

@@ -0,0 +1,44 @@
+require 'test_helper'
+
+class TestPlaintext < Minitest::Test
+  def setup
+    @markdown = <<-MD
+Hi *there*!
+
+1. I am a numeric list.
+2. I continue the list.
+* Suddenly, an unordered list!
+* What fun!
+
+Okay, _enough_.
+
+| a   | b   |
+| --- | --- |
+| c   | d   |
+    MD
+  end
+
+  def render_doc(doc)
+    CommonMarker.render_doc(doc, :DEFAULT, %i[table])
+  end
+
+  def test_to_commonmark
+    compare = render_doc(@markdown).to_plaintext
+
+    assert_equal <<-PLAINTEXT, compare
+Hi there!
+
+1.  I am a numeric list.
+2.  I continue the list.
+
+  - Suddenly, an unordered list!
+  - What fun!
+
+Okay, enough.
+
+| a | b |
+| --- | --- |
+| c | d |
+      PLAINTEXT
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: commonmarker
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Garen Torikian
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-22 00:00:00.000000000 Z
+date: 2017-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-enum
@@ -164,6 +164,139 @@ files:
 - ext/commonmarker/cmark/bench/statistics.py
 - ext/commonmarker/cmark/bench/stats.py
 - ext/commonmarker/cmark/benchmarks.md
+- ext/commonmarker/cmark/build/CMakeCache.txt
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CMakeCCompiler.cmake
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CMakeCXXCompiler.cmake
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CMakeDetermineCompilerABI_C.bin
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CMakeDetermineCompilerABI_CXX.bin
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CMakeSystem.cmake
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CompilerIdC/CMakeCCompilerId.c
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CompilerIdC/a.out
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CompilerIdCXX/CMakeCXXCompilerId.cpp
+- ext/commonmarker/cmark/build/CMakeFiles/3.8.0/CompilerIdCXX/a.out
+- ext/commonmarker/cmark/build/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/CMakeFiles/CMakeError.log
+- ext/commonmarker/cmark/build/CMakeFiles/CMakeOutput.log
+- ext/commonmarker/cmark/build/CMakeFiles/Makefile.cmake
+- ext/commonmarker/cmark/build/CMakeFiles/Makefile2
+- ext/commonmarker/cmark/build/CMakeFiles/TargetDirectories.txt
+- ext/commonmarker/cmark/build/CMakeFiles/cmake.check_cache
+- ext/commonmarker/cmark/build/CMakeFiles/feature_tests.bin
+- ext/commonmarker/cmark/build/CMakeFiles/feature_tests.c
+- ext/commonmarker/cmark/build/CMakeFiles/feature_tests.cxx
+- ext/commonmarker/cmark/build/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/CTestTestfile.cmake
+- ext/commonmarker/cmark/build/Makefile
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/build.make
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/depend.make
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/flags.make
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/link.txt
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/api_test.dir/progress.make
+- ext/commonmarker/cmark/build/api_test/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/api_test/Makefile
+- ext/commonmarker/cmark/build/api_test/cmake_install.cmake
+- ext/commonmarker/cmark/build/cmake_install.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/build.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/depend.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/flags.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/link.txt
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions.dir/progress.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/C.includecache
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/autolink.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/build.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/cmake_clean_target.cmake
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/core-extensions.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/depend.internal
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/depend.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/ext_scanners.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/flags.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/link.txt
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/progress.make
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/strikethrough.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/table.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/libcmark-gfmextensions_static.dir/tagfilter.c.o
+- ext/commonmarker/cmark/build/extensions/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/extensions/Makefile
+- ext/commonmarker/cmark/build/extensions/cmake_install.cmake
+- ext/commonmarker/cmark/build/extensions/cmarkextensions_export.h
+- ext/commonmarker/cmark/build/extensions/libcmark-gfmextensions.a
+- ext/commonmarker/cmark/build/man/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/man/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/man/Makefile
+- ext/commonmarker/cmark/build/man/cmake_install.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/Export/lib/cmake/cmark-release.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/Export/lib/cmake/cmark.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/build.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/depend.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/flags.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/link.txt
+- ext/commonmarker/cmark/build/src/CMakeFiles/cmark-gfm.dir/progress.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/build.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/depend.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/flags.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/link.txt
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm.dir/progress.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/C.includecache
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/DependInfo.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/arena.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/blocks.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/buffer.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/build.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/cmake_clean.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/cmake_clean_target.cmake
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/cmark.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/cmark_ctype.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/commonmark.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/depend.internal
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/depend.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/flags.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/houdini_href_e.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/houdini_html_e.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/houdini_html_u.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/html.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/inlines.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/iterator.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/latex.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/link.txt
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/linked_list.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/man.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/node.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/plaintext.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/plugin.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/progress.make
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/references.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/registry.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/render.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/scanners.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/syntax_extension.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/utf8.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/libcmark-gfm_static.dir/xml.c.o
+- ext/commonmarker/cmark/build/src/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/src/Makefile
+- ext/commonmarker/cmark/build/src/cmake_install.cmake
+- ext/commonmarker/cmark/build/src/cmark_export.h
+- ext/commonmarker/cmark/build/src/cmark_version.h
+- ext/commonmarker/cmark/build/src/config.h
+- ext/commonmarker/cmark/build/src/libcmark-gfm.a
+- ext/commonmarker/cmark/build/src/libcmark-gfm.pc
+- ext/commonmarker/cmark/build/testdir/CMakeFiles/CMakeDirectoryInformation.cmake
+- ext/commonmarker/cmark/build/testdir/CMakeFiles/progress.marks
+- ext/commonmarker/cmark/build/testdir/CTestTestfile.cmake
+- ext/commonmarker/cmark/build/testdir/Makefile
+- ext/commonmarker/cmark/build/testdir/cmake_install.cmake
 - ext/commonmarker/cmark/changelog.txt
 - ext/commonmarker/cmark/data/CaseFolding.txt
 - ext/commonmarker/cmark/extensions/CMakeLists.txt
@@ -219,6 +352,7 @@ files:
 - ext/commonmarker/cmark/src/node.c
 - ext/commonmarker/cmark/src/node.h
 - ext/commonmarker/cmark/src/parser.h
+- ext/commonmarker/cmark/src/plaintext.c
 - ext/commonmarker/cmark/src/plugin.c
 - ext/commonmarker/cmark/src/plugin.h
 - ext/commonmarker/cmark/src/references.c
@@ -331,6 +465,7 @@ files:
 - test/test_maliciousness.rb
 - test/test_node.rb
 - test/test_pathological_inputs.rb
+- test/test_plaintext.rb
 - test/test_renderer.rb
 - test/test_smartpunct.rb
 - test/test_spec.rb
@@ -377,6 +512,7 @@ test_files:
 - test/test_maliciousness.rb
 - test/test_node.rb
 - test/test_pathological_inputs.rb
+- test/test_plaintext.rb
 - test/test_renderer.rb
 - test/test_smartpunct.rb
 - test/test_spec.rb