commonmarker 0.7.0 → 0.8.0

data/ext/commonmarker/cmark/build/src/cmake_install.cmake

@@ -43,11 +43,11 @@ endif()
 
 if(NOT CMAKE_INSTALL_COMPONENT OR "${CMAKE_INSTALL_COMPONENT}" STREQUAL "Unspecified")
   file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/lib" TYPE SHARED_LIBRARY FILES
-    "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/src/libcmark.0.24.1.dylib"
+    "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/src/libcmark.0.25.2.dylib"
     "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/src/libcmark.dylib"
     )
   foreach(file
-      "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libcmark.0.24.1.dylib"
+      "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libcmark.0.25.2.dylib"
       "$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/lib/libcmark.dylib"
       )
     if(EXISTS "${file}" AND

data/ext/commonmarker/cmark/build/src/cmark_version.h

@@ -1,7 +1,7 @@
 #ifndef CMARK_VERSION_H
 #define CMARK_VERSION_H
 
-#define CMARK_VERSION ((0 << 16) | (24 << 8)  | 1)
-#define CMARK_VERSION_STRING "0.24.1"
+#define CMARK_VERSION ((0 << 16) | (25 << 8)  | 2)
+#define CMARK_VERSION_STRING "0.25.2"
 
 #endif

data/ext/commonmarker/cmark/build/src/libcmark.pc

@@ -5,6 +5,6 @@ includedir=/usr/local/include
 
 Name: libcmark
 Description: CommonMark parsing, rendering, and manipulation
-Version: 0.24.1
+Version: 0.25.2
 Libs: -L${libdir} -lcmark
 Cflags: -I${includedir}

data/ext/commonmarker/cmark/build/testdir/CMakeFiles/CMakeDirectoryInformation.cmake

@@ -1,5 +1,5 @@
 # CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.3
+# Generated by "Unix Makefiles" Generator, CMake Version 3.5
 
 # Relative path conversion top directories.
 set(CMAKE_RELATIVE_PATH_TOP_SOURCE "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark")

data/ext/commonmarker/cmark/build/testdir/CTestTestfile.cmake

@@ -11,3 +11,4 @@ add_test(pathological_tests_library "/usr/local/bin/python3" "/Users/gjtorikian/
 add_test(spectest_executable "/usr/local/bin/python3" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec_tests.py" "--no-normalize" "--spec" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec.txt" "--program" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/testdir/../src/cmark")
 add_test(smartpuncttest_executable "/usr/local/bin/python3" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec_tests.py" "--no-normalize" "--spec" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/smart_punct.txt" "--program" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/testdir/../src/cmark --smart")
 add_test(roundtriptest_executable "/usr/local/bin/python3" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec_tests.py" "--no-normalize" "--spec" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec.txt" "--program" " /Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/testdir/../src/cmark")
+add_test(regressiontest_executable "/usr/local/bin/python3" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/spec_tests.py" "--no-normalize" "--spec" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/test/regression.txt" "--program" "/Users/gjtorikian/Development/commonmarker/ext/commonmarker/cmark/build/testdir/../src/cmark")

data/ext/commonmarker/cmark/build/testdir/Makefile

@@ -1,5 +1,5 @@
 # CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.3
+# Generated by "Unix Makefiles" Generator, CMake Version 3.5
 
 # Default target executed when no arguments are given to make.
 default_target: all
@@ -39,10 +39,10 @@ cmake_force:
 SHELL = /bin/sh
 
 # The CMake executable.
-CMAKE_COMMAND = /usr/local/Cellar/cmake/3.3.2/bin/cmake
+CMAKE_COMMAND = /usr/local/Cellar/cmake/3.5.1/bin/cmake
 
 # The command to remove a file.
-RM = /usr/local/Cellar/cmake/3.3.2/bin/cmake -E remove -f
+RM = /usr/local/Cellar/cmake/3.5.1/bin/cmake -E remove -f
 
 # Escaping for special characters.
 EQUALS = =
@@ -59,7 +59,7 @@ CMAKE_BINARY_DIR = /Users/gjtorikian/Development/commonmarker/ext/commonmarker/c
 # Special rule for the target test
 test:
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running tests..."
-	/usr/local/Cellar/cmake/3.3.2/bin/ctest --force-new-ctest-process $(ARGS)
+	/usr/local/Cellar/cmake/3.5.1/bin/ctest --force-new-ctest-process $(ARGS)
 .PHONY : test
 
 # Special rule for the target test
@@ -70,7 +70,7 @@ test/fast: test
 # Special rule for the target install/local
 install/local: preinstall
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..."
-	/usr/local/Cellar/cmake/3.3.2/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake
+	/usr/local/Cellar/cmake/3.5.1/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake
 .PHONY : install/local
 
 # Special rule for the target install/local
@@ -81,7 +81,7 @@ install/local/fast: install/local
 # Special rule for the target edit_cache
 edit_cache:
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake cache editor..."
-	/usr/local/Cellar/cmake/3.3.2/bin/ccmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
+	/usr/local/Cellar/cmake/3.5.1/bin/ccmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
 .PHONY : edit_cache
 
 # Special rule for the target edit_cache
@@ -92,7 +92,7 @@ edit_cache/fast: edit_cache
 # Special rule for the target rebuild_cache
 rebuild_cache:
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..."
-	/usr/local/Cellar/cmake/3.3.2/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
+	/usr/local/Cellar/cmake/3.5.1/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
 .PHONY : rebuild_cache
 
 # Special rule for the target rebuild_cache
@@ -113,19 +113,19 @@ list_install_components/fast: list_install_components
 # Special rule for the target install
 install: preinstall
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..."
-	/usr/local/Cellar/cmake/3.3.2/bin/cmake -P cmake_install.cmake
+	/usr/local/Cellar/cmake/3.5.1/bin/cmake -P cmake_install.cmake
 .PHONY : install
 
 # Special rule for the target install
 install/fast: preinstall/fast
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..."
-	/usr/local/Cellar/cmake/3.3.2/bin/cmake -P cmake_install.cmake
+	/usr/local/Cellar/cmake/3.5.1/bin/cmake -P cmake_install.cmake
 .PHONY : install/fast
 
 # Special rule for the target install/strip
 install/strip: preinstall
 	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..."
-	/usr/local/Cellar/cmake/3.3.2/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake
+	/usr/local/Cellar/cmake/3.5.1/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake
 .PHONY : install/strip
 
 # Special rule for the target install/strip

data/ext/commonmarker/cmark/changelog.txt

@@ -1,3 +1,75 @@
+[0.25.2]
+
+  * Open files in binary mode (#113, Nick Wellnhofer).  Now that cmark
+    supports different line endings, files must be openend in binary mode
+    on Windows.
+  * Reset `partially_consumed_tab` on every new line (#114, Nick Wellnhofer).
+  * Handle buffer split across a CRLF line ending (#117).  Adds an internal
+    field to the parser struct to keep track of `last_buffer_ended_with_cr`.
+    Added test.
+
+[0.25.1]
+
+  * Release with no code changes.  cmark version was mistakenly set to
+    0.25.1 in the 0.25.0 release (#112), so this release just
+    ensures that this will cause no confusion later.
+
+[0.25.0]
+
+  * Fixed tabs in indentation (#101).  This patch fixes S_advance_offset
+    so that it doesn't gobble a tab character when advancing less than the
+    width of a tab.
+  * Added partially_consumed_tab to parser.  This keeps track of when we
+    have gotten partway through a tab when consuming initial indentation.
+  * Simplified add_line (only need parser parameter).
+  * Properly handle partially consumed tab.  E.g. in
+
+        - foo
+
+         <TAB><TAB>bar
+
+    we should consume two spaces from the second tab, including two spaces
+    in the code block.
+  * Properly handle tabs with blockquotes and fenced blocks.
+  * Fixed handling of tabs in lists.
+  * Clarified logic in S_advance_offset.
+  * Use an assertion to check for in-range html_block_type.
+    It's a programming error if the type is out of range.
+  * Refactored S_processLines to make the logic easier to
+    understand, and added documentation (Mathieu Duponchelle).
+  * Removed unnecessary check for empty string_content.
+  * Factored out contains_inlines.
+  * Moved the cmake minimum version to top line of CMakeLists.txt
+    (tinysun212).
+  * Fix ctype(3) usage on NetBSD (Kamil Rytarowski).  We need to cast value
+    passed to isspace(3) to unsigned char to explicitly prevent possibly
+    undefined behavior.
+  * Compile in plain C mode with MSVC 12.0 or newer (Nick Wellnhofer).
+    Under MSVC, we used to compile in C++ mode to get some C99 features
+    like mixing declarations and code. With newer MSVC versions, it's
+    possible to build in plain C mode.
+  * Switched from "inline" to "CMARK_INLINE" (Nick Wellnhofer).
+    Newer MSVC versions support enough of C99 to be able to compile cmark
+    in plain C mode. Only the "inline" keyword is still unsupported.
+    We have to use "__inline" instead.
+  * Added include guards to config.h
+  * config.h.in - added compatibility snprintf, vsnprintf for MSVC.
+  * Replaced sprintf with snprintf (Marco Benelli).
+  * config.h: include stdio.h for _vscprintf etc.
+  * Include starg.h when needed in config.h.
+  * Removed an unnecessary C99-ism in buffer.c.  This helps compiling on
+    systems like luarocks that don't have all the cmake configuration
+    goodness (thanks to carlmartus).
+  * Don't use variable length arrays (Nick Wellnhofer).
+    They're not supported by MSVC.
+  * Test with multiple MSVC versions under Appveyor (Nick Wellnhofer).
+  * Fix installation dir of man-pages on NetBSD (Kamil Rytarowski).
+  * Fixed typo in cmark.h comments (Chris Eidhof).
+  * Clarify in man page that cmark_node_free frees a node's children too.
+  * Fixed documentation of --width in man page.
+  * Require re2c >= 1.14.2 (#102).
+  * Generated scanners.c with more recent re2c.
+
 [0.24.1]
 
   * Commonmark renderer:

data/ext/commonmarker/cmark/man/CMakeLists.txt

@@ -1,8 +1,10 @@
 if (NOT MSVC)
+
+include(GNUInstallDirs)
+
   install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/man1/cmark.1
-    DESTINATION share/man/man1)
+    DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)
 
   install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/man3/cmark.3
-    DESTINATION share/man/man3)
+    DESTINATION ${CMAKE_INSTALL_MANDIR}/man3)
 endif(NOT MSVC)
-

data/ext/commonmarker/cmark/man/man1/cmark.1

@@ -1,4 +1,4 @@
-.TH "cmark" "1" "November 30, 2014" "LOCAL" "General Commands Manual"
+.TH "cmark" "1" "March 24, 2016" "LOCAL" "General Commands Manual"
 .SH "NAME"
 \fBcmark\fR
 \- convert CommonMark formatted text to HTML
@@ -23,7 +23,7 @@ Specify output format (\f[C]html\f[], \f[C]man\f[], \f[C]xml\f[],
 .B \-\-width \f[I]WIDTH\f[]
 Specify a column width to which to wrap the output. For no wrapping, use
 the value 0 (the default).  This option currently only affects the
-commonmark renderer.
+commonmark, latex, and man renderers.
 .TP 12n
 .B \-\-sourcepos
 Include source position attribute.

data/ext/commonmarker/cmark/man/man3/cmark.3

@@ -1,4 +1,4 @@
-.TH cmark 3 "January 11, 2016" "LOCAL" "Library Functions Manual"
+.TH cmark 3 "March 24, 2016" "LOCAL" "Library Functions Manual"
 .SH
 NAME
 .PP
@@ -14,7 +14,7 @@ Simple Interface
 
 .PP
 Convert \f[I]text\f[] (assumed to be a UTF\-8 encoded string with length
-\f[I]len\f[] from CommonMark Markdown to HTML, returning a
+\f[I]len\f[]) from CommonMark Markdown to HTML, returning a
 null\-terminated, UTF\-8\-encoded string.
 
 .SS
@@ -109,7 +109,7 @@ to assign.
 \fIvoid\f[] \fBcmark_node_free\f[](\fIcmark_node *node\f[])
 
 .PP
-Frees the memory allocated for a node.
+Frees the memory allocated for a node and any children.
 
 .SS
 Tree Traversal
@@ -363,8 +363,8 @@ otherwise 0.
 \fIint\f[] \fBcmark_node_set_list_start\f[](\fIcmark_node *node\f[], \fIint start\f[])
 
 .PP
-Sets starting number of \f[I]node\f[], if it is an ordered list. Returns
-1 on success, 0 on failure.
+Sets starting number of \f[I]node\f[], if it is an ordered list.
+Returns 1 on success, 0 on failure.
 
 .PP
 \fIint\f[] \fBcmark_node_get_list_tight\f[](\fIcmark_node *node\f[])
@@ -389,8 +389,8 @@ Returns the info string from a fenced code block.
 \fIint\f[] \fBcmark_node_set_fence_info\f[](\fIcmark_node *node\f[], \fIconst char *info\f[])
 
 .PP
-Sets the info string in a fenced code block, returning 1 on success and
-0 on failure.
+Sets the info string in a fenced code block, returning 1 on success
+and 0 on failure.
 
 .PP
 \fIconst char *\f[] \fBcmark_node_get_url\f[](\fIcmark_node *node\f[])
@@ -517,8 +517,8 @@ Returns 1 on success, 0 on failure.
 \fIint\f[] \fBcmark_node_append_child\f[](\fIcmark_node *node\f[], \fIcmark_node *child\f[])
 
 .PP
-Adds \f[I]child\f[] to the end of the children of \f[I]node\f[]. Returns
-1 on success, 0 on failure.
+Adds \f[I]child\f[] to the end of the children of \f[I]node\f[].
+Returns 1 on success, 0 on failure.
 
 .PP
 \fIvoid\f[] \fBcmark_consolidate_text_nodes\f[](\fIcmark_node *root\f[])
@@ -543,7 +543,7 @@ Streaming interface:
 .nf
 \f[C]
 cmark_parser *parser = cmark_parser_new(CMARK_OPT_DEFAULT);
-FILE *fp = fopen("myfile.md", "r");
+FILE *fp = fopen("myfile.md", "rb");
 while ((bytes = fread(buffer, 1, sizeof(buffer), fp)) > 0) {
 	   cmark_parser_feed(parser, buffer, bytes);
 	   if (bytes < sizeof(buffer)) {

data/ext/commonmarker/cmark/src/CMakeLists.txt

@@ -169,8 +169,8 @@ elseif(CMAKE_COMPILER_IS_GNUCC OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -std=c99 -pedantic")
 endif()
 
-# Compile as C++ under MSVC
-if(MSVC)
+# Compile as C++ under MSVC older than 12.0
+if(MSVC AND MSVC_VERSION LESS 1800)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /TP")
 endif()
 

data/ext/commonmarker/cmark/src/blocks.c

@@ -1,3 +1,10 @@
+/**
+ * Block parsing implementation.
+ *
+ * For a high-level overview of the block parsing process,
+ * see http://spec.commonmark.org/0.24/#phase-1-block-structure
+ */
+
 #include <stdlib.h>
 #include <assert.h>
 #include <stdio.h>
@@ -17,12 +24,20 @@
 #define CODE_INDENT 4
 #define TAB_STOP 4
 
+#ifndef MIN
+#define MIN(x, y) ((x < y) ? x : y)
+#endif
+
 #define peek_at(i, n) (i)->data[n]
 
-static inline bool S_is_line_end_char(char c) {
+static CMARK_INLINE bool S_is_line_end_char(char c) {
   return (c == '\n' || c == '\r');
 }
 
+static CMARK_INLINE bool S_is_space_or_tab(char c) {
+  return (c == ' ' || c == '\t');
+}
+
 static void S_parser_feed(cmark_parser *parser, const unsigned char *buffer,
                           size_t len, bool eof);
 
@@ -70,10 +85,12 @@ cmark_parser *cmark_parser_new(int options) {
   parser->first_nonspace_column = 0;
   parser->indent = 0;
   parser->blank = false;
+  parser->partially_consumed_tab = false;
   parser->curline = line;
   parser->last_line_length = 0;
   parser->linebuf = buf;
   parser->options = options;
+  parser->last_buffer_ended_with_cr = false;
 
   return parser;
 }
@@ -110,7 +127,7 @@ static bool is_blank(cmark_strbuf *s, bufsize_t offset) {
   return true;
 }
 
-static inline bool can_contain(cmark_node_type parent_type,
+static CMARK_INLINE bool can_contain(cmark_node_type parent_type,
                                cmark_node_type child_type) {
   return (parent_type == CMARK_NODE_DOCUMENT ||
           parent_type == CMARK_NODE_BLOCK_QUOTE ||
@@ -118,15 +135,32 @@ static inline bool can_contain(cmark_node_type parent_type,
           (parent_type == CMARK_NODE_LIST && child_type == CMARK_NODE_ITEM));
 }
 
-static inline bool accepts_lines(cmark_node_type block_type) {
+static CMARK_INLINE bool accepts_lines(cmark_node_type block_type) {
   return (block_type == CMARK_NODE_PARAGRAPH ||
           block_type == CMARK_NODE_HEADING ||
           block_type == CMARK_NODE_CODE_BLOCK);
 }
 
-static void add_line(cmark_node *node, cmark_chunk *ch, bufsize_t offset) {
+static CMARK_INLINE bool contains_inlines(cmark_node_type block_type) {
+  return (block_type == CMARK_NODE_PARAGRAPH ||
+          block_type == CMARK_NODE_HEADING);
+}
+
+static void add_line(cmark_node *node, cmark_chunk *ch, cmark_parser *parser) {
+  int chars_to_tab;
+  int i;
   assert(node->open);
-  cmark_strbuf_put(&node->string_content, ch->data + offset, ch->len - offset);
+  if (parser->partially_consumed_tab) {
+    parser->offset += 1; // skip over tab
+    // add space characters:
+    chars_to_tab = TAB_STOP - (parser->column % TAB_STOP);
+    for (i = 0; i < chars_to_tab; i++) {
+      cmark_strbuf_putc(&node->string_content, ' ');
+    }
+  }
+  cmark_strbuf_put(&node->string_content,
+		  ch->data + parser->offset,
+		  ch->len - parser->offset);
 }
 
 static void remove_trailing_blank_lines(cmark_strbuf *ln) {
@@ -336,8 +370,7 @@ static void process_inlines(cmark_node *root, cmark_reference_map *refmap,
   while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
     cur = cmark_iter_get_node(iter);
     if (ev_type == CMARK_EVENT_ENTER) {
-      if (cur->type == CMARK_NODE_PARAGRAPH ||
-          cur->type == CMARK_NODE_HEADING) {
+      if (contains_inlines(cur->type)) {
         cmark_parse_inlines(cur, refmap, options);
       }
     }
@@ -474,6 +507,11 @@ static void S_parser_feed(cmark_parser *parser, const unsigned char *buffer,
   const unsigned char *end = buffer + len;
   static const uint8_t repl[] = {239, 191, 189};
 
+  if (parser->last_buffer_ended_with_cr && *buffer == '\n') {
+    // skip NL if last buffer ended with CR ; see #117
+    buffer++;
+  }
+  parser->last_buffer_ended_with_cr = false;
   while (buffer < end) {
     const unsigned char *eol;
     bufsize_t chunk_len;
@@ -514,8 +552,11 @@ static void S_parser_feed(cmark_parser *parser, const unsigned char *buffer,
 
     buffer += chunk_len;
     // skip over line ending characters:
-    if (buffer < end && *buffer == '\r')
+    if (buffer < end && *buffer == '\r') {
       buffer++;
+      if (buffer == end)
+	parser->last_buffer_ended_with_cr = true;
+    }
     if (buffer < end && *buffer == '\n')
       buffer++;
   }
@@ -532,13 +573,15 @@ static void chop_trailing_hashtags(cmark_chunk *ch) {
     n--;
 
   // Check for a space before the final #s:
-  if (n != orig_n && n >= 0 &&
-      (peek_at(ch, n) == ' ' || peek_at(ch, n) == '\t')) {
+  if (n != orig_n && n >= 0 && S_is_space_or_tab(peek_at(ch, n))) {
     ch->len = n;
     cmark_chunk_rtrim(ch);
   }
 }
 
+// Find first nonspace character from current offset, setting
+// parser->first_nonspace, parser->first_nonspace_column,
+// parser->indent, and parser->blank. Does not advance parser->offset.
 static void S_find_first_nonspace(cmark_parser *parser, cmark_chunk *input) {
   char c;
   int chars_to_tab = TAB_STOP - (parser->column % TAB_STOP);
@@ -566,17 +609,37 @@ static void S_find_first_nonspace(cmark_parser *parser, cmark_chunk *input) {
   parser->blank = S_is_line_end_char(peek_at(input, parser->first_nonspace));
 }
 
+// Advance parser->offset and parser->column.  parser->offset is the
+// byte position in input; parser->column is a virtual column number
+// that takes into account tabs. (Multibyte characters are not taken
+// into account, because the Markdown line prefixes we are interested in
+// analyzing are entirely ASCII.)  The count parameter indicates
+// how far to advance the offset.  If columns is true, then count
+// indicates a number of columns; otherwise, a number of bytes.
+// If advancing a certain number of columns partially consumes
+// a tab character, parser->partially_consumed_tab is set to true.
 static void S_advance_offset(cmark_parser *parser, cmark_chunk *input,
                              bufsize_t count, bool columns) {
   char c;
   int chars_to_tab;
+  int chars_to_advance;
   while (count > 0 && (c = peek_at(input, parser->offset))) {
     if (c == '\t') {
       chars_to_tab = TAB_STOP - (parser->column % TAB_STOP);
-      parser->column += chars_to_tab;
-      parser->offset += 1;
-      count -= (columns ? chars_to_tab : 1);
+      if (columns) {
+	parser->partially_consumed_tab = chars_to_tab > count;
+	chars_to_advance = MIN(count, chars_to_tab);
+	parser->column += chars_to_advance;
+        parser->offset += (parser->partially_consumed_tab ? 0 : 1);
+	count -= chars_to_advance;
+      } else {
+	parser->partially_consumed_tab = false;
+	parser->column += chars_to_tab;
+	parser->offset += 1;
+	count -= 1;
+      }
     } else {
+      parser->partially_consumed_tab = false;
       parser->offset += 1;
       parser->column += 1; // assume ascii; block starts are ascii
       count -= 1;
@@ -584,263 +647,314 @@ static void S_advance_offset(cmark_parser *parser, cmark_chunk *input,
   }
 }
 
-static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
-                           bufsize_t bytes) {
-  cmark_node *last_matched_container;
+static bool S_last_child_is_open(cmark_node *container) {
+  return container->last_child && container->last_child->open;
+}
+
+static bool parse_block_quote_prefix(cmark_parser *parser,
+                                cmark_chunk *input)
+{
+  bool res = false;
   bufsize_t matched = 0;
-  int lev = 0;
-  int i;
-  cmark_list *data = NULL;
-  bool all_matched = true;
-  cmark_node *container;
-  bool indented;
-  cmark_chunk input;
-  bool maybe_lazy;
 
-  if (parser->options & CMARK_OPT_VALIDATE_UTF8) {
-    cmark_utf8proc_check(parser->curline, buffer, bytes);
-  } else {
-    cmark_strbuf_put(parser->curline, buffer, bytes);
-  }
-  // ensure line ends with a newline:
-  if (bytes == 0 || !S_is_line_end_char(parser->curline->ptr[bytes - 1])) {
-    cmark_strbuf_putc(parser->curline, '\n');
-  }
-  parser->offset = 0;
-  parser->column = 0;
-  parser->blank = false;
+  matched =
+      parser->indent <= 3 && peek_at(input, parser->first_nonspace) == '>';
+  if (matched) {
 
-  input.data = parser->curline->ptr;
-  input.len = parser->curline->size;
+    S_advance_offset(parser, input, parser->indent + 1, true);
 
-  // container starts at the document root.
-  container = parser->root;
+    if (S_is_space_or_tab(peek_at(input, parser->offset))) {
+      S_advance_offset(parser, input, 1, true);
+    }
 
-  parser->line_number++;
+    res = true;
+  }
+  return res;
+}
 
-  // for each containing node, try to parse the associated line start.
-  // bail out on failure:  container will point to the last matching node.
+static bool parse_node_item_prefix(cmark_parser *parser,
+                              cmark_chunk *input,
+                              cmark_node *container)
+{
+  bool res = false;
+
+  if (parser->indent >=
+      container->as.list.marker_offset + container->as.list.padding) {
+    S_advance_offset(parser, input, container->as.list.marker_offset +
+                                         container->as.list.padding,
+                     true);
+    res = true;
+  } else if (parser->blank && container->first_child != NULL) {
+    // if container->first_child is NULL, then the opening line
+    // of the list item was blank after the list marker; in this
+    // case, we are done with the list item.
+    S_advance_offset(parser, input,
+                     parser->first_nonspace - parser->offset, false);
+    res = true;
+  }
+  return res;
+}
 
-  while (container->last_child && container->last_child->open) {
-    container = container->last_child;
+static bool parse_code_block_prefix(cmark_parser *parser,
+                               cmark_chunk *input,
+                               cmark_node *container,
+                               bool *should_continue)
+{
+  bool res = false;
+
+  if (!container->as.code.fenced) { // indented
+    if (parser->indent >= CODE_INDENT) {
+      S_advance_offset(parser, input, CODE_INDENT, true);
+      res = true;
+    } else if (parser->blank) {
+      S_advance_offset(parser, input,
+                       parser->first_nonspace - parser->offset, false);
+      res = true;
+    }
+  } else { // fenced
+    bufsize_t matched = 0;
 
-    S_find_first_nonspace(parser, &input);
+    if (parser->indent <= 3 && (peek_at(input, parser->first_nonspace) ==
+                                container->as.code.fence_char)) {
+      matched = scan_close_code_fence(input, parser->first_nonspace);
+    }
 
-    if (container->type == CMARK_NODE_BLOCK_QUOTE) {
-      matched =
-          parser->indent <= 3 && peek_at(&input, parser->first_nonspace) == '>';
-      if (matched) {
-        S_advance_offset(parser, &input, parser->indent + 1, true);
-        if (peek_at(&input, parser->offset) == ' ')
-          parser->offset++;
-      } else {
-        all_matched = false;
-      }
+    if (matched >= container->as.code.fence_length) {
+      // closing fence - and since we're at
+      // the end of a line, we can stop processing it:
+      *should_continue = false;
+      S_advance_offset(parser, input, matched, false);
+      parser->current = finalize(parser, container);
+    } else {
+      // skip opt. spaces of fence parser->offset
+      int i = container->as.code.fence_offset;
 
-    } else if (container->type == CMARK_NODE_ITEM) {
-      if (parser->indent >=
-          container->as.list.marker_offset + container->as.list.padding) {
-        S_advance_offset(parser, &input, container->as.list.marker_offset +
-                                             container->as.list.padding,
-                         true);
-      } else if (parser->blank && container->first_child != NULL) {
-        // if container->first_child is NULL, then the opening line
-        // of the list item was blank after the list marker; in this
-        // case, we are done with the list item.
-        S_advance_offset(parser, &input,
-                         parser->first_nonspace - parser->offset, false);
-      } else {
-        all_matched = false;
+      while (i > 0 && S_is_space_or_tab(peek_at(input, parser->offset))) {
+        S_advance_offset(parser, input, 1, true);
+        i--;
       }
+      res = true;
+    }
+  }
 
-    } else if (container->type == CMARK_NODE_CODE_BLOCK) {
+  return res;
+}
 
-      if (!container->as.code.fenced) { // indented
-        if (parser->indent >= CODE_INDENT) {
-          S_advance_offset(parser, &input, CODE_INDENT, true);
-        } else if (parser->blank) {
-          S_advance_offset(parser, &input,
-                           parser->first_nonspace - parser->offset, false);
-        } else {
-          all_matched = false;
-        }
-      } else { // fenced
-        matched = 0;
-        if (parser->indent <= 3 && (peek_at(&input, parser->first_nonspace) ==
-                                    container->as.code.fence_char)) {
-          matched = scan_close_code_fence(&input, parser->first_nonspace);
-        }
-        if (matched >= container->as.code.fence_length) {
-          // closing fence - and since we're at
-          // the end of a line, we can return:
-          all_matched = false;
-          S_advance_offset(parser, &input, matched, false);
-          parser->current = finalize(parser, container);
-          goto finished;
-        } else {
-          // skip opt. spaces of fence parser->offset
-          i = container->as.code.fence_offset;
-          while (i > 0 && peek_at(&input, parser->offset) == ' ') {
-            S_advance_offset(parser, &input, 1, false);
-            i--;
-          }
-        }
-      }
-    } else if (container->type == CMARK_NODE_HEADING) {
+static bool parse_html_block_prefix(cmark_parser *parser,
+                               cmark_node *container)
+{
+  bool res = false;
+  int html_block_type = container->as.html_block_type;
+
+  assert(html_block_type >= 1 && html_block_type <= 7);
+  switch (html_block_type) {
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+      // these types of blocks can accept blanks
+      res = true;
+      break;
+    case 6:
+    case 7:
+      res = !parser->blank;
+      break;
+  }
 
-      // a heading can never contain more than one line
-      all_matched = false;
+  return res;
+}
 
-    } else if (container->type == CMARK_NODE_HTML_BLOCK) {
+/**
+ * For each containing node, try to parse the associated line start.
+ *
+ * Will not close unmatched blocks, as we may have a lazy continuation
+ * line -> http://spec.commonmark.org/0.24/#lazy-continuation-line
+ *
+ * Returns: The last matching node, or NULL
+ */
+static cmark_node *check_open_blocks(cmark_parser *parser,
+                                     cmark_chunk *input,
+                                     bool *all_matched)
+{
+  bool should_continue = true;
+  *all_matched = false;
+  cmark_node *container = parser->root;
+  cmark_node_type cont_type;
+
+  while (S_last_child_is_open(container)) {
+    container = container->last_child;
+    cont_type = container->type;
 
-      switch (container->as.html_block_type) {
-      case 1:
-      case 2:
-      case 3:
-      case 4:
-      case 5:
-        // these types of blocks can accept blanks
+    S_find_first_nonspace(parser, input);
+
+    switch (cont_type) {
+      case CMARK_NODE_BLOCK_QUOTE:
+        if (!parse_block_quote_prefix(parser, input))
+          goto done;
         break;
-      case 6:
-      case 7:
-        if (parser->blank) {
-          all_matched = false;
-        }
+      case CMARK_NODE_ITEM:
+        if (!parse_node_item_prefix(parser, input, container))
+          goto done;
+        break;
+      case CMARK_NODE_CODE_BLOCK:
+        if (!parse_code_block_prefix(parser, input, container, &should_continue))
+          goto done;
+        break;
+      case CMARK_NODE_HEADING:
+        // a heading can never contain more than one line
+        goto done;
+      case CMARK_NODE_HTML_BLOCK:
+        if (!parse_html_block_prefix(parser, container))
+          goto done;
+        break;
+      case CMARK_NODE_PARAGRAPH:
+        if (parser->blank)
+          goto done;
         break;
       default:
-        fprintf(stderr, "Error (%s:%d): Unknown HTML block type %d\n", __FILE__,
-                __LINE__, container->as.html_block_type);
-        exit(1);
-      }
+        break;
+    }
+  }
 
-    } else if (container->type == CMARK_NODE_PARAGRAPH) {
+  *all_matched = true;
 
-      if (parser->blank) {
-        all_matched = false;
-      }
-    }
+done:
+  if (!*all_matched) {
+    container = container->parent; // back up to last matching node
+  }
 
-    if (!all_matched) {
-      container = container->parent; // back up to last matching node
-      break;
-    }
+  if (!should_continue) {
+    container = NULL;
   }
 
-  last_matched_container = container;
+  return container;
+}
 
-  // check to see if we've hit 2nd blank line, break out of list:
-  if (parser->blank && container->last_line_blank) {
-    break_out_of_lists(parser, &container);
-  }
+static void open_new_blocks(cmark_parser *parser,
+                            cmark_node **container,
+                            cmark_chunk *input,
+                            bool all_matched)
+{
+  bool indented;
+  cmark_list *data = NULL;
+  bool maybe_lazy = parser->current->type == CMARK_NODE_PARAGRAPH;
+  cmark_node_type cont_type = (*container)->type;
+  bufsize_t matched = 0;
+  int lev = 0;
+  bool save_partially_consumed_tab;
+  int save_offset;
+  int save_column;
 
-  maybe_lazy = parser->current->type == CMARK_NODE_PARAGRAPH;
-  // try new container starts:
-  while (container->type != CMARK_NODE_CODE_BLOCK &&
-         container->type != CMARK_NODE_HTML_BLOCK) {
+  while (cont_type != CMARK_NODE_CODE_BLOCK &&
+         cont_type != CMARK_NODE_HTML_BLOCK) {
 
-    S_find_first_nonspace(parser, &input);
+    S_find_first_nonspace(parser, input);
     indented = parser->indent >= CODE_INDENT;
 
-    if (!indented && peek_at(&input, parser->first_nonspace) == '>') {
+    if (!indented && peek_at(input, parser->first_nonspace) == '>') {
 
-      S_advance_offset(parser, &input,
+      S_advance_offset(parser, input,
                        parser->first_nonspace + 1 - parser->offset, false);
       // optional following character
-      if (peek_at(&input, parser->offset) == ' ')
-        S_advance_offset(parser, &input, 1, false);
-      container = add_child(parser, container, CMARK_NODE_BLOCK_QUOTE,
+      if (S_is_space_or_tab(peek_at(input, parser->offset))) {
+        S_advance_offset(parser, input, 1, true);
+      }
+      *container = add_child(parser, *container, CMARK_NODE_BLOCK_QUOTE,
                             parser->offset + 1);
 
     } else if (!indented && (matched = scan_atx_heading_start(
-                                 &input, parser->first_nonspace))) {
+                                 input, parser->first_nonspace))) {
+      bufsize_t hashpos;
+      int level = 0;
 
-      S_advance_offset(parser, &input,
+      S_advance_offset(parser, input,
                        parser->first_nonspace + matched - parser->offset,
                        false);
-      container =
-          add_child(parser, container, CMARK_NODE_HEADING, parser->offset + 1);
+      *container =
+          add_child(parser, *container, CMARK_NODE_HEADING, parser->offset + 1);
 
-      bufsize_t hashpos =
-          cmark_chunk_strchr(&input, '#', parser->first_nonspace);
-      int level = 0;
+      hashpos = cmark_chunk_strchr(input, '#', parser->first_nonspace);
 
-      while (peek_at(&input, hashpos) == '#') {
+      while (peek_at(input, hashpos) == '#') {
         level++;
         hashpos++;
       }
-      container->as.heading.level = level;
-      container->as.heading.setext = false;
 
-    } else if (!indented && (matched = scan_open_code_fence(
-                                 &input, parser->first_nonspace))) {
+      (*container)->as.heading.level = level;
+      (*container)->as.heading.setext = false;
 
-      container = add_child(parser, container, CMARK_NODE_CODE_BLOCK,
+    } else if (!indented && (matched = scan_open_code_fence(
+                                 input, parser->first_nonspace))) {
+      *container = add_child(parser, *container, CMARK_NODE_CODE_BLOCK,
                             parser->first_nonspace + 1);
-      container->as.code.fenced = true;
-      container->as.code.fence_char = peek_at(&input, parser->first_nonspace);
-      container->as.code.fence_length = matched;
-      container->as.code.fence_offset =
+      (*container)->as.code.fenced = true;
+      (*container)->as.code.fence_char = peek_at(input, parser->first_nonspace);
+      (*container)->as.code.fence_length = matched;
+      (*container)->as.code.fence_offset =
           (int8_t)(parser->first_nonspace - parser->offset);
-      container->as.code.info = cmark_chunk_literal("");
-      S_advance_offset(parser, &input,
+      (*container)->as.code.info = cmark_chunk_literal("");
+      S_advance_offset(parser, input,
                        parser->first_nonspace + matched - parser->offset,
                        false);
 
     } else if (!indented && ((matched = scan_html_block_start(
-                                  &input, parser->first_nonspace)) ||
-                             (container->type != CMARK_NODE_PARAGRAPH &&
+                                  input, parser->first_nonspace)) ||
+                             (cont_type != CMARK_NODE_PARAGRAPH &&
                               (matched = scan_html_block_start_7(
-                                   &input, parser->first_nonspace))))) {
-
-      container = add_child(parser, container, CMARK_NODE_HTML_BLOCK,
+                                   input, parser->first_nonspace))))) {
+      *container = add_child(parser, *container, CMARK_NODE_HTML_BLOCK,
                             parser->first_nonspace + 1);
-      container->as.html_block_type = matched;
+      (*container)->as.html_block_type = matched;
       // note, we don't adjust parser->offset because the tag is part of the
       // text
-
-    } else if (!indented && container->type == CMARK_NODE_PARAGRAPH &&
+    } else if (!indented && cont_type == CMARK_NODE_PARAGRAPH &&
                (lev =
-                    scan_setext_heading_line(&input, parser->first_nonspace))) {
-
-      container->type = CMARK_NODE_HEADING;
-      container->as.heading.level = lev;
-      container->as.heading.setext = true;
-      S_advance_offset(parser, &input, input.len - 1 - parser->offset, false);
-
+                    scan_setext_heading_line(input, parser->first_nonspace))) {
+      (*container)->type = CMARK_NODE_HEADING;
+      (*container)->as.heading.level = lev;
+      (*container)->as.heading.setext = true;
+      S_advance_offset(parser, input, input->len - 1 - parser->offset, false);
     } else if (!indented &&
-               !(container->type == CMARK_NODE_PARAGRAPH && !all_matched) &&
+               !(cont_type == CMARK_NODE_PARAGRAPH && !all_matched) &&
                (matched =
-                    scan_thematic_break(&input, parser->first_nonspace))) {
-
+                    scan_thematic_break(input, parser->first_nonspace))) {
       // it's only now that we know the line is not part of a setext heading:
-      container = add_child(parser, container, CMARK_NODE_THEMATIC_BREAK,
+      *container = add_child(parser, *container, CMARK_NODE_THEMATIC_BREAK,
                             parser->first_nonspace + 1);
-      S_advance_offset(parser, &input, input.len - 1 - parser->offset, false);
-
+      S_advance_offset(parser, input, input->len - 1 - parser->offset, false);
     } else if ((matched =
-                    parse_list_marker(&input, parser->first_nonspace, &data)) &&
-               (!indented || container->type == CMARK_NODE_LIST)) {
+                    parse_list_marker(input, parser->first_nonspace, &data)) &&
+               (!indented || cont_type == CMARK_NODE_LIST)) {
       // Note that we can have new list items starting with >= 4
       // spaces indent, as long as the list container is still open.
+      int i = 0;
 
       // compute padding:
-      S_advance_offset(parser, &input,
+      S_advance_offset(parser, input,
                        parser->first_nonspace + matched - parser->offset,
                        false);
-      i = 0;
-      while (i <= 5 && peek_at(&input, parser->offset + i) == ' ') {
-        i++;
+
+      save_partially_consumed_tab = parser->partially_consumed_tab;
+      save_offset = parser->offset;
+      save_column = parser->column;
+
+      while (parser->column - save_column <= 5 &&
+		S_is_space_or_tab(peek_at(input, parser->offset))) {
+        S_advance_offset(parser, input, 1, true);
       }
-      // i = number of spaces after marker, up to 5
-      if (i >= 5 || i < 1 ||
-          S_is_line_end_char(peek_at(&input, parser->offset))) {
+
+      i = parser->column - save_column;
+      if (i >= 5 || i < 1) {
         data->padding = matched + 1;
+	parser->offset = save_offset;
+	parser->column = save_column;
+	parser->partially_consumed_tab = save_partially_consumed_tab;
         if (i > 0) {
-          S_advance_offset(parser, &input, 1, false);
+          S_advance_offset(parser, input, 1, true);
         }
       } else {
         data->padding = matched + i;
-        S_advance_offset(parser, &input, i, true);
       }
 
       // check container; if it's a list, see if this list item
@@ -848,50 +962,57 @@ static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
 
       data->marker_offset = parser->indent;
 
-      if (container->type != CMARK_NODE_LIST ||
-          !lists_match(&container->as.list, data)) {
-        container = add_child(parser, container, CMARK_NODE_LIST,
+      if (cont_type != CMARK_NODE_LIST ||
+          !lists_match(&((*container)->as.list), data)) {
+        *container = add_child(parser, *container, CMARK_NODE_LIST,
                               parser->first_nonspace + 1);
 
-        memcpy(&container->as.list, data, sizeof(*data));
+        memcpy(&((*container)->as.list), data, sizeof(*data));
       }
 
       // add the list item
-      container = add_child(parser, container, CMARK_NODE_ITEM,
+      *container = add_child(parser, *container, CMARK_NODE_ITEM,
                             parser->first_nonspace + 1);
       /* TODO: static */
-      memcpy(&container->as.list, data, sizeof(*data));
+      memcpy(&((*container)->as.list), data, sizeof(*data));
       free(data);
-
     } else if (indented && !maybe_lazy && !parser->blank) {
-      S_advance_offset(parser, &input, CODE_INDENT, true);
-      container = add_child(parser, container, CMARK_NODE_CODE_BLOCK,
+      S_advance_offset(parser, input, CODE_INDENT, true);
+      *container = add_child(parser, *container, CMARK_NODE_CODE_BLOCK,
                             parser->offset + 1);
-      container->as.code.fenced = false;
-      container->as.code.fence_char = 0;
-      container->as.code.fence_length = 0;
-      container->as.code.fence_offset = 0;
-      container->as.code.info = cmark_chunk_literal("");
+      (*container)->as.code.fenced = false;
+      (*container)->as.code.fence_char = 0;
+      (*container)->as.code.fence_length = 0;
+      (*container)->as.code.fence_offset = 0;
+      (*container)->as.code.info = cmark_chunk_literal("");
 
     } else {
       break;
     }
 
-    if (accepts_lines(container->type)) {
+    if (accepts_lines((*container)->type)) {
       // if it's a line container, it can't contain other containers
       break;
     }
+
+    cont_type = (*container)->type;
     maybe_lazy = false;
   }
+}
 
+static void add_text_to_container (cmark_parser *parser,
+                                   cmark_node *container,
+                                   cmark_node *last_matched_container,
+                                   cmark_chunk *input)
+{
+  cmark_node *tmp;
   // what remains at parser->offset is a text line.  add the text to the
   // appropriate container.
 
-  S_find_first_nonspace(parser, &input);
+  S_find_first_nonspace(parser, input);
 
-  if (parser->blank && container->last_child) {
+  if (parser->blank && container->last_child)
     container->last_child->last_line_blank = true;
-  }
 
   // block quote lines are never blank as they start with >
   // and we don't count blanks in fenced code for purposes of tight/loose
@@ -906,61 +1027,61 @@ static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
        !(container->type == CMARK_NODE_ITEM && container->first_child == NULL &&
          container->start_line == parser->line_number));
 
-  cmark_node *cont = container;
-  while (cont->parent) {
-    cont->parent->last_line_blank = false;
-    cont = cont->parent;
+  tmp = container;
+  while (tmp->parent) {
+    tmp->parent->last_line_blank = false;
+    tmp = tmp->parent;
   }
 
+  // If the last line processed belonged to a paragraph node,
+  // and we didn't match all of the line prefixes for the open containers,
+  // and we didn't start any new containers,
+  // and the line isn't blank,
+  // then treat this as a "lazy continuation line" and add it to
+  // the open paragraph.
   if (parser->current != last_matched_container &&
-      container == last_matched_container && !parser->blank &&
-      parser->current->type == CMARK_NODE_PARAGRAPH &&
-      cmark_strbuf_len(&parser->current->string_content) > 0) {
-
-    add_line(parser->current, &input, parser->offset);
-
+      container == last_matched_container &&
+      !parser->blank &&
+      parser->current->type == CMARK_NODE_PARAGRAPH) {
+    add_line(parser->current, input, parser);
   } else { // not a lazy continuation
-
-    // finalize any blocks that were not matched and set cur to container:
+    // Finalize any blocks that were not matched and set cur to container:
     while (parser->current != last_matched_container) {
       parser->current = finalize(parser, parser->current);
       assert(parser->current != NULL);
     }
 
     if (container->type == CMARK_NODE_CODE_BLOCK) {
-
-      add_line(container, &input, parser->offset);
-
+      add_line(container, input, parser);
     } else if (container->type == CMARK_NODE_HTML_BLOCK) {
-
-      add_line(container, &input, parser->offset);
+      add_line(container, input, parser);
 
       int matches_end_condition;
       switch (container->as.html_block_type) {
       case 1:
         // </script>, </style>, </pre>
         matches_end_condition =
-            scan_html_block_end_1(&input, parser->first_nonspace);
+            scan_html_block_end_1(input, parser->first_nonspace);
         break;
       case 2:
         // -->
         matches_end_condition =
-            scan_html_block_end_2(&input, parser->first_nonspace);
+            scan_html_block_end_2(input, parser->first_nonspace);
         break;
       case 3:
         // ?>
         matches_end_condition =
-            scan_html_block_end_3(&input, parser->first_nonspace);
+            scan_html_block_end_3(input, parser->first_nonspace);
         break;
       case 4:
         // >
         matches_end_condition =
-            scan_html_block_end_4(&input, parser->first_nonspace);
+            scan_html_block_end_4(input, parser->first_nonspace);
         break;
       case 5:
         // ]]>
         matches_end_condition =
-            scan_html_block_end_5(&input, parser->first_nonspace);
+            scan_html_block_end_5(input, parser->first_nonspace);
         break;
       default:
         matches_end_condition = 0;
@@ -971,28 +1092,69 @@ static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
         container = finalize(parser, container);
         assert(parser->current != NULL);
       }
-
     } else if (parser->blank) {
-
       // ??? do nothing
-
     } else if (accepts_lines(container->type)) {
-
       if (container->type == CMARK_NODE_HEADING &&
           container->as.heading.setext == false) {
-        chop_trailing_hashtags(&input);
+        chop_trailing_hashtags(input);
       }
-      add_line(container, &input, parser->first_nonspace);
-
+      S_advance_offset(parser, input, parser->first_nonspace - parser->offset, false);
+      add_line(container, input, parser);
     } else {
       // create paragraph container for line
       container = add_child(parser, container, CMARK_NODE_PARAGRAPH,
                             parser->first_nonspace + 1);
-      add_line(container, &input, parser->first_nonspace);
+      S_advance_offset(parser, input, parser->first_nonspace - parser->offset, false);
+      add_line(container, input, parser);
     }
 
     parser->current = container;
   }
+}
+
+/* See http://spec.commonmark.org/0.24/#phase-1-block-structure */
+static void S_process_line(cmark_parser *parser, const unsigned char *buffer,
+                           bufsize_t bytes) {
+  cmark_node *last_matched_container;
+  bool all_matched = true;
+  cmark_node *container;
+  cmark_chunk input;
+
+  if (parser->options & CMARK_OPT_VALIDATE_UTF8)
+    cmark_utf8proc_check(parser->curline, buffer, bytes);
+  else
+    cmark_strbuf_put(parser->curline, buffer, bytes);
+
+  // ensure line ends with a newline:
+  if (bytes == 0 || !S_is_line_end_char(parser->curline->ptr[bytes - 1]))
+    cmark_strbuf_putc(parser->curline, '\n');
+
+  parser->offset = 0;
+  parser->column = 0;
+  parser->blank = false;
+  parser->partially_consumed_tab = false;
+
+  input.data = parser->curline->ptr;
+  input.len = parser->curline->size;
+
+  parser->line_number++;
+
+  last_matched_container = check_open_blocks(parser, &input, &all_matched);
+
+  if (!last_matched_container)
+    goto finished;
+
+  container = last_matched_container;
+
+  // check to see if we've hit 2nd blank line, break out of list:
+  if (parser->blank && container->last_line_blank)
+    break_out_of_lists(parser, &container);
+
+  open_new_blocks(parser, &container, &input, all_matched);
+
+  add_text_to_container(parser, container, last_matched_container, &input);
+
 finished:
   parser->last_line_length = input.len;
   if (parser->last_line_length &&