asciidoctor 0.1.3 → 0.1.4

data/lib/asciidoctor/inline.rb

@@ -1,6 +1,9 @@
 module Asciidoctor
 # Public: Methods for managing inline elements in AsciiDoc block
 class Inline < AbstractNode
+  # Public: Get/Set the String name of the render template
+  attr_accessor :template_name
+
   # Public: Get the text of this inline element
   attr_reader :text
 
@@ -12,13 +15,10 @@ class Inline < AbstractNode
 
   def initialize(parent, context, text = nil, opts = {})
     super(parent, context)
+    @template_name = "inline_#{context}"
 
     @text = text 
 
-    #@id = opts[:id] if opts.has_key?(:id)
-    #@type = opts[:type] if opts.has_key?(:type)
-    #@target = opts[:target] if opts.has_key?(:target)
-
     @id = opts[:id]
     @type = opts[:type]
     @target = opts[:target]
@@ -29,7 +29,7 @@ class Inline < AbstractNode
   end
 
   def render
-    renderer.render("inline_#{@context}", self).chomp
+    renderer.render(@template_name, self).chomp
   end
 
 end

data/lib/asciidoctor/lexer.rb

@@ -77,10 +77,7 @@ class Lexer
     # special case, block title is not allowed above document title,
     # carry attributes over to the document body
     if block_attributes.has_key?('title')
-      document.clear_playback_attributes block_attributes
-      document.save_attributes
-      block_attributes['invalid-header'] = true
-      return block_attributes
+      return document.finalize_header block_attributes, false
     end
 
     # yep, document title logic in AsciiDoc is just insanity
@@ -101,7 +98,7 @@ class Lexer
         assigned_doctitle = doctitle
       end
       document.attributes['doctitle'] = section_title = doctitle
-      # QUESTION: should this be encapsulated in document?
+      # QUESTION: should the id assignment on Document be encapsulated in the Document class?
       if document.id.nil? && block_attributes.has_key?('id')
         document.id = block_attributes.delete('id')
       end
@@ -118,13 +115,50 @@ class Lexer
     if assigned_doctitle
       document.attributes['doctitle'] = assigned_doctitle
     end
+
+    # parse title and consume name section of manpage document
+    parse_manpage_header(reader, document) if document.doctype == 'manpage'
  
-    document.clear_playback_attributes block_attributes
-    document.save_attributes
- 
-    # NOTE these are the block-level attributes (not document attributes) that
+    # NOTE block_attributes are the block-level attributes (not document attributes) that
     # precede the first line of content (document title, first section or first block)
-    block_attributes
+    document.finalize_header block_attributes
+  end
+
+  # Public: Parses the manpage header of the AsciiDoc source read from the Reader
+  #
+  # returns Nothing
+  def self.parse_manpage_header(reader, document)
+    if (m = document.attributes['doctitle'].match(REGEXP[:mantitle_manvolnum]))
+      document.attributes['mantitle'] = document.sub_attributes(m[1].rstrip.downcase)
+      document.attributes['manvolnum'] = m[2].strip
+    else
+      warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed manpage title"
+    end
+
+    reader.skip_blank_lines
+
+    if is_next_line_section?(reader, {})
+      name_section = initialize_section(reader, document, {})
+      if name_section.level == 1
+        name_section_buffer = reader.read_lines_until(:break_on_blank_lines => true).join.tr_s("\n ", ' ')
+        if (m = name_section_buffer.match(REGEXP[:manname_manpurpose]))
+          document.attributes['manname'] = m[1] 
+          document.attributes['manpurpose'] = m[2] 
+          # TODO parse multiple man names
+
+          if document.backend == 'manpage'
+            document.attributes['docname'] = document.attributes['manname']
+            document.attributes['outfilesuffix'] = ".#{document.attributes['manvolnum']}"
+          end
+        else
+          warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed name section body"
+        end
+      else
+        warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section title must be at level 1"
+      end
+    else
+      warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section expected"
+    end
   end
 
   # Public: Return the next section from the Reader.
@@ -176,7 +210,7 @@ class Lexer
         (parent.has_header? || attributes.delete('invalid-header') || !is_next_line_section?(reader, attributes))
 
       if parent.has_header?
-        preamble = Block.new(parent, :preamble)
+        preamble = Block.new(parent, :preamble, :content_model => :compound)
         parent << preamble
       end
       section = parent
@@ -225,9 +259,9 @@ class Lexer
         doctype = parent.document.doctype
         if next_level > current_level || (section.is_a?(Document) && next_level == 0)
           if next_level == 0 && doctype != 'book'
-            puts "asciidoctor: ERROR: line #{reader.lineno + 1}: only book doctypes can contain level 0 sections"
+            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
           elsif !expected_next_levels.nil? && !expected_next_levels.include?(next_level)
-            puts "asciidoctor: WARNING: line #{reader.lineno + 1}: section title out of sequence: " +
+            warn "asciidoctor: WARNING: #{reader.line_info}: section title out of sequence: " +
                 "expected #{expected_next_levels.size > 1 ? 'levels' : 'level'} #{expected_next_levels * ' or '}, " +
                 "got level #{next_level}"
           end
@@ -236,7 +270,7 @@ class Lexer
           section << new_section
         else
           if next_level == 0 && doctype != 'book'
-            puts "asciidoctor: ERROR: line #{reader.lineno + 1}: only book doctypes can contain level 0 sections"
+            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
           end
           # close this section (and break out of the nesting) to begin a new one
           break
@@ -258,7 +292,7 @@ class Lexer
 
     if preamble && !preamble.blocks?
       # drop the preamble if it has no content
-      section.delete_at(0)
+      section.blocks.delete_at(0)
     end
 
     # The attributes returned here are orphaned attributes that fall at the end
@@ -305,7 +339,14 @@ class Lexer
     #parse_sections = options.fetch(:parse_sections, false)
 
     document = parent.document
-    parent_context = parent.is_a?(Block) ? parent.context : nil
+    if (extensions = document.extensions)
+      block_extensions = extensions.blocks?
+      macro_extensions = extensions.block_macros?
+    else
+      block_extensions = macro_extensions = false
+    end
+    #parent_context = parent.is_a?(Block) ? parent.context : nil
+    in_list = parent.is_a?(List)
     block = nil
     style = nil
     explicit_style = nil
@@ -320,19 +361,20 @@ class Lexer
       #  break
       end
 
-      # QUESTION introduce parsing context object?
-      this_line = reader.get_line
+      # QUESTION should we introduce a parsing context object?
+      this_line = reader.read_line
       delimited_block = false
       block_context = nil
+      cloaked_context = nil
       terminator = nil
       # QUESTION put this inside call to rekey attributes?
       if attributes[1]
-        style, explicit_style = parse_style_attribute(attributes)
+        style, explicit_style = parse_style_attribute(attributes, reader)
       end
 
       if delimited_blk_match = is_delimited_block?(this_line, true)
         delimited_block = true
-        block_context = delimited_blk_match.context
+        block_context = cloaked_context = delimited_blk_match.context
         terminator = delimited_blk_match.terminator
         if !style
           style = attributes['style'] = block_context.to_s
@@ -341,8 +383,10 @@ class Lexer
             block_context = style.to_sym
           elsif delimited_blk_match.masq.include?('admonition') && ADMONITION_STYLES.include?(style)
             block_context = :admonition
+          elsif block_extensions && extensions.processor_registered_for_block?(style, block_context)
+            block_context = style.to_sym
           else
-            puts "asciidoctor: WARNING: line #{reader.lineno}: invalid style for #{block_context} block: #{style}"
+            warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for #{block_context} block: #{style}"
             style = block_context.to_s
           end
         end
@@ -365,16 +409,17 @@ class Lexer
           end
 
           # process lines normally
-          if !text_only
+          unless text_only
+            first_char = Compliance.markdown_syntax ? this_line.lstrip[0..0] : this_line[0..0]
             # NOTE we're letting break lines (ruler, page_break, etc) have attributes
-            if (match = this_line.match(REGEXP[:break_line]))
-              block = Block.new(parent, BREAK_LINES[match[0][0..2]])
+            if BREAK_LINES.has_key?(first_char) && this_line.length > 3 &&
+                (match = this_line.match(Compliance.markdown_syntax ? REGEXP[:break_line_plus] : REGEXP[:break_line]))
+              block = Block.new(parent, BREAK_LINES[first_char], :content_model => :empty)
               break
 
-            # TODO make this a media_blk and handle image, video & audio
             elsif (match = this_line.match(REGEXP[:media_blk_macro]))
               blk_ctx = match[1].to_sym
-              block = Block.new(parent, blk_ctx)
+              block = Block.new(parent, blk_ctx, :content_model => :empty)
               if blk_ctx == :image
                 posattrs = ['alt', 'width', 'height']
               elsif blk_ctx == :video
@@ -394,53 +439,74 @@ class Lexer
                   :sub_input => true,
                   :sub_result => false,
                   :into => attributes)
-              target = block.sub_attributes(match[2])
+              target = block.sub_attributes(match[2], :attribute_missing => 'drop-line')
               if target.empty?
-                # drop the line if target resolves to nothing
-                return nil
+                if document.attributes.fetch('attribute-missing', COMPLIANCE[:attribute_missing]) == 'skip'
+                  # retain as unparsed
+                  return Block.new(parent, :paragraph, :source => [this_line.chomp])
+                else
+                  # drop the line if target resolves to nothing
+                  return nil
+                end
               end
 
               attributes['target'] = target
               block.title = attributes.delete('title') if attributes.has_key?('title')
               if blk_ctx == :image
                 document.register(:images, target)
-                attributes['alt'] ||= File.basename(target, File.extname(target))
+                attributes['alt'] ||= File.basename(target, File.extname(target)).tr('_-', ' ')
                 # QUESTION should video or audio have an auto-numbered caption?
                 block.assign_caption attributes.delete('caption'), 'figure'
               end
               break
 
             # NOTE we're letting the toc macro have attributes
-            elsif (match = this_line.match(REGEXP[:toc]))
-              block = Block.new(parent, :toc)
+            elsif first_char == 't' && (match = this_line.match(REGEXP[:toc]))
+              block = Block.new(parent, :toc, :content_model => :empty)
               block.parse_attributes(match[1], [], :sub_result => false, :into => attributes)
               break
 
+            elsif macro_extensions && (match = this_line.match(REGEXP[:generic_blk_macro])) &&
+                extensions.processor_registered_for_block_macro?(match[1])
+              name = match[1]
+              target = match[2]
+              raw_attributes = match[3]
+              processor = extensions.load_block_macro_processor name, document
+              unless raw_attributes.empty?
+                document.parse_attributes(raw_attributes, processor.options.fetch(:pos_attrs, []),
+                    :sub_input => true, :sub_result => false, :into => attributes)
+              end
+              if !(default_attrs = processor.options.fetch(:default_attrs, {})).empty?
+                default_attrs.each {|k, v| attributes[k] ||= v }
+              end
+              block = processor.process parent, target, attributes
+              return nil if block.nil?
+              break
             end
           end
 
           # haven't found anything yet, continue
           if (match = this_line.match(REGEXP[:colist]))
-            block = Block.new(parent, :colist)
+            block = List.new(parent, :colist)
             attributes['style'] = 'arabic'
-            items = []
-            block.buffer = items
             reader.unshift_line this_line
             expected_index = 1
             begin
               # might want to move this check to a validate method
               if match[1].to_i != expected_index
-                puts "asciidoctor: WARNING: line #{reader.lineno + 1}: callout list item index: expected #{expected_index} got #{match[1]}"
+                # FIXME this lineno - 2 hack means we need a proper look-behind cursor
+                warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: callout list item index: expected #{expected_index} got #{match[1]}"
               end
               list_item = next_list_item(reader, block, match)
               expected_index += 1
               if !list_item.nil?
-                items << list_item
-                coids = document.callouts.callout_ids(items.size)
+                block << list_item
+                coids = document.callouts.callout_ids(block.items.size)
                 if !coids.empty?
                   list_item.attributes['coids'] = coids
                 else
-                  puts "asciidoctor: WARNING: line #{reader.lineno}: no callouts refer to list item #{items.size}"
+                  # FIXME this lineno - 2 hack means we need a proper look-behind cursor
+                  warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: no callouts refer to list item #{block.items.size}"
                 end
               end
             end while reader.has_more_lines? && match = reader.peek_line.match(REGEXP[:colist])
@@ -457,10 +523,10 @@ class Lexer
             reader.unshift_line this_line
             block = next_outline_list(reader, :olist, parent)
             # QUESTION move this logic to next_outline_list?
-            if !(attributes.has_key? 'style') && !(block.attributes.has_key? 'style')
-              marker = block.buffer.first.marker
+            if !attributes['style'] && !block.attributes['style']
+              marker = block.items.first.marker
               if marker.start_with? '.'
-                # first one makes more sense, but second on is AsciiDoc-compliant
+                # first one makes more sense, but second one is AsciiDoc-compliant
                 #attributes['style'] = (ORDERED_LIST_STYLES[block.level - 1] || ORDERED_LIST_STYLES.first).to_s
                 attributes['style'] = (ORDERED_LIST_STYLES[marker.length - 1] || ORDERED_LIST_STYLES.first).to_s
               else
@@ -475,11 +541,12 @@ class Lexer
             block = next_labeled_list(reader, match, parent)
             break
 
-          elsif (style == 'float' || style == 'discrete') && is_section_title?(this_line, reader.peek_line)
+          elsif (style == 'float' || style == 'discrete') &&
+              is_section_title?(this_line, (Compliance.underline_style_section_titles ? reader.peek_line(true) : nil))
             reader.unshift_line this_line
             float_id, float_title, float_level, _ = parse_section_title(reader, document)
             float_id ||= attributes['id'] if attributes.has_key?('id')
-            block = Block.new(parent, :floating_title)
+            block = Block.new(parent, :floating_title, :content_model => :empty)
             if float_id.nil? || float_id.empty?
               # FIXME remove hack of creating throwaway Section to get at the generate_id method
               tmp_sect = Section.new(parent)
@@ -494,31 +561,40 @@ class Lexer
             break
 
           # FIXME create another set for "passthrough" styles
+          # FIXME make this more DRY!
           elsif !style.nil? && style != 'normal'
             if PARAGRAPH_STYLES.include?(style)
               block_context = style.to_sym
+              cloaked_context = :paragraph
               reader.unshift_line this_line
               # advance to block parsing =>
               break
             elsif ADMONITION_STYLES.include?(style)
               block_context = :admonition
+              cloaked_context = :paragraph
+              reader.unshift_line this_line
+              # advance to block parsing =>
+              break
+            elsif block_extensions && extensions.processor_registered_for_block?(style, :paragraph)
+              block_context = style.to_sym
+              cloaked_context = :paragraph
               reader.unshift_line this_line
               # advance to block parsing =>
               break
             else
-              puts "asciidoctor: WARNING: line #{reader.lineno}: invalid style for paragraph: #{style}"
+              warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for paragraph: #{style}"
               style = nil
               # continue to process paragraph
             end
           end
 
-          break_at_list = (skipped == 0 && parent_context.to_s.end_with?('list'))
+          break_at_list = (skipped == 0 && in_list)
 
           # a literal paragraph is contiguous lines starting at least one space
           if style != 'normal' && this_line.match(REGEXP[:lit_par])
-            # So we need to actually include this one in the grab_lines group
+            # So we need to actually include this one in the read_lines group
             reader.unshift_line this_line
-            buffer = reader.grab_lines_until(
+            lines = reader.read_lines_until(
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true,
                 :preserve_last_line => true) {|line|
@@ -530,20 +606,17 @@ class Lexer
               (COMPLIANCE[:block_terminates_paragraph] && (is_delimited_block?(line) || line.match(REGEXP[:attr_line])))
             }
 
-            reset_block_indent! buffer
+            reset_block_indent! lines
 
-            block = Block.new(parent, :literal, buffer)
+            block = Block.new(parent, :literal, :content_model => :verbatim, :source => lines, :attributes => attributes)
             # a literal gets special meaning inside of a definition list
-            if LIST_CONTEXTS.include?(parent_context)
-              attributes['options'] ||= []
-              # TODO this feels hacky, better way to distinguish from explicit literal block?
-              attributes['options'] << 'listparagraph'
-            end
+            # TODO this feels hacky, better way to distinguish from explicit literal block?
+            block.set_option('listparagraph') if in_list
 
           # a paragraph is contiguous nonblank/noncontinuation lines
           else
             reader.unshift_line this_line
-            buffer = reader.grab_lines_until(
+            lines = reader.read_lines_until(
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true,
                 :preserve_last_line => true,
@@ -559,23 +632,23 @@ class Lexer
             # NOTE we need this logic because we've asked the reader to skip
             # line comments, which may leave us w/ an empty buffer if those
             # were the only lines found
-            if buffer.empty?
-              # call get_line since the reader preserved the last line
-              reader.get_line
+            if lines.empty?
+              # call advance since the reader preserved the last line
+              reader.advance
               return nil
             end
 
-            catalog_inline_anchors(buffer.join, document)
+            catalog_inline_anchors(lines.join, document)
 
-            first_line = buffer.first
+            first_line = lines.first
             if !text_only && (admonition_match = first_line.match(REGEXP[:admonition_inline]))
-              buffer[0] = admonition_match.post_match.lstrip
-              block = Block.new(parent, :admonition, buffer)
+              lines[0] = admonition_match.post_match.lstrip
               attributes['style'] = admonition_match[1]
               attributes['name'] = admonition_name = admonition_match[1].downcase
               attributes['caption'] ||= document.attributes["#{admonition_name}-caption"]
-            elsif !text_only && COMPLIANCE[:markdown_syntax] && first_line.start_with?('> ')
-              buffer.map! {|line|
+              block = Block.new(parent, :admonition, :source => lines, :attributes => attributes)
+            elsif !text_only && Compliance.markdown_syntax && first_line.start_with?('> ')
+              lines.map! {|line|
                 if line.start_with?('> ')
                   line[2..-1]
                 elsif line.chomp == '>'
@@ -585,10 +658,10 @@ class Lexer
                 end
               }
 
-              if buffer.last.start_with?('-- ')
-                attribution, citetitle = buffer.pop[3..-1].split(', ')
-                buffer.pop while buffer.last.chomp.empty?
-                buffer[-1] = buffer.last.chomp
+              if lines.last.start_with?('-- ')
+                attribution, citetitle = lines.pop[3..-1].split(', ', 2)
+                lines.pop while lines.last.chomp.empty?
+                lines[-1] = lines.last.chomp
               else
                 attribution, citetitle = nil
               end
@@ -597,27 +670,34 @@ class Lexer
               attributes['citetitle'] = citetitle unless citetitle.nil?
               # NOTE will only detect headings that are floating titles (not section titles)
               # TODO could assume a floating title when inside a block context
-              block = build_block(:quote, :complex, false, parent, Reader.new(buffer), attributes)
-            elsif !text_only && buffer.size > 1 && first_line.start_with?('"') &&
-                buffer.last.start_with?('-- ') && buffer[-2].chomp.end_with?('"')
-              buffer[0] = first_line[1..-1]
-              attribution, citetitle = buffer.pop[3..-1].split(', ')
-              buffer.pop while buffer.last.chomp.empty?
-              buffer[-1] = buffer.last.chomp.chop
+              # FIXME Reader needs to be created w/ line info
+              block = build_block(:quote, :compound, false, parent, Reader.new(lines), attributes)
+            elsif !text_only && lines.size > 1 && first_line.start_with?('"') &&
+                lines.last.start_with?('-- ') && lines[-2].chomp.end_with?('"')
+              lines[0] = first_line[1..-1]
+              attribution, citetitle = lines.pop[3..-1].split(', ', 2)
+              lines.pop while lines.last.chomp.empty?
+              lines[-1] = lines.last.chomp.chop
               attributes['style'] = 'quote'
               attributes['attribution'] = attribution unless attribution.nil?
               attributes['citetitle'] = citetitle unless citetitle.nil?
-              block = Block.new(parent, :quote, buffer)
-              #block = Block.new(parent, :quote)
-              #block << Block.new(block, :paragraph, buffer)
+              block = Block.new(parent, :quote, :source => lines, :attributes => attributes)
+              #block = Block.new(parent, :quote, :content_model => :compound, :attributes => attributes)
+              #block << Block.new(block, :paragraph, :source => lines)
             else
-              # QUESTION is this necessary?
-              #if style == 'normal' && [' ', "\t"].include?(buffer.first[0..0])
-              #  # QUESTION should we only trim leading blanks?
-              #  buffer.map! &:lstrip
-              #end
+              # if [normal] is used over an indented paragraph, unindent it
+              if style == 'normal' && ((first_char = lines.first[0..0]) == ' ' || first_char == "\t")
+                first_line = lines.first
+                first_line_shifted = first_line.lstrip
+                indent = line_length(first_line) - line_length(first_line_shifted)
+                lines[0] = first_line_shifted
+                # QUESTION should we fix the rest of the lines, since in XML output it's insignificant?
+                lines.size.times do |i|
+                  lines[i] = lines[i][indent..-1] if i > 0
+                end
+              end
 
-              block = Block.new(parent, :paragraph, buffer)
+              block = Block.new(parent, :paragraph, :source => lines, :attributes => attributes)
             end
           end
 
@@ -636,21 +716,24 @@ class Lexer
         when :admonition
           attributes['name'] = admonition_name = style.downcase
           attributes['caption'] ||= document.attributes["#{admonition_name}-caption"]
-          block = build_block(block_context, :complex, terminator, parent, reader, attributes)
+          block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :comment
-          reader.grab_lines_until(:break_on_blank_lines => true, :chomp_last_line => false)
+          build_block(block_context, :skip, terminator, parent, reader, attributes)
           return nil
 
         when :example
-          block = build_block(block_context, :complex, terminator, parent, reader, attributes, {:supports_caption => true})
+          block = build_block(block_context, :compound, terminator, parent, reader, attributes, {:supports_caption => true})
 
         when :listing, :fenced_code, :source
           if block_context == :fenced_code
             style = attributes['style'] = 'source'
-            lang = this_line[3..-1].strip
-            attributes['language'] = lang unless lang.empty?
-            terminator = terminator[0..2] if terminator.length > 3
+            language, linenums = this_line[3...-1].split(',', 2)
+            if language && !(language = language.strip).empty?
+              attributes['language'] = language
+              attributes['linenums'] = '' if linenums && !linenums.strip.empty?
+            end
+            terminator = terminator[0..2]
           elsif block_context == :source
             AttributeList.rekey(attributes, [nil, 'language', 'linenums'])
           end
@@ -660,13 +743,14 @@ class Lexer
           block = build_block(block_context, :verbatim, terminator, parent, reader, attributes)
         
         when :pass
-          block = build_block(block_context, :simple, terminator, parent, reader, attributes)
+          block = build_block(block_context, :raw, terminator, parent, reader, attributes)
 
         when :open, :sidebar
-          block = build_block(block_context, :complex, terminator, parent, reader, attributes)
+          block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :table
-          block_reader = Reader.new reader.grab_lines_until(:terminator => terminator, :skip_line_comments => true)
+          cursor = reader.cursor
+          block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_line_comments => true), cursor
           case terminator[0..0]
             when ','
               attributes['format'] = 'csv'
@@ -677,11 +761,26 @@ class Lexer
 
         when :quote, :verse
           AttributeList.rekey(attributes, [nil, 'attribution', 'citetitle'])
-          block = build_block(block_context, (block_context == :verse ? :verbatim : :complex), terminator, parent, reader, attributes)
+          block = build_block(block_context, (block_context == :verse ? :verbatim : :compound), terminator, parent, reader, attributes)
 
         else
-          # this should only happen if there is a misconfiguration
-          raise "Unsupported block type #{block_context} at line #{reader.lineno}"
+          if block_extensions && extensions.processor_registered_for_block?(block_context, cloaked_context)
+            processor = extensions.load_block_processor block_context, document
+            
+            if (content_model = processor.options[:content_model]) != :skip
+              if !(pos_attrs = processor.options.fetch(:pos_attrs, [])).empty?
+                AttributeList.rekey(attributes, [nil].concat(pos_attrs))
+              end
+              if !(default_attrs = processor.options.fetch(:default_attrs, {})).empty?
+                default_attrs.each {|k, v| attributes[k] ||= v }
+              end
+            end
+            block = build_block(block_context, content_model, terminator, parent, reader, attributes, :processor => processor)
+            return nil if block.nil?
+          else
+            # this should only happen if there's a misconfiguration
+            raise "Unsupported block type #{block_context} at #{reader.line_info}"
+          end
         end
       end
     end
@@ -689,21 +788,34 @@ class Lexer
     # when looking for nested content, one or more line comments, comment
     # blocks or trailing attribute lists could leave us without a block,
     # so handle accordingly
-    # REVIEW we may no longer need this check
+    # REVIEW we may no longer need this nil check
     if !block.nil?
       # REVIEW seems like there is a better way to organize this wrap-up
       block.id      ||= attributes['id'] if attributes.has_key?('id')
       block.title     = attributes['title'] unless block.title?
       block.caption ||= attributes.delete('caption')
+      # TODO eventualy remove the style attribute from the attributes hash
+      #block.style     = attributes.delete('style')
+      block.style     = attributes['style']
       # AsciiDoc always use [id] as the reftext in HTML output,
       # but I'd like to do better in Asciidoctor
       if block.id && block.title? && !attributes.has_key?('reftext')
         document.register(:ids, [block.id, block.title])
       end
       block.update_attributes(attributes)
-
-      if block.context == :listing || block.context == :literal
-        catalog_callouts(block.buffer.join, document)
+      block.lock_in_subs
+
+      #if document.attributes.has_key? :pending_attribute_entries
+      #  document.attributes.delete(:pending_attribute_entries).each do |entry|
+      #    entry.save_to block.attributes
+      #  end
+      #end
+
+      if block.sub? :callouts
+        if !(catalog_callouts block.source, document)
+          # No need to look for callouts if they aren't there
+          block.remove_sub :callouts
+        end
       end
     end
 
@@ -713,48 +825,68 @@ class Lexer
   # Public: Determines whether this line is the start of any of the delimited blocks
   #
   # returns the match data if this line is the first line of a delimited block or nil if not
-  def self.is_delimited_block?(line, return_match_data = false)
-    line_len = line.length
-    # optimized for best performance
-    if line_len > 2
-      if line_len == 3
-        tip = line.chop
-        tl = 2
+  def self.is_delimited_block? line, return_match_data = false
+    # highly optimized for best performance
+    line_len = line.length - 1
+    return nil unless line_len > 1 && DELIMITED_BLOCK_LEADERS.include?(line[0..1])
+    line = line.chomp
+    # counts endline character in line length
+    if line_len == 2
+      tip = line
+      tl = 2
+    elsif line_len < 3
+      return nil
+    else
+      if line_len < 5
+        tip = line
+        tl = line_len
       else
         tip = line[0..3]
         tl = 4
+      end
 
-        if COMPLIANCE[:markdown_syntax]
-          # special case for fenced code blocks
-          tip_alt = tip.chop
-          if tip_alt == '```' || tip_alt == '~~~'
-            tip = tip_alt
-            tl = 3
+      # special case for fenced code blocks
+      if Compliance.markdown_syntax
+        tip_alt = tip.chop if tl == 4
+        if tip_alt == '```'
+          if tip.end_with? '`'
+            return nil
           end
+          tip = tip_alt
+          tl = 3
+        elsif tip_alt == '~~~'
+          if tip.end_with? '~'
+            return nil
+          end
+          tip = tip_alt
+          tl = 3
         end
       end
+    end
 
-      if DELIMITED_BLOCKS.has_key? tip
-        # if tip is the full line
-        if tl == line_len - 1
-          #return_match_data ? BlockMatchData.new(DELIMITED_BLOCKS[tip], tip, tip) : true
-          if return_match_data
-            context, masq = *DELIMITED_BLOCKS[tip]
-            BlockMatchData.new(context, masq, tip, tip)
-          else
-            true
-          end
-        elsif match = line.match(REGEXP[:any_blk])
-          #return_match_data ? BlockMatchData.new(DELIMITED_BLOCKS[tip], tip, match[0]) : true
-          if return_match_data
-            context, masq = *DELIMITED_BLOCKS[tip]
-            BlockMatchData.new(context, masq, tip, match[0])
-          else
-            true
-          end
+    if DELIMITED_BLOCKS.has_key? tip
+      # tip is the full line when delimiter is minimum length
+      if tl == 3 || tl == line_len
+        if return_match_data
+          context, masq = *DELIMITED_BLOCKS[tip]
+          BlockMatchData.new(context, masq, tip, tip)
+        else
+          true
+        end
+      elsif %(#{tip}#{tip[-1..-1] * (line_len - tl)}) == line
+        if return_match_data
+          context, masq = *DELIMITED_BLOCKS[tip]
+          BlockMatchData.new(context, masq, tip, line)
         else
-          nil
+          true
         end
+      #elsif match = line.match(REGEXP[:any_blk])
+      #  if return_match_data
+      #    context, masq = *DELIMITED_BLOCKS[tip]
+      #    BlockMatchData.new(context, masq, tip, match[0])
+      #  else
+      #    true
+      #  end
       else
         nil
       end
@@ -766,53 +898,93 @@ class Lexer
   # whether a block supports complex content should be a config setting
   # if terminator is false, that means the all the lines in the reader should be parsed
   # NOTE could invoke filter in here, before and after parsing
-  def self.build_block(block_context, content_type, terminator, parent, reader, attributes, options = {})
+  def self.build_block(block_context, content_model, terminator, parent, reader, attributes, options = {})
+    if content_model == :skip || content_model == :raw
+      skip_processing = content_model == :skip
+      parse_as_content_model = :simple
+    else
+      skip_processing = false
+      parse_as_content_model = content_model
+    end
+
     if terminator.nil?
-      if content_type == :verbatim
-        buffer = reader.grab_lines_until(:break_on_blank_lines => true, :break_on_list_continuation => true)
+      if parse_as_content_model == :verbatim
+        lines = reader.read_lines_until(:break_on_blank_lines => true, :break_on_list_continuation => true)
       else
-        buffer = reader.grab_lines_until(
+        content_model = :simple if content_model == :compound
+        lines = reader.read_lines_until(
             :break_on_blank_lines => true,
             :break_on_list_continuation => true,
             :preserve_last_line => true,
-            :skip_line_comments => true) {|line|
+            :skip_line_comments => true,
+            :skip_processing => skip_processing) {|line|
           COMPLIANCE[:block_terminates_paragraph] && (is_delimited_block?(line) || line.match(REGEXP[:attr_line]))
         }
-        # QUESTION check for empty buffer?
+        # QUESTION check for empty lines after grabbing lines for simple content model?
       end
-    elsif content_type != :complex
-      buffer = reader.grab_lines_until(:terminator => terminator, :chomp_last_line => true)
+      block_reader = nil
+    elsif parse_as_content_model != :compound
+      lines = reader.read_lines_until(:terminator => terminator, :chomp_last_line => true, :skip_processing => skip_processing)
+      block_reader = nil
+    # terminator is false when reader has already been prepared
     elsif terminator == false
-      buffer = nil
+      lines = nil
       block_reader = reader
     else
-      buffer = nil
-      block_reader = Reader.new reader.grab_lines_until(:terminator => terminator)
+      lines = nil
+      cursor = reader.cursor
+      block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_processing => skip_processing), cursor
     end
 
-    if content_type == :verbatim && attributes.has_key?('indent')
-      reset_block_indent! buffer, attributes['indent'].to_i
+    if content_model == :skip
+      attributes.clear
+      return lines
+    end
+
+    if content_model == :verbatim && attributes.has_key?('indent')
+      reset_block_indent! lines, attributes['indent'].to_i
+    end
+
+    if (processor = options[:processor])
+      attributes.delete('style')
+      processor.options[:content_model] = content_model
+      block = processor.process(parent, block_reader || Reader.new(lines), attributes)
+    else
+      block = Block.new(parent, block_context, :content_model => content_model, :attributes => attributes, :source => lines)
     end
 
-    block = Block.new(parent, block_context, buffer)
     # should supports_caption be necessary?
     if options.fetch(:supports_caption, false)
       block.title = attributes.delete('title') if attributes.has_key?('title')
       block.assign_caption attributes.delete('caption')
     end
 
-    if buffer.nil?
+    if content_model == :compound
       # we can look for blocks until there are no more lines (and not worry
       # about sections) since the reader is confined within the boundaries of a
       # delimited block
-      while block_reader.has_more_lines?
-        parsed_block = next_block(block_reader, block)
-        block.blocks << parsed_block unless parsed_block.nil?
-      end
+      parse_blocks block_reader, block
     end
     block
   end
 
+  # Public: Parse blocks from this reader until there are no more lines.
+  #
+  # This method calls Lexer#next_block until there are no more lines in the
+  # Reader. It does not consider sections because it's assumed the Reader only
+  # has lines which are within a delimited block region.
+  #
+  # reader - The Reader containing the lines to process
+  # parent - The parent Block to which to attach the parsed blocks
+  #
+  # Returns nothing.
+  def self.parse_blocks(reader, parent)
+    while reader.has_more_lines?
+      block = Lexer.next_block(reader, parent)
+      parent << block unless block.nil?
+    end
+  end
+
   # Internal: Parse and construct an outline list Block from the current position of the Reader
   #
   # reader    - The Reader from which to retrieve the outline list
@@ -821,39 +993,36 @@ class Lexer
   #
   # Returns the Block encapsulating the parsed outline (unordered or ordered) list
   def self.next_outline_list(reader, list_type, parent)
-    list_block = Block.new(parent, list_type)
-    items = []
-    list_block.buffer = items
+    list_block = List.new(parent, list_type)
     if parent.context == list_type
       list_block.level = parent.level + 1
     else
       list_block.level = 1
     end
-    Debug.debug { "Created #{list_type} block: #{list_block}" }
+    #Debug.debug { "Created #{list_type} block: #{list_block}" }
 
     while reader.has_more_lines? && (match = reader.peek_line.match(REGEXP[list_type]))
-
       marker = resolve_list_marker(list_type, match[1])
 
       # if we are moving to the next item, and the marker is different
       # determine if we are moving up or down in nesting
-      if items.size > 0 && marker != items.first.marker
+      if list_block.items? && marker != list_block.items.first.marker
         # assume list is nested by default, but then check to see if we are
         # popping out of a nested list by matching an ancestor's list marker
         this_item_level = list_block.level + 1
-        p = parent
-        while p.context == list_type
-          if marker == p.buffer.first.marker
-            this_item_level = p.level
+        ancestor = parent
+        while ancestor.context == list_type
+          if marker == ancestor.items.first.marker
+            this_item_level = ancestor.level
             break
           end
-          p = p.parent
+          ancestor = ancestor.parent
         end
       else
         this_item_level = list_block.level
       end
 
-      if items.size == 0 || this_item_level == list_block.level
+      if !list_block.items? || this_item_level == list_block.level
         list_item = next_list_item(reader, list_block, match)
       elsif this_item_level < list_block.level
         # leave this block
@@ -861,10 +1030,10 @@ class Lexer
       elsif this_item_level > list_block.level
         # If this next list level is down one from the
         # current Block's, append it to content of the current list item
-        items.last.blocks << next_block(reader, list_block)
+        list_block.items.last << next_block(reader, list_block)
       end
 
-      items << list_item unless list_item.nil?
+      list_block << list_item unless list_item.nil?
       list_item = nil
 
       reader.skip_blank_lines
@@ -878,14 +1047,21 @@ class Lexer
   # text     - The String of text in which to look for callouts
   # document - The current document on which the callouts are stored
   #
-  # Returns nothing
+  # Returns A Boolean indicating whether callouts were found
   def self.catalog_callouts(text, document)
-    text.scan(REGEXP[:callout_scan]) {
-      # alias match for Ruby 1.8.7 compat
-      m = $~
-      next if m[0].start_with? '\\'
-      document.callouts.register(m[1])
-    }
+    found = false
+    if text.include? '<'
+      text.scan(REGEXP[:callout_quick_scan]) {
+        # alias match for Ruby 1.8.7 compat
+        m = $~
+        if m[0][0..0] != '\\'
+          document.callouts.register(m[2])
+        end
+        # we have to mark as found even if it's escaped so it can be unescaped
+        found = true
+      }
+    end
+    found
   end
 
   # Internal: Catalog any inline anchors found in the text, but don't process them
@@ -917,18 +1093,25 @@ class Lexer
   #
   # Returns the Block encapsulating the parsed labeled list
   def self.next_labeled_list(reader, match, parent)
-    pairs = []
-    block = Block.new(parent, :dlist)
-    block.buffer = pairs
+    list_block = List.new(parent, :dlist)
+    previous_pair = nil
     # allows us to capture until we find a labeled item
     # that uses the same delimiter (::, :::, :::: or ;;)
     sibling_pattern = REGEXP[:dlist_siblings][match[2]]
 
     begin
-      pairs << next_list_item(reader, block, match, sibling_pattern)
+      term, item = next_list_item(reader, list_block, match, sibling_pattern)
+      if !previous_pair.nil? && previous_pair.last.nil?
+        previous_pair.pop
+        previous_pair[0] << term
+        previous_pair << item
+      else
+        # FIXME this misses the automatic parent assignment
+        list_block.items << (previous_pair = [[term], item])
+      end
     end while reader.has_more_lines? && match = reader.peek_line.match(sibling_pattern)
 
-    block
+    list_block
   end
 
   # Internal: Parse and construct the next ListItem for the current bulleted
@@ -957,25 +1140,46 @@ class Lexer
       has_text = !match[3].to_s.empty?
     else
       # Create list item using first line as the text of the list item
-      list_item = ListItem.new(list_block, match[2])
+      text = match[2]
+      checkbox = false
+      if list_type == :ulist && text.start_with?('[')
+        if text.start_with? '[ ] '
+          checkbox = true
+          checked = false
+          text = text[3..-1].lstrip
+        elsif text.start_with?('[*] ') || text.start_with?('[x] ')
+          checkbox = true
+          checked = true
+          text = text[3..-1].lstrip
+        end
+      end
+      list_item = ListItem.new(list_block, text)
+
+      if checkbox
+        # FIXME checklist never makes it into the options attribute
+        list_block.attributes['checklist-option'] = ''
+        list_item.attributes['checkbox'] = ''
+        list_item.attributes['checked'] = '' if checked
+      end
 
       if !sibling_trait
-        sibling_trait = resolve_list_marker(list_type, match[1], list_block.buffer.size, true)
+        sibling_trait = resolve_list_marker(list_type, match[1], list_block.items.size, true, reader)
       end
       list_item.marker = sibling_trait
       has_text = true
     end
 
     # first skip the line with the marker / term
-    reader.get_line
-    list_item_reader = Reader.new grab_lines_for_list_item(reader, list_type, sibling_trait, has_text)
+    reader.advance
+    cursor = reader.cursor
+    list_item_reader = Reader.new read_lines_for_list_item(reader, list_type, sibling_trait, has_text), cursor
     if list_item_reader.has_more_lines?
-      comment_lines = list_item_reader.consume_line_comments
+      comment_lines = list_item_reader.skip_line_comments
       subsequent_line = list_item_reader.peek_line
-      list_item_reader.unshift(*comment_lines) unless comment_lines.empty? 
+      list_item_reader.unshift_lines comment_lines unless comment_lines.empty? 
 
       if !subsequent_line.nil?
-        continuation_connects_first_block = (subsequent_line == "\n")
+        continuation_connects_first_block = (subsequent_line == ::Asciidoctor::EOL)
         # if there's no continuation connecting the first block, then
         # treat the lines as paragraph text (activated when has_text = false)
         if !continuation_connects_first_block && list_type != :dlist
@@ -995,7 +1199,7 @@ class Lexer
       # list
       while list_item_reader.has_more_lines?
         new_block = next_block(list_item_reader, list_block, {}, options)
-        list_item.blocks << new_block unless new_block.nil?
+        list_item << new_block unless new_block.nil?
       end
 
       list_item.fold_first(continuation_connects_first_block, content_adjacent)
@@ -1025,7 +1229,7 @@ class Lexer
   # has_text        - Whether the list item has text defined inline (always true except for labeled lists)
   #
   # Returns an Array of lines belonging to the current list item.
-  def self.grab_lines_for_list_item(reader, list_type, sibling_trait = nil, has_text = true)
+  def self.read_lines_for_list_item(reader, list_type, sibling_trait = nil, has_text = true)
     buffer = []
 
     # three states for continuation: :inactive, :active & :frozen
@@ -1043,7 +1247,7 @@ class Lexer
     detached_continuation = nil
 
     while reader.has_more_lines?
-      this_line = reader.get_line
+      this_line = reader.read_line
 
       # if we've arrived at a sibling item in this list, we've captured
       # the complete list item and can begin processing it
@@ -1057,7 +1261,7 @@ class Lexer
         if continuation == :inactive
           continuation = :active
           has_text = true
-          buffer[-1] = "\n" unless within_nested_list
+          buffer[-1] = ::Asciidoctor::EOL unless within_nested_list
         end
 
         # dealing with adjacent list continuations (which is really a syntax error)
@@ -1078,7 +1282,7 @@ class Lexer
           buffer << this_line
           # grab all the lines in the block, leaving the delimiters in place
           # we're being more strict here about the terminator, but I think that's a good thing
-          buffer.concat reader.grab_lines_until(:terminator => match.terminator, :grab_last_line => true)
+          buffer.concat reader.read_lines_until(:terminator => match.terminator, :read_last_line => true)
           continuation = :inactive
         else
           break
@@ -1095,7 +1299,7 @@ class Lexer
           # list item will throw off the exit from it
           if this_line.match(REGEXP[:lit_par])
             reader.unshift_line this_line
-            buffer.concat reader.grab_lines_until(
+            buffer.concat reader.read_lines_until(
                 :preserve_last_line => true,
                 :break_on_blank_lines => true,
                 :break_on_list_continuation => true) {|line|
@@ -1122,7 +1326,7 @@ class Lexer
           # advance to the next line of content
           if this_line.chomp.empty?
             reader.skip_blank_lines
-            this_line = reader.get_line 
+            this_line = reader.read_line 
             # if we hit eof or a sibling, stop reading
             break if this_line.nil? || is_sibling_list_item?(this_line, list_type, sibling_trait)
           end
@@ -1138,7 +1342,7 @@ class Lexer
               # slurp up any literal paragraph offset by blank lines
               if this_line.match(REGEXP[:lit_par])
                 reader.unshift_line this_line
-                buffer.concat reader.grab_lines_until(
+                buffer.concat reader.read_lines_until(
                     :preserve_last_line => true,
                     :break_on_blank_lines => true,
                     :break_on_list_continuation => true) {|line|
@@ -1211,13 +1415,15 @@ class Lexer
   # parent     - the parent Section or Document of this Section
   # attributes - a Hash of attributes to assign to this section (default: {})
   def self.initialize_section(reader, parent, attributes = {})
-    section = Section.new parent
-    section.id, section.title, section.level, _ = parse_section_title(reader, section.document)
+    document = parent.document
+    sect_id, sect_title, sect_level, _ = parse_section_title(reader, document)
+    section = Section.new parent, sect_level, document.attributes.has_key?('numbered')
+    section.id = sect_id
+    section.title = sect_title
     # parse style, id and role from first positional attribute
     if attributes[1]
-      section.sectname, _ = parse_style_attribute(attributes)
+      section.sectname, _ = parse_style_attribute(attributes, reader)
       section.special = true
-      document = parent.document
       # HACK needs to be refactored so it's driven by config
       if section.sectname == 'abstract' && document.doctype == 'book'
         section.sectname = "sect1"
@@ -1231,6 +1437,9 @@ class Lexer
         section.caption = "#{document.attributes['appendix-caption']} #{number}: "
         Document::AttributeEntry.new('appendix-number', number).save_to(attributes)
       end
+    elsif sect_title.downcase == 'synopsis' && document.doctype == 'manpage'
+      section.special = true
+      section.sectname = 'synopsis'
     else
       section.sectname = "sect#{section.level}"
     end
@@ -1276,7 +1485,7 @@ class Lexer
   def self.is_next_line_section?(reader, attributes)
     return false if !(val = attributes[1]).nil? && ['float', 'discrete'].include?(val)
     return false if !reader.has_more_lines?
-    is_section_title?(*reader.peek_lines(2))
+    Compliance.underline_style_section_titles ? is_section_title?(*reader.peek_lines(2)) : is_section_title?(reader.peek_line)
   end
 
   # Internal: Convenience API for checking if the next line on the Reader is the document title
@@ -1299,7 +1508,7 @@ class Lexer
   def self.is_section_title?(line1, line2 = nil)
     if (level = is_single_line_section_title?(line1))
       level
-    elsif (level = is_two_line_section_title?(line1, line2))
+    elsif line2 && (level = is_two_line_section_title?(line1, line2))
       level
     else
       false
@@ -1307,7 +1516,8 @@ class Lexer
   end
 
   def self.is_single_line_section_title?(line1)
-    if !line1.nil? && (line1.start_with?('=') || (COMPLIANCE[:markdown_syntax] && line1.start_with?('#'))) &&
+    first_char = line1.nil? ? nil : line1[0..0]
+    if (first_char == '=' || (Compliance.markdown_syntax && first_char == '#')) &&
         (match = line1.match(REGEXP[:section_title]))
       single_line_section_level match[1]
     else
@@ -1319,7 +1529,7 @@ class Lexer
     if !line1.nil? && !line2.nil? && SECTION_LEVELS.has_key?(line2[0..0]) &&
         line2.match(REGEXP[:section_underline]) && line1.match(REGEXP[:section_name]) &&
         # chomp so that a (non-visible) endline does not impact calculation
-        (line1.chomp.size - line2.chomp.size).abs <= 1
+        (line_length(line1) - line_length(line2)).abs <= 1
       section_level line2
     else
       false
@@ -1370,23 +1580,24 @@ class Lexer
   #--
   # NOTE for efficiency, we don't reuse methods that check for a section title
   def self.parse_section_title(reader, document)
-    line1 = reader.get_line
+    line1 = reader.read_line
     sect_id = nil
     sect_title = nil
     sect_level = -1
     single_line = true
 
-    if (line1.start_with?('=') || (COMPLIANCE[:markdown_syntax] && line1.start_with?('#'))) &&
+    first_char = line1[0..0]
+    if (first_char == '=' || (Compliance.markdown_syntax && first_char == '#')) &&
         (match = line1.match(REGEXP[:section_title]))
       sect_id = match[3]
       sect_title = match[2]
       sect_level = single_line_section_level match[1]
-    else
-      line2 = reader.peek_line
+    elsif Compliance.underline_style_section_titles
+      line2 = reader.peek_line true
       if !line2.nil? && SECTION_LEVELS.has_key?(line2[0..0]) && line2.match(REGEXP[:section_underline]) &&
         (name_match = line1.match(REGEXP[:section_name])) &&
         # chomp so that a (non-visible) endline does not impact calculation
-        (line1.chomp.size - line2.chomp.size).abs <= 1
+        (line_length(line1) - line_length(line2)).abs <= 1
         if anchor_match = name_match[1].match(REGEXP[:anchor_embedded]) 
           sect_id = anchor_match[2]
           sect_title = anchor_match[1]
@@ -1395,7 +1606,7 @@ class Lexer
         end
         sect_level = section_level line2
         single_line = false
-        reader.get_line
+        reader.advance
       end
     end
     if sect_level >= 0
@@ -1404,6 +1615,15 @@ class Lexer
     [sect_id, sect_title, sect_level, single_line]
   end
 
+  # Public: Calculate the number of unicode characters in the line, excluding the endline
+  #
+  # line - the String to calculate
+  #
+  # returns the number of unicode characters in the line
+  def self.line_length(line)
+    FORCE_UNICODE_LINE_LENGTH ? line.chomp.scan(/./u).length : line.chomp.length
+  end
+
   # Public: Consume and parse the two header lines (line 1 = author info, line 2 = revision info).
   #
   # Returns the Hash of header metadata. If a Document object is supplied, the metadata
@@ -1425,8 +1645,8 @@ class Lexer
     implicit_author = nil
     implicit_authors = nil
 
-    if reader.has_more_lines? && !reader.peek_line.chomp.empty?
-      author_metadata = process_authors reader.get_line
+    if reader.has_more_lines? && !reader.next_line_empty?
+      author_metadata = process_authors reader.read_line
 
       unless author_metadata.empty?
         # apply header subs and assign to document
@@ -1449,8 +1669,8 @@ class Lexer
 
       rev_metadata = {}
 
-      if reader.has_more_lines? && !reader.peek_line.chomp.empty?
-        rev_line = reader.get_line 
+      if reader.has_more_lines? && !reader.next_line_empty?
+        rev_line = reader.read_line 
         if match = rev_line.match(REGEXP[:revision_info])
           rev_metadata['revdate'] = match[2].strip
           rev_metadata['revnumber'] = match[1].rstrip unless match[1].nil?
@@ -1640,7 +1860,7 @@ class Lexer
     next_line = reader.peek_line
     if (commentish = next_line.start_with?('//')) && (match = next_line.match(REGEXP[:comment_blk]))
       terminator = match[0]
-      reader.grab_lines_until(:skip_first_line => true, :preserve_last_line => true, :terminator => terminator, :preprocess => false)
+      reader.read_lines_until(:skip_first_line => true, :preserve_last_line => true, :terminator => terminator, :skip_processing => true)
     elsif commentish && next_line.match(REGEXP[:comment])
       # do nothing, we'll skip it
     elsif !options[:text] && (match = next_line.match(REGEXP[:attr_entry]))
@@ -1648,7 +1868,7 @@ class Lexer
     elsif match = next_line.match(REGEXP[:anchor])
       id, reftext = match[1].split(',')
       attributes['id'] = id
-      # AsciiDoc always use [id] as the reftext in HTML output,
+      # AsciiDoc always uses [id] as the reftext in HTML output,
       # but I'd like to do better in Asciidoctor
       #parent.document.register(:ids, id)
       if reftext
@@ -1716,6 +1936,10 @@ class Lexer
       # a nil value signals the attribute should be deleted (undefined)
       value = nil
       name = name.chop
+    elsif name.start_with?('!')
+      # a nil value signals the attribute should be deleted (undefined)
+      value = nil
+      name = name[1..-1]
     end
 
     name = sanitize_attribute_name(name)
@@ -1747,9 +1971,9 @@ class Lexer
   # validate   - Whether to validate the value of the marker
   #
   # Returns the String 0-index marker for this list item
-  def self.resolve_list_marker(list_type, marker, ordinal = 0, validate = false)
+  def self.resolve_list_marker(list_type, marker, ordinal = 0, validate = false, reader = nil)
     if list_type == :olist && !marker.start_with?('.')
-      resolve_ordered_list_marker(marker, ordinal, validate)
+      resolve_ordered_list_marker(marker, ordinal, validate, reader)
     elsif list_type == :colist
       '<1>'
     else
@@ -1778,7 +2002,7 @@ class Lexer
   #  # => 'A.'
   #
   # Returns the String of the first marker in this number series 
-  def self.resolve_ordered_list_marker(marker, ordinal = 0, validate = false)
+  def self.resolve_ordered_list_marker(marker, ordinal = 0, validate = false, reader = nil)
     number_style = ORDERED_LIST_STYLES.detect {|s| marker.match(ORDERED_LIST_MARKER_PATTERNS[s]) }
     expected = actual = nil
     case number_style
@@ -1817,8 +2041,7 @@ class Lexer
     end
 
     if validate && expected != actual
-      # FIXME I need a reader reference or line number to report line number
-      puts "asciidoctor: WARNING: list item index: expected #{expected}, got #{actual}"
+      warn "asciidoctor: WARNING: #{reader.line_info}: list item index: expected #{expected}, got #{actual}"
     end
 
     marker
@@ -1872,11 +2095,19 @@ class Lexer
       explicit_col_specs = false
     end
 
-    table_reader.skip_blank_lines
+    skipped = table_reader.skip_blank_lines
 
-    parser_ctx = Table::ParserContext.new(table, attributes)
+    parser_ctx = Table::ParserContext.new(table_reader, table, attributes)
+    loop_idx = -1
     while table_reader.has_more_lines?
-      line = table_reader.get_line
+      loop_idx += 1
+      line = table_reader.read_line
+
+      if skipped == 0 && loop_idx.zero? && !attributes.has_key?('options') &&
+          !(next_line = table_reader.peek_line).nil? && next_line == ::Asciidoctor::EOL
+        table.has_header_option = true
+        table.set_option 'header'
+      end
 
       if parser_ctx.format == 'psv'
         if parser_ctx.starts_with_delimiter? line
@@ -1889,8 +2120,7 @@ class Lexer
           if !next_cell_spec.nil?
             parser_ctx.close_open_cell next_cell_spec
           else
-            # QUESTION do we not advance to next line? if so, when
-            # will we if we came into this block?
+            # QUESTION do we not advance to next line? if so, when will we if we came into this block?
           end
         end
       end
@@ -1924,7 +2154,7 @@ class Lexer
           # no other delimiters to see here
           # suck up this line into the buffer and move on
           parser_ctx.buffer = %(#{parser_ctx.buffer}#{line})
-          # QUESTION make this an option? (unwrap-option?)
+          # QUESTION make stripping endlines in csv data an option? (unwrap-option?)
           if parser_ctx.format == 'csv'
             parser_ctx.buffer = %(#{parser_ctx.buffer.rstrip} )
           end
@@ -1938,7 +2168,7 @@ class Lexer
         end
       end
 
-      table_reader.skip_blank_lines unless parser_ctx.cell_open?
+      skipped = table_reader.skip_blank_lines unless parser_ctx.cell_open?
 
       if !table_reader.has_more_lines?
         parser_ctx.close_cell true
@@ -1995,7 +2225,8 @@ class Lexer
           end
         end
 
-        # TODO support percentage width
+        # to_i permits us to support percentage width by stripping the %
+        # NOTE this is slightly out of compliance w/ AsciiDoc, but makes way more sense
         spec['width'] = !m[3].nil? ? m[3].to_i : 1
 
         # make this an operation
@@ -2069,54 +2300,103 @@ class Lexer
   # both the original style attribute and the parsed value from the first
   # positional attribute.
   #
-  # attributes - The Hash of attributes to process
+  # attributes - The Hash of attributes to process and update
   #
   # Examples
   #
   #   puts attributes
-  #   => {1 => "abstract#intro.lead", "style" => "preamble"}
+  #   => {1 => "abstract#intro.lead%fragment", "style" => "preamble"}
   #
   #   parse_style_attribute(attributes)
   #   => ["abstract", "preamble"]
   #
   #   puts attributes
-  #   => {1 => "abstract#intro.lead", "style" => "abstract", "id" => "intro", "role" => "lead"}
+  #   => {1 => "abstract#intro.lead", "style" => "abstract", "id" => "intro",
+  #         "role" => "lead", "options" => ["fragment"], "fragment-option" => ''}
   #
   # Returns a two-element Array of the parsed style from the
   # first positional attribute and the original style that was
   # replaced
-  def self.parse_style_attribute(attributes)
+  def self.parse_style_attribute(attributes, reader = nil)
     original_style = attributes['style']
     raw_style = attributes[1]
+    # NOTE spaces are not allowed in shorthand, so if we find one, this ain't shorthand
     if !raw_style || raw_style.include?(' ')
       attributes['style'] = raw_style
       [raw_style, original_style]
-    # FIXME this logic could be condensed
     else
-      hash_index = raw_style.index('#')
-      dot_index = raw_style.index('.') 
-      if !hash_index.nil? && (dot_index.nil? || hash_index < dot_index)
-        parsed_style = attributes['style'] = (hash_index > 0 ? raw_style[0..(hash_index - 1)] : nil)
-        id = raw_style[(hash_index + 1)..-1]
-        if !dot_index.nil?
-          id, roles = id.split('.', 2)
-          attributes['id'] = id
-          attributes['role'] = roles.tr('.', ' ')
+      type = :style
+      collector = []
+      parsed = {}
+      # QUESTION should this be a private method? (though, it's never called if shorthand isn't used)
+      save_current = lambda {
+        if collector.empty?
+          if type != :style
+            warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} invalid empty #{type} detected in style attribute"
+          end
         else
-          attributes['id'] = id
+          case type
+          when :role, :option
+            parsed[type] ||= []
+            parsed[type].push collector.join
+          when :id
+            if parsed.has_key? :id
+              warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} multiple ids detected in style attribute"
+            end
+            parsed[type] = collector.join
+          else
+            parsed[type] = collector.join
+          end
+          collector = []
         end
-      elsif !dot_index.nil? && (hash_index.nil? || dot_index < hash_index)
-        parsed_style = attributes['style'] = (dot_index > 0 ? raw_style[0..(dot_index - 1)] : nil)
-        roles = raw_style[(dot_index + 1)..-1]
-        if !hash_index.nil?
-          roles, id = roles.split('#', 2)
-          attributes['id'] = id
-          attributes['role'] = roles.tr('.', ' ')
+      }
+
+      raw_style.split('').each do |c|
+        if c == '.' || c == '#' || c == '%'
+          save_current.call
+          case c
+          when '.'
+            type = :role
+          when '#'
+            type = :id
+          when '%'
+            type = :option
+          end
         else
-          attributes['role'] = roles.tr('.', ' ')
+          collector.push c
         end
-      else
+      end
+      
+      # small optimization if no shorthand is found
+      if type == :style
         parsed_style = attributes['style'] = raw_style
+      else
+        save_current.call
+
+        if parsed.has_key? :style
+          parsed_style = attributes['style'] = parsed[:style]
+        else
+          parsed_style = nil
+        end
+
+        if parsed.has_key? :id
+          attributes['id'] = parsed[:id]
+        end
+
+        if parsed.has_key? :role
+          attributes['role'] = parsed[:role] * ' '
+        end
+
+        if parsed.has_key? :option
+          (options = parsed[:option]).each do |option|
+            attributes["#{option}-option"] = ''
+          end
+          if (existing_opts = attributes['options'])
+            attributes['options'] = (options + existing_opts.split(',')) * ',' 
+          else
+            attributes['options'] = options * ','
+          end
+        end
       end
 
       [parsed_style, original_style]
@@ -2159,6 +2439,8 @@ class Lexer
   #   # => end
   #
   # returns the Array of String lines with block offset removed
+  #--
+  # FIXME refactor gsub matchers into compiled regex
   def self.reset_block_indent!(lines, indent = 0)
     return if indent.nil? || lines.empty?