asciidoctor 0.1.4 → 1.5.0

data/lib/asciidoctor/{lexer.rb → parser.rb}

@@ -1,11 +1,11 @@
 module Asciidoctor
 # Public: Methods to parse lines of AsciiDoc into an object hierarchy
 # representing the structure of the document. All methods are class methods and
-# should be invoked from the Lexer class. The main entry point is ::next_block.
-# No Lexer instances shall be discovered running around. (Any attempt to
-# instantiate a Lexer will be futile).
+# should be invoked from the Parser class. The main entry point is ::next_block.
+# No Parser instances shall be discovered running around. (Any attempt to
+# instantiate a Parser will be futile).
 #
-# The object hierarchy created by the Lexer consists of zero or more Section
+# The object hierarchy created by the Parser consists of zero or more Section
 # and Block objects. Section objects may be nested and a Section object
 # contains zero or more Block objects. Block objects may be nested, but may
 # only contain other Block objects. Block objects which represent lists may
@@ -14,18 +14,18 @@ module Asciidoctor
 # Examples
 #
 #   # Create a Reader for the AsciiDoc lines and retrieve the next block from it.
-#   # Lexer::next_block requires a parent, so we begin by instantiating an empty Document.
+#   # Parser.next_block requires a parent, so we begin by instantiating an empty Document.
 #
 #   doc = Document.new
 #   reader = Reader.new lines
-#   block = Lexer.next_block(reader, doc)
+#   block = Parser.next_block(reader, doc)
 #   block.class
 #   # => Asciidoctor::Block
-class Lexer
+class Parser
 
-  BlockMatchData = Struct.new(:context, :masq, :tip, :terminator)
+  BlockMatchData = Struct.new :context, :masq, :tip, :terminator
 
-  # Public: Make sure the Lexer object doesn't get initialized.
+  # Public: Make sure the Parser object doesn't get initialized.
   #
   # Raises RuntimeError if this constructor is invoked.
   def initialize
@@ -34,7 +34,7 @@ class Lexer
 
   # Public: Parses AsciiDoc source read from the Reader into the Document
   #
-  # This method is the main entry-point into the Lexer when parsing a full document.
+  # This method is the main entry-point into the Parser when parsing a full document.
   # It first looks for and, if found, processes the document title. It then
   # proceeds to iterate through the lines in the Reader, parsing the document
   # into nested Sections and Blocks.
@@ -50,8 +50,21 @@ class Lexer
     unless options[:header_only]
       while reader.has_more_lines?
         new_section, block_attributes = next_section(reader, document, block_attributes)
-        document << new_section unless new_section.nil?
+        document << new_section if new_section
       end
+      # NOTE we could try to avoid creating a preamble in the first place, though
+      # that would require reworking assumptions in next_section since the preamble
+      # is treated like an untitled section
+      # NOTE logic relocated to end of next_section
+      #if Compliance.unwrap_standalone_preamble &&
+      #    document.blocks.size == 1 && (first_block = document.blocks[0]).context == :preamble &&
+      #    first_block.blocks? && (document.doctype != 'book' || first_block.blocks[0].style != 'abstract')
+      #  preamble = document.blocks.shift
+      #  while (child_block = preamble.blocks.shift)
+      #    child_block.parent = document
+      #    document << child_block
+      #  end
+      #end
     end
 
     document
@@ -83,7 +96,7 @@ class Lexer
     # yep, document title logic in AsciiDoc is just insanity
     # definitely an area for spec refinement
     assigned_doctitle = nil
-    unless (val = document.attributes.fetch('doctitle', '')).empty?
+    unless (val = document.attributes['doctitle']).nil_or_empty?
       document.title = val
       assigned_doctitle = val
     end
@@ -92,20 +105,24 @@ class Lexer
     # check if the first line is the document title
     # if so, add a header to the document and parse the header metadata
     if is_next_line_document_title?(reader, block_attributes)
-      document.id, doctitle, _, _ = parse_section_title(reader, document)
+      source_location = reader.cursor if document.sourcemap
+      document.id, _, doctitle, _, single_line = parse_section_title(reader, document)
       unless assigned_doctitle
         document.title = doctitle
         assigned_doctitle = doctitle
       end
+      # default to compat-mode if document uses atx-style doctitle
+      document.set_attribute 'compat-mode', '' unless single_line
+      document.header.source_location = source_location if source_location
       document.attributes['doctitle'] = section_title = doctitle
       # QUESTION: should the id assignment on Document be encapsulated in the Document class?
-      if document.id.nil? && block_attributes.has_key?('id')
+      unless document.id
         document.id = block_attributes.delete('id')
       end
       parse_header_metadata(reader, document)
     end
 
-    if !(val = document.attributes.fetch('doctitle', '')).empty? &&
+    if !(val = document.attributes['doctitle']).nil_or_empty? &&
         val != section_title
       document.title = val
       assigned_doctitle = val
@@ -128,11 +145,11 @@ class Lexer
   #
   # returns Nothing
   def self.parse_manpage_header(reader, document)
-    if (m = document.attributes['doctitle'].match(REGEXP[:mantitle_manvolnum]))
+    if (m = ManpageTitleVolnumRx.match(document.attributes['doctitle']))
       document.attributes['mantitle'] = document.sub_attributes(m[1].rstrip.downcase)
       document.attributes['manvolnum'] = m[2].strip
     else
-      warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed manpage title"
+      warn %(asciidoctor: ERROR: #{reader.prev_line_info}: malformed manpage title)
     end
 
     reader.skip_blank_lines
@@ -140,24 +157,24 @@ class Lexer
     if is_next_line_section?(reader, {})
       name_section = initialize_section(reader, document, {})
       if name_section.level == 1
-        name_section_buffer = reader.read_lines_until(:break_on_blank_lines => true).join.tr_s("\n ", ' ')
-        if (m = name_section_buffer.match(REGEXP[:manname_manpurpose]))
-          document.attributes['manname'] = m[1] 
+        name_section_buffer = reader.read_lines_until(:break_on_blank_lines => true).join(' ').tr_s(' ', ' ')
+        if (m = ManpageNamePurposeRx.match(name_section_buffer))
+          document.attributes['manname'] = document.sub_attributes m[1]
           document.attributes['manpurpose'] = m[2] 
           # TODO parse multiple man names
 
           if document.backend == 'manpage'
             document.attributes['docname'] = document.attributes['manname']
-            document.attributes['outfilesuffix'] = ".#{document.attributes['manvolnum']}"
+            document.attributes['outfilesuffix'] = %(.#{document.attributes['manvolnum']})
           end
         else
-          warn "asciidoctor: ERROR: #{reader.prev_line_info}: malformed name section body"
+          warn %(asciidoctor: ERROR: #{reader.prev_line_info}: malformed name section body)
         end
       else
-        warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section title must be at level 1"
+        warn %(asciidoctor: ERROR: #{reader.prev_line_info}: name section title must be at level 1)
       end
     else
-      warn "asciidoctor: ERROR: #{reader.prev_line_info}: name section expected"
+      warn %(asciidoctor: ERROR: #{reader.prev_line_info}: name section expected)
     end
   end
 
@@ -184,33 +201,35 @@ class Lexer
   # Examples
   #
   #   source
-  #   # => "Greetings\n---------\nThis is my doc.\n\nSalutations\n-----------\nIt is awesome."
+  #   # => "= Greetings\n\nThis is my doc.\n\n== Salutations\n\nIt is awesome."
   #
-  #   reader = Reader.new source.lines.entries
+  #   reader = Reader.new source, nil, :normalize => true
   #   # create empty document to parent the section
   #   # and hold attributes extracted from header
   #   doc = Document.new
   #
-  #   Lexer.next_section(reader, doc).first.title
+  #   Parser.next_section(reader, doc).first.title
   #   # => "Greetings"
   #
-  #   Lexer.next_section(reader, doc).first.title
+  #   Parser.next_section(reader, doc).first.title
   #   # => "Salutations"
   #
   # returns a two-element Array containing the Section and Hash of orphaned attributes
   def self.next_section(reader, parent, attributes = {})
     preamble = false
+    part = false
+    intro = false
 
     # FIXME if attributes[1] is a verbatim style, then don't check for section
 
     # check if we are at the start of processing the document
     # NOTE we could drop a hint in the attributes to indicate
     # that we are at a section title (so we don't have to check)
-    if parent.is_a?(Document) && parent.blocks.empty? &&
-        (parent.has_header? || attributes.delete('invalid-header') || !is_next_line_section?(reader, attributes))
-
-      if parent.has_header?
-        preamble = Block.new(parent, :preamble, :content_model => :compound)
+    if parent.context == :document && parent.blocks.empty? &&
+        ((has_header = parent.has_header?) || attributes.delete('invalid-header') || !is_next_line_section?(reader, attributes))
+      doctype = parent.doctype
+      if has_header || (doctype == 'book' && attributes[1] != 'abstract')
+        preamble = intro = Block.new(parent, :preamble, :content_model => :compound)
         parent << preamble
       end
       section = parent
@@ -219,21 +238,26 @@ class Lexer
       if parent.attributes.has_key? 'fragment'
         expected_next_levels = nil
       # small tweak to allow subsequent level-0 sections for book doctype
-      elsif parent.doctype == 'book'
+      elsif doctype == 'book'
         expected_next_levels = [0, 1]
       else
         expected_next_levels = [1]
       end
     else
+      doctype = parent.document.doctype
       section = initialize_section(reader, parent, attributes)
       # clear attributes, except for title which carries over
       # section title to next block of content
-      attributes = attributes.delete_if {|k, v| k != 'title'}
+      attributes = (title = attributes['title']) ? { 'title' => title } : {}
       current_level = section.level
-      # subsections in preface & appendix in multipart books start at level 2
-      if current_level == 0 && section.special &&
-          section.document.doctype == 'book' && ['preface', 'appendix'].include?(section.sectname)
-        expected_next_levels = [current_level + 2]
+      if current_level == 0 && doctype == 'book'
+        part = !section.special
+        # subsections in preface & appendix in multipart books start at level 2
+        if section.special && (['preface', 'appendix'].include? section.sectname)
+          expected_next_levels = [current_level + 2]
+        else
+          expected_next_levels = [current_level + 1]
+        end
       else
         expected_next_levels = [current_level + 1]
       end
@@ -253,46 +277,103 @@ class Lexer
     while reader.has_more_lines?
       parse_block_metadata_lines(reader, section, attributes)
 
-      next_level = is_next_line_section? reader, attributes
-      if next_level
+      if (next_level = is_next_line_section? reader, attributes)
         next_level += section.document.attr('leveloffset', 0).to_i
-        doctype = parent.document.doctype
-        if next_level > current_level || (section.is_a?(Document) && next_level == 0)
+        if next_level > current_level || (section.context == :document && next_level == 0)
           if next_level == 0 && doctype != 'book'
-            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
-          elsif !expected_next_levels.nil? && !expected_next_levels.include?(next_level)
-            warn "asciidoctor: WARNING: #{reader.line_info}: section title out of sequence: " +
-                "expected #{expected_next_levels.size > 1 ? 'levels' : 'level'} #{expected_next_levels * ' or '}, " +
-                "got level #{next_level}"
+            warn %(asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections)
+          elsif expected_next_levels && !expected_next_levels.include?(next_level)
+            warn %(asciidoctor: WARNING: #{reader.line_info}: section title out of sequence: ) +
+                %(expected #{expected_next_levels.size > 1 ? 'levels' : 'level'} #{expected_next_levels * ' or '}, ) +
+                %(got level #{next_level})
           end
           # the attributes returned are those that are orphaned
           new_section, attributes = next_section(reader, section, attributes)
           section << new_section
         else
           if next_level == 0 && doctype != 'book'
-            warn "asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections"
+            warn %(asciidoctor: ERROR: #{reader.line_info}: only book doctypes can contain level 0 sections)
           end
           # close this section (and break out of the nesting) to begin a new one
           break
         end
       else
         # just take one block or else we run the risk of overrunning section boundaries
-        new_block = next_block(reader, (preamble || section), attributes, :parse_metadata => false)
-        if !new_block.nil?
-          (preamble || section) << new_block
+        block_line_info = reader.line_info
+        if (new_block = next_block reader, (intro || section), attributes, :parse_metadata => false)
+          # REVIEW this may be doing too much
+          if part
+            if !section.blocks?
+              # if this block wasn't marked as [partintro], emulate behavior as if it had
+              if new_block.style != 'partintro'
+                # emulate [partintro] paragraph
+                if new_block.context == :paragraph
+                  new_block.context = :open
+                  new_block.style = 'partintro'
+                # emulate [partintro] open block
+                else
+                  intro = Block.new section, :open, :content_model => :compound
+                  intro.style = 'partintro'
+                  new_block.parent = intro
+                  section << intro
+                end
+              end
+            elsif section.blocks.size == 1
+              first_block = section.blocks[0]
+              # open the [partintro] open block for appending
+              if !intro && first_block.content_model == :compound
+                #new_block.parent = (intro = first_block)
+                warn %(asciidoctor: ERROR: #{block_line_info}: illegal block content outside of partintro block)
+              # rebuild [partintro] paragraph as an open block
+              elsif first_block.content_model != :compound
+                intro = Block.new section, :open, :content_model => :compound
+                intro.style = 'partintro'
+                section.blocks.shift 
+                if first_block.style == 'partintro'
+                  first_block.context = :paragraph
+                  first_block.style = nil
+                end
+                first_block.parent = intro
+                intro << first_block
+                new_block.parent = intro
+                section << intro
+              end
+            end
+          end
+
+          (intro || section) << new_block
           attributes = {}
-        else
-          # don't clear attributes if we don't find a block because they may
-          # be trailing attributes that didn't get associated with a block
+        #else
+        #  # don't clear attributes if we don't find a block because they may
+        #  # be trailing attributes that didn't get associated with a block
         end
       end
 
       reader.skip_blank_lines
     end
 
-    if preamble && !preamble.blocks?
+    if part
+      unless section.blocks? && section.blocks[-1].context == :section
+        warn %(asciidoctor: ERROR: #{reader.line_info}: invalid part, must have at least one section (e.g., chapter, appendix, etc.))
+      end
+    # NOTE we could try to avoid creating a preamble in the first place, though
+    # that would require reworking assumptions in next_section since the preamble
+    # is treated like an untitled section
+    elsif preamble # implies parent == document
+      document = parent
+      if preamble.blocks?
+        # unwrap standalone preamble (i.e., no sections), if permissible
+        if Compliance.unwrap_standalone_preamble && document.blocks.size == 1 && doctype != 'book'
+          document.blocks.shift
+          while (child_block = preamble.blocks.shift)
+            child_block.parent = document
+            document << child_block
+          end
+        end
       # drop the preamble if it has no content
-      section.blocks.delete_at(0)
+      else
+        document.blocks.shift
+      end
     end
 
     # The attributes returned here are orphaned attributes that fall at the end
@@ -324,13 +405,12 @@ class Lexer
     skipped = reader.skip_blank_lines
 
     # bail if we've reached the end of the parent block or document
-    return nil unless reader.has_more_lines?
+    return unless reader.has_more_lines?
 
-    text_only = options[:text]
     # check for option to find list item text only
     # if skipped a line, assume a list continuation was
     # used and block content is acceptable
-    if text_only && skipped > 0
+    if (text_only = options[:text]) && skipped > 0
       options.delete(:text)
       text_only = false
     end
@@ -341,27 +421,30 @@ class Lexer
     document = parent.document
     if (extensions = document.extensions)
       block_extensions = extensions.blocks?
-      macro_extensions = extensions.block_macros?
+      block_macro_extensions = extensions.block_macros?
     else
-      block_extensions = macro_extensions = false
+      block_extensions = block_macro_extensions = false
     end
     #parent_context = parent.is_a?(Block) ? parent.context : nil
-    in_list = parent.is_a?(List)
+    in_list = (parent.is_a? List)
     block = nil
     style = nil
     explicit_style = nil
+    sourcemap = document.sourcemap
+    source_location = nil
 
-    while reader.has_more_lines? && block.nil?
+    while !block && reader.has_more_lines?
       # if parsing metadata, read until there is no more to read
       if parse_metadata && parse_block_metadata_line(reader, document, attributes, options)
         reader.advance
         next
-      #elsif parse_sections && parent_context.nil? && is_next_line_section?(reader, attributes)
+      #elsif parse_sections && !parent_context && is_next_line_section?(reader, attributes)
       #  block, attributes = next_section(reader, parent, attributes)
       #  break
       end
 
       # QUESTION should we introduce a parsing context object?
+      source_location = reader.cursor if sourcemap
       this_line = reader.read_line
       delimited_block = false
       block_context = nil
@@ -372,7 +455,7 @@ class Lexer
         style, explicit_style = parse_style_attribute(attributes, reader)
       end
 
-      if delimited_blk_match = is_delimited_block?(this_line, true)
+      if (delimited_blk_match = is_delimited_block? this_line, true)
         delimited_block = true
         block_context = cloaked_context = delimited_blk_match.context
         terminator = delimited_blk_match.terminator
@@ -383,16 +466,16 @@ class Lexer
             block_context = style.to_sym
           elsif delimited_blk_match.masq.include?('admonition') && ADMONITION_STYLES.include?(style)
             block_context = :admonition
-          elsif block_extensions && extensions.processor_registered_for_block?(style, block_context)
+          elsif block_extensions && extensions.registered_for_block?(style, block_context)
             block_context = style.to_sym
           else
-            warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for #{block_context} block: #{style}"
+            warn %(asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for #{block_context} block: #{style})
             style = block_context.to_s
           end
         end
       end
 
-      if !delimited_block
+      unless delimited_block
 
         # this loop only executes once; used for flow control
         # break once a block is found or at end of loop
@@ -401,7 +484,7 @@ class Lexer
         while true
 
           # process lines verbatim
-          if !style.nil? && COMPLIANCE[:strict_verbatim_paragraphs] && VERBATIM_STYLES.include?(style)
+          if style && Compliance.strict_verbatim_paragraphs && VERBATIM_STYLES.include?(style)
             block_context = style.to_sym
             reader.unshift_line this_line
             # advance to block parsing =>
@@ -410,14 +493,14 @@ class Lexer
 
           # process lines normally
           unless text_only
-            first_char = Compliance.markdown_syntax ? this_line.lstrip[0..0] : this_line[0..0]
-            # NOTE we're letting break lines (ruler, page_break, etc) have attributes
-            if BREAK_LINES.has_key?(first_char) && this_line.length > 3 &&
-                (match = this_line.match(Compliance.markdown_syntax ? REGEXP[:break_line_plus] : REGEXP[:break_line]))
-              block = Block.new(parent, BREAK_LINES[first_char], :content_model => :empty)
+            first_char = Compliance.markdown_syntax ? this_line.lstrip.chr : this_line.chr
+            # NOTE we're letting break lines (horizontal rule, page_break, etc) have attributes
+            if (LAYOUT_BREAK_LINES.has_key? first_char) && this_line.length >= 3 &&
+                (Compliance.markdown_syntax ? LayoutBreakLinePlusRx : LayoutBreakLineRx) =~ this_line
+              block = Block.new(parent, LAYOUT_BREAK_LINES[first_char], :content_model => :empty)
               break
 
-            elsif (match = this_line.match(REGEXP[:media_blk_macro]))
+            elsif this_line.end_with?(']') && (match = MediaBlockMacroRx.match(this_line))
               blk_ctx = match[1].to_sym
               block = Block.new(parent, blk_ctx, :content_model => :empty)
               if blk_ctx == :image
@@ -428,7 +511,7 @@ class Lexer
                 posattrs = []
               end
 
-              unless style.nil? || explicit_style
+              unless !style || explicit_style
                 attributes['alt'] = style if blk_ctx == :image
                 attributes.delete('style')
                 style = nil
@@ -441,52 +524,66 @@ class Lexer
                   :into => attributes)
               target = block.sub_attributes(match[2], :attribute_missing => 'drop-line')
               if target.empty?
-                if document.attributes.fetch('attribute-missing', COMPLIANCE[:attribute_missing]) == 'skip'
-                  # retain as unparsed
-                  return Block.new(parent, :paragraph, :source => [this_line.chomp])
+                # retain as unparsed if attribute-missing is skip
+                if document.attributes.fetch('attribute-missing', Compliance.attribute_missing) == 'skip'
+                  return Block.new(parent, :paragraph, :content_model => :simple, :source => [this_line])
+                # otherwise, drop the line
                 else
-                  # drop the line if target resolves to nothing
-                  return nil
+                  attributes.clear
+                  return
                 end
               end
 
               attributes['target'] = target
-              block.title = attributes.delete('title') if attributes.has_key?('title')
-              if blk_ctx == :image
-                document.register(:images, target)
-                attributes['alt'] ||= File.basename(target, File.extname(target)).tr('_-', ' ')
-                # QUESTION should video or audio have an auto-numbered caption?
-                block.assign_caption attributes.delete('caption'), 'figure'
-              end
+              # now done down below
+              #block.title = attributes.delete('title') if attributes.has_key?('title')
+              #if blk_ctx == :image
+              #  if attributes.has_key? 'scaledwidth'
+              #    # append % to scaledwidth if ends in number (no units present)
+              #    if (48..57).include?((attributes['scaledwidth'][-1] || 0).ord)
+              #      attributes['scaledwidth'] = %(#{attributes['scaledwidth']}%)
+              #    end
+              #  end
+              #  document.register(:images, target)
+              #  attributes['alt'] ||= ::File.basename(target, ::File.extname(target)).tr('_-', ' ')
+              #  # QUESTION should video or audio have an auto-numbered caption?
+              #  block.assign_caption attributes.delete('caption'), 'figure'
+              #end
               break
 
             # NOTE we're letting the toc macro have attributes
-            elsif first_char == 't' && (match = this_line.match(REGEXP[:toc]))
+            elsif first_char == 't' && (match = TocBlockMacroRx.match(this_line))
               block = Block.new(parent, :toc, :content_model => :empty)
               block.parse_attributes(match[1], [], :sub_result => false, :into => attributes)
               break
 
-            elsif macro_extensions && (match = this_line.match(REGEXP[:generic_blk_macro])) &&
-                extensions.processor_registered_for_block_macro?(match[1])
-              name = match[1]
+            elsif block_macro_extensions && (match = GenericBlockMacroRx.match(this_line)) &&
+                (extension = extensions.registered_for_block_macro?(match[1]))
               target = match[2]
               raw_attributes = match[3]
-              processor = extensions.load_block_macro_processor name, document
-              unless raw_attributes.empty?
-                document.parse_attributes(raw_attributes, processor.options.fetch(:pos_attrs, []),
-                    :sub_input => true, :sub_result => false, :into => attributes)
+              if extension.config[:content_model] == :attributes
+                unless raw_attributes.empty?
+                  document.parse_attributes(raw_attributes, (extension.config[:pos_attrs] || []),
+                      :sub_input => true, :sub_result => false, :into => attributes)
+                end
+              else
+                attributes['text'] = raw_attributes
               end
-              if !(default_attrs = processor.options.fetch(:default_attrs, {})).empty?
+              if (default_attrs = extension.config[:default_attrs])
                 default_attrs.each {|k, v| attributes[k] ||= v }
               end
-              block = processor.process parent, target, attributes
-              return nil if block.nil?
+              if (block = extension.process_method[parent, target, attributes.dup])
+                attributes.replace block.attributes
+              else
+                attributes.clear
+                return
+              end
               break
             end
           end
 
           # haven't found anything yet, continue
-          if (match = this_line.match(REGEXP[:colist]))
+          if (match = CalloutListRx.match(this_line))
             block = List.new(parent, :colist)
             attributes['style'] = 'arabic'
             reader.unshift_line this_line
@@ -495,48 +592,48 @@ class Lexer
               # might want to move this check to a validate method
               if match[1].to_i != expected_index
                 # FIXME this lineno - 2 hack means we need a proper look-behind cursor
-                warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: callout list item index: expected #{expected_index} got #{match[1]}"
+                warn %(asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: callout list item index: expected #{expected_index} got #{match[1]})
               end
               list_item = next_list_item(reader, block, match)
               expected_index += 1
-              if !list_item.nil?
+              if list_item
                 block << list_item
                 coids = document.callouts.callout_ids(block.items.size)
                 if !coids.empty?
                   list_item.attributes['coids'] = coids
                 else
                   # FIXME this lineno - 2 hack means we need a proper look-behind cursor
-                  warn "asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: no callouts refer to list item #{block.items.size}"
+                  warn %(asciidoctor: WARNING: #{reader.path}: line #{reader.lineno - 2}: no callouts refer to list item #{block.items.size})
                 end
               end
-            end while reader.has_more_lines? && match = reader.peek_line.match(REGEXP[:colist])
+            end while reader.has_more_lines? && (match = CalloutListRx.match(reader.peek_line))
 
             document.callouts.next_list
             break
 
-          elsif (match = this_line.match(REGEXP[:ulist]))
+          elsif UnorderedListRx =~ this_line
             reader.unshift_line this_line
             block = next_outline_list(reader, :ulist, parent)
             break
 
-          elsif (match = this_line.match(REGEXP[:olist]))
+          elsif (match = OrderedListRx.match(this_line))
             reader.unshift_line this_line
             block = next_outline_list(reader, :olist, parent)
             # QUESTION move this logic to next_outline_list?
             if !attributes['style'] && !block.attributes['style']
-              marker = block.items.first.marker
+              marker = block.items[0].marker
               if marker.start_with? '.'
                 # first one makes more sense, but second one is AsciiDoc-compliant
-                #attributes['style'] = (ORDERED_LIST_STYLES[block.level - 1] || ORDERED_LIST_STYLES.first).to_s
-                attributes['style'] = (ORDERED_LIST_STYLES[marker.length - 1] || ORDERED_LIST_STYLES.first).to_s
+                #attributes['style'] = (ORDERED_LIST_STYLES[block.level - 1] || ORDERED_LIST_STYLES[0]).to_s
+                attributes['style'] = (ORDERED_LIST_STYLES[marker.length - 1] || ORDERED_LIST_STYLES[0]).to_s
               else
-                style = ORDERED_LIST_STYLES.detect{|s| marker.match(ORDERED_LIST_MARKER_PATTERNS[s]) }
-                attributes['style'] = (style || ORDERED_LIST_STYLES.first).to_s
+                style = ORDERED_LIST_STYLES.detect{|s| OrderedListMarkerRxMap[s] =~ marker }
+                attributes['style'] = (style || ORDERED_LIST_STYLES[0]).to_s
               end
             end
             break
 
-          elsif (match = this_line.match(REGEXP[:dlist]))
+          elsif (match = DefinitionListRx.match(this_line))
             reader.unshift_line this_line
             block = next_labeled_list(reader, match, parent)
             break
@@ -544,10 +641,11 @@ class Lexer
           elsif (style == 'float' || style == 'discrete') &&
               is_section_title?(this_line, (Compliance.underline_style_section_titles ? reader.peek_line(true) : nil))
             reader.unshift_line this_line
-            float_id, float_title, float_level, _ = parse_section_title(reader, document)
+            float_id, float_reftext, float_title, float_level, _ = parse_section_title(reader, document)
+            attributes['reftext'] = float_reftext if float_reftext
             float_id ||= attributes['id'] if attributes.has_key?('id')
             block = Block.new(parent, :floating_title, :content_model => :empty)
-            if float_id.nil? || float_id.empty?
+            if float_id.nil_or_empty?
               # FIXME remove hack of creating throwaway Section to get at the generate_id method
               tmp_sect = Section.new(parent)
               tmp_sect.title = float_title
@@ -555,14 +653,13 @@ class Lexer
             else
               block.id = float_id
             end
-            document.register(:ids, [block.id, float_title]) if block.id
             block.level = float_level
             block.title = float_title
             break
 
           # FIXME create another set for "passthrough" styles
           # FIXME make this more DRY!
-          elsif !style.nil? && style != 'normal'
+          elsif style && style != 'normal'
             if PARAGRAPH_STYLES.include?(style)
               block_context = style.to_sym
               cloaked_context = :paragraph
@@ -575,14 +672,14 @@ class Lexer
               reader.unshift_line this_line
               # advance to block parsing =>
               break
-            elsif block_extensions && extensions.processor_registered_for_block?(style, :paragraph)
+            elsif block_extensions && extensions.registered_for_block?(style, :paragraph)
               block_context = style.to_sym
               cloaked_context = :paragraph
               reader.unshift_line this_line
               # advance to block parsing =>
               break
             else
-              warn "asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for paragraph: #{style}"
+              warn %(asciidoctor: WARNING: #{reader.prev_line_info}: invalid style for paragraph: #{style})
               style = nil
               # continue to process paragraph
             end
@@ -591,7 +688,7 @@ class Lexer
           break_at_list = (skipped == 0 && in_list)
 
           # a literal paragraph is contiguous lines starting at least one space
-          if style != 'normal' && this_line.match(REGEXP[:lit_par])
+          if style != 'normal' && LiteralParagraphRx =~ this_line
             # So we need to actually include this one in the read_lines group
             reader.unshift_line this_line
             lines = reader.read_lines_until(
@@ -602,8 +699,8 @@ class Lexer
               # and therefore we should not break at a list item
               # (this won't stop breaking on item of same level since we've already parsed them out)
               # QUESTION can we turn this block into a lambda or function call?
-              (break_at_list && line.match(REGEXP[:any_list])) ||
-              (COMPLIANCE[:block_terminates_paragraph] && (is_delimited_block?(line) || line.match(REGEXP[:attr_line])))
+              (break_at_list && AnyListRx =~ line) ||
+              (Compliance.block_terminates_paragraph && (is_delimited_block?(line) || BlockAttributeLineRx =~ line))
             }
 
             reset_block_indent! lines
@@ -625,8 +722,8 @@ class Lexer
               # and therefore we should not break at a list item
               # (this won't stop breaking on item of same level since we've already parsed them out)
               # QUESTION can we turn this block into a lambda or function call?
-              (break_at_list && line.match(REGEXP[:any_list])) ||
-              (COMPLIANCE[:block_terminates_paragraph] && (is_delimited_block?(line) || line.match(REGEXP[:attr_line])))
+              (break_at_list && AnyListRx =~ line) ||
+              (Compliance.block_terminates_paragraph && (is_delimited_block?(line) || BlockAttributeLineRx =~ line))
             }
 
             # NOTE we need this logic because we've asked the reader to skip
@@ -635,59 +732,57 @@ class Lexer
             if lines.empty?
               # call advance since the reader preserved the last line
               reader.advance
-              return nil
+              return
             end
 
-            catalog_inline_anchors(lines.join, document)
+            catalog_inline_anchors(lines.join(EOL), document)
 
-            first_line = lines.first
-            if !text_only && (admonition_match = first_line.match(REGEXP[:admonition_inline]))
+            first_line = lines[0]
+            if !text_only && (admonition_match = AdmonitionParagraphRx.match(first_line))
               lines[0] = admonition_match.post_match.lstrip
               attributes['style'] = admonition_match[1]
               attributes['name'] = admonition_name = admonition_match[1].downcase
-              attributes['caption'] ||= document.attributes["#{admonition_name}-caption"]
-              block = Block.new(parent, :admonition, :source => lines, :attributes => attributes)
+              attributes['caption'] ||= document.attributes[%(#{admonition_name}-caption)]
+              block = Block.new(parent, :admonition, :content_model => :simple, :source => lines, :attributes => attributes)
             elsif !text_only && Compliance.markdown_syntax && first_line.start_with?('> ')
               lines.map! {|line|
-                if line.start_with?('> ')
-                  line[2..-1]
-                elsif line.chomp == '>'
+                if line == '>'
                   line[1..-1]
+                elsif line.start_with? '> '
+                  line[2..-1]
                 else
                   line
                 end
               }
 
-              if lines.last.start_with?('-- ')
+              if lines[-1].start_with? '-- '
                 attribution, citetitle = lines.pop[3..-1].split(', ', 2)
-                lines.pop while lines.last.chomp.empty?
-                lines[-1] = lines.last.chomp
+                lines.pop while lines[-1].empty?
               else
                 attribution, citetitle = nil
               end
               attributes['style'] = 'quote'
-              attributes['attribution'] = attribution unless attribution.nil?
-              attributes['citetitle'] = citetitle unless citetitle.nil?
+              attributes['attribution'] = attribution if attribution
+              attributes['citetitle'] = citetitle if citetitle
               # NOTE will only detect headings that are floating titles (not section titles)
               # TODO could assume a floating title when inside a block context
               # FIXME Reader needs to be created w/ line info
               block = build_block(:quote, :compound, false, parent, Reader.new(lines), attributes)
             elsif !text_only && lines.size > 1 && first_line.start_with?('"') &&
-                lines.last.start_with?('-- ') && lines[-2].chomp.end_with?('"')
+                lines[-1].start_with?('-- ') && lines[-2].end_with?('"')
               lines[0] = first_line[1..-1]
               attribution, citetitle = lines.pop[3..-1].split(', ', 2)
-              lines.pop while lines.last.chomp.empty?
-              lines[-1] = lines.last.chomp.chop
+              lines.pop while lines[-1].empty?
+              # strip trailing quote
+              lines[-1] = lines[-1].chop
               attributes['style'] = 'quote'
-              attributes['attribution'] = attribution unless attribution.nil?
-              attributes['citetitle'] = citetitle unless citetitle.nil?
-              block = Block.new(parent, :quote, :source => lines, :attributes => attributes)
-              #block = Block.new(parent, :quote, :content_model => :compound, :attributes => attributes)
-              #block << Block.new(block, :paragraph, :source => lines)
+              attributes['attribution'] = attribution if attribution
+              attributes['citetitle'] = citetitle if citetitle
+              block = Block.new(parent, :quote, :content_model => :simple, :source => lines, :attributes => attributes)
             else
               # if [normal] is used over an indented paragraph, unindent it
-              if style == 'normal' && ((first_char = lines.first[0..0]) == ' ' || first_char == "\t")
-                first_line = lines.first
+              if style == 'normal' && ((first_char = lines[0].chr) == ' ' || first_char == TAB)
+                first_line = lines[0]
                 first_line_shifted = first_line.lstrip
                 indent = line_length(first_line) - line_length(first_line_shifted)
                 lines[0] = first_line_shifted
@@ -697,7 +792,7 @@ class Lexer
                 end
               end
 
-              block = Block.new(parent, :paragraph, :source => lines, :attributes => attributes)
+              block = Block.new(parent, :paragraph, :content_model => :simple, :source => lines, :attributes => attributes)
             end
           end
 
@@ -707,7 +802,7 @@ class Lexer
       end
 
       # either delimited block or styled paragraph
-      if block.nil? && !block_context.nil?
+      if !block && block_context
         # abstract and partintro should be handled by open block
         # FIXME kind of hackish...need to sort out how to generalize this
         block_context = :open if block_context == :abstract || block_context == :partintro
@@ -715,29 +810,36 @@ class Lexer
         case block_context
         when :admonition
           attributes['name'] = admonition_name = style.downcase
-          attributes['caption'] ||= document.attributes["#{admonition_name}-caption"]
+          attributes['caption'] ||= document.attributes[%(#{admonition_name}-caption)]
           block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :comment
           build_block(block_context, :skip, terminator, parent, reader, attributes)
-          return nil
+          return
 
         when :example
-          block = build_block(block_context, :compound, terminator, parent, reader, attributes, {:supports_caption => true})
+          block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :listing, :fenced_code, :source
           if block_context == :fenced_code
             style = attributes['style'] = 'source'
-            language, linenums = this_line[3...-1].split(',', 2)
+            language, linenums = this_line[3..-1].split(',', 2)
             if language && !(language = language.strip).empty?
               attributes['language'] = language
               attributes['linenums'] = '' if linenums && !linenums.strip.empty?
+            elsif (default_language = document.attributes['source-language'])
+              attributes['language'] = default_language
             end
             terminator = terminator[0..2]
           elsif block_context == :source
             AttributeList.rekey(attributes, [nil, 'language', 'linenums'])
+            unless attributes.has_key? 'language'
+              if (default_language = document.attributes['source-language'])
+                attributes['language'] = default_language
+              end
+            end
           end
-          block = build_block(:listing, :verbatim, terminator, parent, reader, attributes, {:supports_caption => true})
+          block = build_block(:listing, :verbatim, terminator, parent, reader, attributes)
 
         when :literal
           block = build_block(block_context, :verbatim, terminator, parent, reader, attributes)
@@ -745,13 +847,25 @@ class Lexer
         when :pass
           block = build_block(block_context, :raw, terminator, parent, reader, attributes)
 
+        when :stem, :latexmath, :asciimath
+          if block_context == :stem
+            attributes['style'] = if (explicit_stem_syntax = attributes[2])
+              explicit_stem_syntax.include?('tex') ? 'latexmath' : 'asciimath'
+            elsif (default_stem_syntax = document.attributes['stem']).nil_or_empty?
+              'asciimath'
+            else
+              default_stem_syntax
+            end
+          end
+          block = build_block(:stem, :raw, terminator, parent, reader, attributes)
+
         when :open, :sidebar
           block = build_block(block_context, :compound, terminator, parent, reader, attributes)
 
         when :table
           cursor = reader.cursor
           block_reader = Reader.new reader.read_lines_until(:terminator => terminator, :skip_line_comments => true), cursor
-          case terminator[0..0]
+          case terminator.chr
             when ','
               attributes['format'] = 'csv'
             when ':'
@@ -764,22 +878,24 @@ class Lexer
           block = build_block(block_context, (block_context == :verse ? :verbatim : :compound), terminator, parent, reader, attributes)
 
         else
-          if block_extensions && extensions.processor_registered_for_block?(block_context, cloaked_context)
-            processor = extensions.load_block_processor block_context, document
-            
-            if (content_model = processor.options[:content_model]) != :skip
-              if !(pos_attrs = processor.options.fetch(:pos_attrs, [])).empty?
+          if block_extensions && (extension = extensions.registered_for_block?(block_context, cloaked_context))
+            # TODO pass cloaked_context to extension somehow (perhaps a new instance for each cloaked_context?)
+            if (content_model = extension.config[:content_model]) != :skip
+              if !(pos_attrs = extension.config[:pos_attrs] || []).empty?
                 AttributeList.rekey(attributes, [nil].concat(pos_attrs))
               end
-              if !(default_attrs = processor.options.fetch(:default_attrs, {})).empty?
+              if (default_attrs = extension.config[:default_attrs])
                 default_attrs.each {|k, v| attributes[k] ||= v }
               end
             end
-            block = build_block(block_context, content_model, terminator, parent, reader, attributes, :processor => processor)
-            return nil if block.nil?
+            block = build_block block_context, content_model, terminator, parent, reader, attributes, :extension => extension
+            unless block && content_model != :skip
+              attributes.clear
+              return
+            end
           else
             # this should only happen if there's a misconfiguration
-            raise "Unsupported block type #{block_context} at #{reader.line_info}"
+            raise %(Unsupported block type #{block_context} at #{reader.line_info})
           end
         end
       end
@@ -789,20 +905,38 @@ class Lexer
     # blocks or trailing attribute lists could leave us without a block,
     # so handle accordingly
     # REVIEW we may no longer need this nil check
-    if !block.nil?
+    # FIXME we've got to clean this up, it's horrible!
+    if block
+      block.source_location = source_location if source_location
       # REVIEW seems like there is a better way to organize this wrap-up
-      block.id      ||= attributes['id'] if attributes.has_key?('id')
       block.title     = attributes['title'] unless block.title?
-      block.caption ||= attributes.delete('caption')
+      # FIXME HACK don't hardcode logic for alt, caption and scaledwidth on images down here
+      if block.context == :image
+        resolved_target = attributes['target']
+        block.document.register(:images, resolved_target)
+        attributes['alt'] ||= ::File.basename(resolved_target, ::File.extname(resolved_target)).tr('_-', ' ')
+        attributes['alt'] = block.sub_specialcharacters attributes['alt']
+        block.assign_caption attributes.delete('caption'), 'figure'
+        if (scaledwidth = attributes['scaledwidth'])
+          # append % to scaledwidth if ends in number (no units present)
+          if (48..57).include?((scaledwidth[-1] || 0).ord)
+            attributes['scaledwidth'] = %(#{scaledwidth}%)
+          end
+        end
+      else
+        block.caption ||= attributes.delete('caption')
+      end
       # TODO eventualy remove the style attribute from the attributes hash
       #block.style     = attributes.delete('style')
       block.style     = attributes['style']
       # AsciiDoc always use [id] as the reftext in HTML output,
       # but I'd like to do better in Asciidoctor
-      if block.id && block.title? && !attributes.has_key?('reftext')
-        document.register(:ids, [block.id, block.title])
+      if (block_id = (block.id ||= attributes['id']))
+        # TODO sub reftext
+        document.register(:ids, [block_id, (attributes['reftext'] || (block.title? ? block.title : nil))])
       end
-      block.update_attributes(attributes)
+      # FIXME remove the need for this update!
+      block.attributes.update(attributes) unless attributes.empty?
       block.lock_in_subs
 
       #if document.attributes.has_key? :pending_attribute_entries
@@ -812,7 +946,7 @@ class Lexer
       #end
 
       if block.sub? :callouts
-        if !(catalog_callouts block.source, document)
+        unless (catalog_callouts block.source, document)
           # No need to look for callouts if they aren't there
           block.remove_sub :callouts
         end
@@ -827,17 +961,14 @@ class Lexer
   # returns the match data if this line is the first line of a delimited block or nil if not
   def self.is_delimited_block? line, return_match_data = false
     # highly optimized for best performance
-    line_len = line.length - 1
-    return nil unless line_len > 1 && DELIMITED_BLOCK_LEADERS.include?(line[0..1])
-    line = line.chomp
-    # counts endline character in line length
+    return unless (line_len = line.length) > 1 && (DELIMITED_BLOCK_LEADERS.include? line[0..1])
+    # catches open block
     if line_len == 2
       tip = line
       tl = 2
-    elsif line_len < 3
-      return nil
     else
-      if line_len < 5
+      # catches all other delimited blocks, including fenced code
+      if line_len <= 4
         tip = line
         tl = line_len
       else
@@ -846,27 +977,27 @@ class Lexer
       end
 
       # special case for fenced code blocks
+      # REVIEW review this logic
+      fenced_code = false
       if Compliance.markdown_syntax
-        tip_alt = tip.chop if tl == 4
-        if tip_alt == '```'
-          if tip.end_with? '`'
-            return nil
-          end
-          tip = tip_alt
-          tl = 3
-        elsif tip_alt == '~~~'
-          if tip.end_with? '~'
-            return nil
+        tip_3 = (tl == 4 ? tip.chop : tip)
+        if tip_3 == '```'
+          if tl == 4 && tip.end_with?('`')
+            return
           end
-          tip = tip_alt
+          tip = tip_3
           tl = 3
+          fenced_code = true
         end
       end
+
+      # short circuit if not a fenced code block
+      return if tl == 3 && !fenced_code
     end
 
     if DELIMITED_BLOCKS.has_key? tip
       # tip is the full line when delimiter is minimum length
-      if tl == 3 || tl == line_len
+      if tl < 4 || tl == line_len
         if return_match_data
           context, masq = *DELIMITED_BLOCKS[tip]
           BlockMatchData.new(context, masq, tip, tip)
@@ -880,7 +1011,8 @@ class Lexer
         else
           true
         end
-      #elsif match = line.match(REGEXP[:any_blk])
+      # only enable if/when we decide to support non-congruent block delimiters
+      #elsif (match = BlockDelimiterRx.match(line))
       #  if return_match_data
       #    context, masq = *DELIMITED_BLOCKS[tip]
       #    BlockMatchData.new(context, masq, tip, match[0])
@@ -918,13 +1050,13 @@ class Lexer
             :preserve_last_line => true,
             :skip_line_comments => true,
             :skip_processing => skip_processing) {|line|
-          COMPLIANCE[:block_terminates_paragraph] && (is_delimited_block?(line) || line.match(REGEXP[:attr_line]))
+          Compliance.block_terminates_paragraph && (is_delimited_block?(line) || BlockAttributeLineRx =~ line)
         }
         # QUESTION check for empty lines after grabbing lines for simple content model?
       end
       block_reader = nil
     elsif parse_as_content_model != :compound
-      lines = reader.read_lines_until(:terminator => terminator, :chomp_last_line => true, :skip_processing => skip_processing)
+      lines = reader.read_lines_until(:terminator => terminator, :skip_processing => skip_processing)
       block_reader = nil
     # terminator is false when reader has already been prepared
     elsif terminator == false
@@ -938,24 +1070,37 @@ class Lexer
 
     if content_model == :skip
       attributes.clear
+      # FIXME we shouldn't be mixing return types
       return lines
     end
 
-    if content_model == :verbatim && attributes.has_key?('indent')
-      reset_block_indent! lines, attributes['indent'].to_i
+    if content_model == :verbatim && (indent = attributes['indent'])
+      reset_block_indent! lines, indent.to_i
     end
 
-    if (processor = options[:processor])
+    if (extension = options[:extension])
+      # QUESTION do we want to delete the style?
       attributes.delete('style')
-      processor.options[:content_model] = content_model
-      block = processor.process(parent, block_reader || Reader.new(lines), attributes)
+      if (block = extension.process_method[parent, block_reader || (Reader.new lines), attributes.dup])
+        attributes.replace block.attributes
+        # FIXME if the content model is set to compound, but we only have simple in this context, then
+        # forcefully set the content_model to simple to prevent parsing blocks from children
+        # TODO document this behavior!!
+        if block.content_model == :compound && !(lines = block.lines).nil_or_empty?
+          content_model = :compound
+          block_reader = Reader.new lines
+        end
+      else
+        # FIXME need a test to verify this returns nil at the right time
+        return
+      end
     else
-      block = Block.new(parent, block_context, :content_model => content_model, :attributes => attributes, :source => lines)
+      block = Block.new(parent, block_context, :content_model => content_model, :source => lines, :attributes => attributes)
     end
 
-    # should supports_caption be necessary?
-    if options.fetch(:supports_caption, false)
-      block.title = attributes.delete('title') if attributes.has_key?('title')
+    # QUESTION should we have an explicit map or can we rely on check for *-caption attribute?
+    if (attributes.has_key? 'title') && (block.document.attr? %(#{block.context}-caption))
+      block.title = attributes.delete 'title'
       block.assign_caption attributes.delete('caption')
     end
 
@@ -970,7 +1115,7 @@ class Lexer
 
   # Public: Parse blocks from this reader until there are no more lines.
   #
-  # This method calls Lexer#next_block until there are no more lines in the
+  # This method calls Parser#next_block until there are no more lines in the
   # Reader. It does not consider sections because it's assumed the Reader only
   # has lines which are within a delimited block region.
   #
@@ -980,8 +1125,8 @@ class Lexer
   # Returns nothing.
   def self.parse_blocks(reader, parent)
     while reader.has_more_lines?
-      block = Lexer.next_block(reader, parent)
-      parent << block unless block.nil?
+      block = Parser.next_block(reader, parent)
+      parent << block if block
     end
   end
 
@@ -1001,18 +1146,18 @@ class Lexer
     end
     #Debug.debug { "Created #{list_type} block: #{list_block}" }
 
-    while reader.has_more_lines? && (match = reader.peek_line.match(REGEXP[list_type]))
+    while reader.has_more_lines? && (match = ListRxMap[list_type].match(reader.peek_line))
       marker = resolve_list_marker(list_type, match[1])
 
       # if we are moving to the next item, and the marker is different
       # determine if we are moving up or down in nesting
-      if list_block.items? && marker != list_block.items.first.marker
+      if list_block.items? && marker != list_block.items[0].marker
         # assume list is nested by default, but then check to see if we are
         # popping out of a nested list by matching an ancestor's list marker
         this_item_level = list_block.level + 1
         ancestor = parent
         while ancestor.context == list_type
-          if marker == ancestor.items.first.marker
+          if marker == ancestor.items[0].marker
             this_item_level = ancestor.level
             break
           end
@@ -1030,10 +1175,10 @@ class Lexer
       elsif this_item_level > list_block.level
         # If this next list level is down one from the
         # current Block's, append it to content of the current list item
-        list_block.items.last << next_block(reader, list_block)
+        list_block.items[-1] << next_block(reader, list_block)
       end
 
-      list_block << list_item unless list_item.nil?
+      list_block << list_item if list_item
       list_item = nil
 
       reader.skip_blank_lines
@@ -1051,10 +1196,10 @@ class Lexer
   def self.catalog_callouts(text, document)
     found = false
     if text.include? '<'
-      text.scan(REGEXP[:callout_quick_scan]) {
+      text.scan(CalloutQuickScanRx) {
         # alias match for Ruby 1.8.7 compat
         m = $~
-        if m[0][0..0] != '\\'
+        if m[0].chr != '\\'
           document.callouts.register(m[2])
         end
         # we have to mark as found even if it's escaped so it can be unescaped
@@ -1071,17 +1216,21 @@ class Lexer
   #
   # Returns nothing
   def self.catalog_inline_anchors(text, document)
-    text.scan(REGEXP[:anchor_macro]) {
-      # alias match for Ruby 1.8.7 compat
-      m = $~
-      next if m[0].start_with? '\\'
-      id, reftext = m[1].split(',')
-      id.sub!(REGEXP[:dbl_quoted], '\2')
-      if !reftext.nil?
-        reftext.sub!(REGEXP[:m_dbl_quoted], '\2')
-      end
-      document.register(:ids, [id, reftext])
-    }
+    if text.include? '['
+      text.scan(InlineAnchorRx) {
+        # alias match for Ruby 1.8.7 compat
+        m = $~
+        next if m[0].start_with? '\\'
+        id = m[1] || m[3]
+        reftext = m[2] || m[4]
+        # enable if we want to allow double quoted values
+        #id = id.sub(DoubleQuotedRx, '\2')
+        #if reftext
+        #  reftext = reftext.sub(DoubleQuotedMultiRx, '\2')
+        #end
+        document.register(:ids, [id, reftext])
+      }
+    end
     nil
   end
 
@@ -1097,11 +1246,11 @@ class Lexer
     previous_pair = nil
     # allows us to capture until we find a labeled item
     # that uses the same delimiter (::, :::, :::: or ;;)
-    sibling_pattern = REGEXP[:dlist_siblings][match[2]]
+    sibling_pattern = DefinitionListSiblingRx[match[2]]
 
     begin
       term, item = next_list_item(reader, list_block, match, sibling_pattern)
-      if !previous_pair.nil? && previous_pair.last.nil?
+      if previous_pair && !previous_pair[-1]
         previous_pair.pop
         previous_pair[0] << term
         previous_pair << item
@@ -1109,7 +1258,7 @@ class Lexer
         # FIXME this misses the automatic parent assignment
         list_block.items << (previous_pair = [[term], item])
       end
-    end while reader.has_more_lines? && match = reader.peek_line.match(sibling_pattern)
+    end while reader.has_more_lines? && (match = sibling_pattern.match(reader.peek_line))
 
     list_block
   end
@@ -1132,22 +1281,20 @@ class Lexer
   # Returns the next ListItem or ListItem pair (depending on the list type)
   # for the parent list Block.
   def self.next_list_item(reader, list_block, match, sibling_trait = nil)
-    list_type = list_block.context
-
-    if list_type == :dlist
+    if (list_type = list_block.context) == :dlist
       list_term = ListItem.new(list_block, match[1])
       list_item = ListItem.new(list_block, match[3])
-      has_text = !match[3].to_s.empty?
+      has_text = !match[3].nil_or_empty?
     else
       # Create list item using first line as the text of the list item
       text = match[2]
       checkbox = false
       if list_type == :ulist && text.start_with?('[')
-        if text.start_with? '[ ] '
+        if text.start_with?('[ ] ')
           checkbox = true
           checked = false
           text = text[3..-1].lstrip
-        elsif text.start_with?('[*] ') || text.start_with?('[x] ')
+        elsif text.start_with?('[x] ') || text.start_with?('[*] ')
           checkbox = true
           checked = true
           text = text[3..-1].lstrip
@@ -1162,9 +1309,7 @@ class Lexer
         list_item.attributes['checked'] = '' if checked
       end
 
-      if !sibling_trait
-        sibling_trait = resolve_list_marker(list_type, match[1], list_block.items.size, true, reader)
-      end
+      sibling_trait ||= resolve_list_marker(list_type, match[1], list_block.items.size, true, reader)
       list_item.marker = sibling_trait
       has_text = true
     end
@@ -1179,13 +1324,13 @@ class Lexer
       list_item_reader.unshift_lines comment_lines unless comment_lines.empty? 
 
       if !subsequent_line.nil?
-        continuation_connects_first_block = (subsequent_line == ::Asciidoctor::EOL)
+        continuation_connects_first_block = subsequent_line.empty?
         # if there's no continuation connecting the first block, then
         # treat the lines as paragraph text (activated when has_text = false)
         if !continuation_connects_first_block && list_type != :dlist
           has_text = false
         end
-        content_adjacent = !subsequent_line.chomp.empty?
+        content_adjacent = !continuation_connects_first_block && !subsequent_line.empty?
       else
         continuation_connects_first_block = false
         content_adjacent = false
@@ -1199,7 +1344,7 @@ class Lexer
       # list
       while list_item_reader.has_more_lines?
         new_block = next_block(list_item_reader, list_block, {}, options)
-        list_item << new_block unless new_block.nil?
+        list_item << new_block if new_block
       end
 
       list_item.fold_first(continuation_connects_first_block, content_adjacent)
@@ -1255,17 +1400,17 @@ class Lexer
       # the termination of the list
       break if is_sibling_list_item?(this_line, list_type, sibling_trait)
 
-      prev_line = buffer.empty? ? nil : buffer.last.chomp
+      prev_line = buffer.empty? ? nil : buffer[-1]
 
       if prev_line == LIST_CONTINUATION
         if continuation == :inactive
           continuation = :active
           has_text = true
-          buffer[-1] = ::Asciidoctor::EOL unless within_nested_list
+          buffer[-1] = '' unless within_nested_list
         end
 
         # dealing with adjacent list continuations (which is really a syntax error)
-        if this_line.chomp == LIST_CONTINUATION
+        if this_line == LIST_CONTINUATION
           if continuation != :frozen
             continuation = :frozen
             buffer << this_line
@@ -1277,7 +1422,7 @@ class Lexer
 
       # a delimited block immediately breaks the list unless preceded
       # by a list continuation (they are harsh like that ;0)
-      if match = is_delimited_block?(this_line, true)
+      if (match = is_delimited_block?(this_line, true))
         if continuation == :active
           buffer << this_line
           # grab all the lines in the block, leaving the delimiters in place
@@ -1287,17 +1432,18 @@ class Lexer
         else
           break
         end
-      # technically attr_line only breaks if ensuing line is not a list item
-      # which really means attr_line only breaks if it's acting as a block delimiter
-      elsif list_type == :dlist && continuation != :active && this_line.match(REGEXP[:attr_line])
+      # technically BlockAttributeLineRx only breaks if ensuing line is not a list item
+      # which really means BlockAttributeLineRx only breaks if it's acting as a block delimiter
+      # FIXME to be AsciiDoc compliant, we shouldn't break if style in attribute line is "literal" (i.e., [literal])
+      elsif list_type == :dlist && continuation != :active && BlockAttributeLineRx =~ this_line
         break
       else
-        if continuation == :active && !this_line.chomp.empty?
+        if continuation == :active && !this_line.empty?
           # literal paragraphs have special considerations (and this is one of 
           # two entry points into one)
           # if we don't process it as a whole, then a line in it that looks like a
           # list item will throw off the exit from it
-          if this_line.match(REGEXP[:lit_par])
+          if LiteralParagraphRx =~ this_line
             reader.unshift_line this_line
             buffer.concat reader.read_lines_until(
                 :preserve_last_line => true,
@@ -1309,12 +1455,12 @@ class Lexer
             }
             continuation = :inactive
           # let block metadata play out until we find the block
-          elsif this_line.match(REGEXP[:blk_title]) || this_line.match(REGEXP[:attr_line]) || this_line.match(REGEXP[:attr_entry])
+          elsif BlockTitleRx =~ this_line || BlockAttributeLineRx =~ this_line || AttributeEntryRx =~ this_line
             buffer << this_line
           else
-            if nested_list_type = (within_nested_list ? [:dlist] : NESTABLE_LIST_CONTEXTS).detect {|ctx| this_line.match(REGEXP[ctx]) }
+            if nested_list_type = (within_nested_list ? [:dlist] : NESTABLE_LIST_CONTEXTS).detect {|ctx| ListRxMap[ctx] =~ this_line }
               within_nested_list = true
-              if nested_list_type == :dlist && $~[3].to_s.empty?
+              if nested_list_type == :dlist && $~[3].nil_or_empty?
                 # get greedy again
                 has_text = false
               end
@@ -1322,16 +1468,16 @@ class Lexer
             buffer << this_line
             continuation = :inactive
           end
-        elsif !prev_line.nil? && prev_line.chomp.empty?
+        elsif !prev_line.nil? && prev_line.empty?
           # advance to the next line of content
-          if this_line.chomp.empty?
+          if this_line.empty?
             reader.skip_blank_lines
             this_line = reader.read_line 
             # if we hit eof or a sibling, stop reading
             break if this_line.nil? || is_sibling_list_item?(this_line, list_type, sibling_trait)
           end
 
-          if this_line.chomp == LIST_CONTINUATION
+          if this_line == LIST_CONTINUATION
             detached_continuation = buffer.size
             buffer << this_line
           else
@@ -1339,8 +1485,19 @@ class Lexer
             # for all other lists, has_text is always true
             # in this block, we have to see whether we stay in the list
             if has_text
+              # TODO any way to combine this with the check after skipping blank lines?
+              if is_sibling_list_item?(this_line, list_type, sibling_trait)
+                break
+              elsif nested_list_type = NESTABLE_LIST_CONTEXTS.detect {|ctx| ListRxMap[ctx] =~ this_line }
+                buffer << this_line
+                within_nested_list = true
+                if nested_list_type == :dlist && $~[3].nil_or_empty?
+                  # get greedy again
+                  has_text = false
+                end
               # slurp up any literal paragraph offset by blank lines
-              if this_line.match(REGEXP[:lit_par])
+              # NOTE we have to check for indented list items first
+              elsif LiteralParagraphRx =~ this_line
                 reader.unshift_line this_line
                 buffer.concat reader.read_lines_until(
                     :preserve_last_line => true,
@@ -1350,16 +1507,6 @@ class Lexer
                   # so we need to make sure we don't slurp up a legitimate sibling
                   list_type == :dlist && is_sibling_list_item?(line, list_type, sibling_trait)
                 }
-              # TODO any way to combine this with the check after skipping blank lines?
-              elsif is_sibling_list_item?(this_line, list_type, sibling_trait)
-                break
-              elsif nested_list_type = NESTABLE_LIST_CONTEXTS.detect {|ctx| this_line.match(REGEXP[ctx]) }
-                buffer << this_line
-                within_nested_list = true
-                if nested_list_type == :dlist && $~[3].to_s.empty?
-                  # get greedy again
-                  has_text = false
-                end
               else
                 break
               end
@@ -1371,10 +1518,10 @@ class Lexer
             end
           end
         else
-          has_text = true if !this_line.chomp.empty?
-          if nested_list_type = (within_nested_list ? [:dlist] : NESTABLE_LIST_CONTEXTS).detect {|ctx| this_line.match(REGEXP[ctx]) }
+          has_text = true if !this_line.empty?
+          if nested_list_type = (within_nested_list ? [:dlist] : NESTABLE_LIST_CONTEXTS).detect {|ctx| ListRxMap[ctx] =~ this_line }
             within_nested_list = true
-            if nested_list_type == :dlist && $~[3].to_s.empty?
+            if nested_list_type == :dlist && $~[3].nil_or_empty?
               # get greedy again
               has_text = false
             end
@@ -1385,23 +1532,21 @@ class Lexer
       this_line = nil
     end
 
-    reader.unshift_line this_line if !this_line.nil?
+    reader.unshift_line this_line if this_line
 
     if detached_continuation
       buffer.delete_at detached_continuation
     end
 
     # strip trailing blank lines to prevent empty blocks
-    buffer.pop while !buffer.empty? && buffer.last.chomp.empty?
+    buffer.pop while !buffer.empty? && buffer[-1].empty?
 
     # We do need to replace the optional trailing continuation
     # a blank line would have served the same purpose in the document
-    if !buffer.empty? && buffer.last.chomp == LIST_CONTINUATION
-      buffer.pop
-    end
+    buffer.pop if !buffer.empty? && buffer[-1] == LIST_CONTINUATION
 
-    #puts "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.join}<BUFFER"
-    #puts "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.inspect}<BUFFER"
+    #warn "BUFFER[#{list_type},#{sibling_trait}]>#{buffer * EOL}<BUFFER"
+    #warn "BUFFER[#{list_type},#{sibling_trait}]>#{buffer.inspect}<BUFFER"
 
     buffer
   end
@@ -1416,35 +1561,37 @@ class Lexer
   # attributes - a Hash of attributes to assign to this section (default: {})
   def self.initialize_section(reader, parent, attributes = {})
     document = parent.document
-    sect_id, sect_title, sect_level, _ = parse_section_title(reader, document)
-    section = Section.new parent, sect_level, document.attributes.has_key?('numbered')
+    source_location = reader.cursor if document.sourcemap
+    sect_id, sect_reftext, sect_title, sect_level, _ = parse_section_title(reader, document)
+    attributes['reftext'] = sect_reftext if sect_reftext
+    section = Section.new parent, sect_level, document.attributes.has_key?('sectnums')
+    section.source_location = source_location if source_location
     section.id = sect_id
     section.title = sect_title
     # parse style, id and role from first positional attribute
     if attributes[1]
-      section.sectname, _ = parse_style_attribute(attributes, reader)
-      section.special = true
-      # HACK needs to be refactored so it's driven by config
-      if section.sectname == 'abstract' && document.doctype == 'book'
-        section.sectname = "sect1"
-        section.special = false
-        section.level = 1
-      # FIXME refactor to use assign_caption (also check requirements)
-      elsif section.sectname == 'appendix' &&
-          !attributes.has_key?('caption') &&
-          !document.attributes.has_key?('caption')
-        number = document.counter('appendix-number', 'A')
-        section.caption = "#{document.attributes['appendix-caption']} #{number}: "
-        Document::AttributeEntry.new('appendix-number', number).save_to(attributes)
+      style, _ = parse_style_attribute attributes, reader
+      # handle case where only id and/or role are given (e.g., #idname.rolename)
+      if style
+        section.sectname = style
+        section.special = true
+        # HACK needs to be refactored so it's driven by config
+        if section.sectname == 'abstract' && document.doctype == 'book'
+          section.sectname = 'sect1'
+          section.special = false
+          section.level = 1
+        end
+      else
+        section.sectname = %(sect#{section.level})
       end
     elsif sect_title.downcase == 'synopsis' && document.doctype == 'manpage'
       section.special = true
       section.sectname = 'synopsis'
     else
-      section.sectname = "sect#{section.level}"
+      section.sectname = %(sect#{section.level})
     end
 
-    if section.id.nil? && (id = attributes['id'])
+    if !section.id && (id = attributes['id'])
       section.id = id
     else
       # generate an id if one was not *embedded* in the heading line
@@ -1453,7 +1600,8 @@ class Lexer
     end
 
     if section.id
-      section.document.register(:ids, [section.id, section.title])
+      # TODO sub reftext
+      section.document.register(:ids, [section.id, (attributes['reftext'] || section.title)])
     end
     section.update_attributes(attributes)
     reader.skip_blank_lines
@@ -1466,7 +1614,7 @@ class Lexer
   #
   # line - the String line from under the section title.
   def self.section_level(line)
-    SECTION_LEVELS[line[0..0]]
+    SECTION_LEVELS[line.chr]
   end
 
   #--
@@ -1483,8 +1631,10 @@ class Lexer
   # returns the section level if the Reader is positioned at a section title,
   # false otherwise
   def self.is_next_line_section?(reader, attributes)
-    return false if !(val = attributes[1]).nil? && ['float', 'discrete'].include?(val)
-    return false if !reader.has_more_lines?
+    if !(val = attributes[1]).nil? && ((ord_0 = val[0].ord) == 100 || ord_0 == 102) && val =~ FloatingTitleStyleRx
+      return false
+    end
+    return false unless reader.has_more_lines?
     Compliance.underline_style_section_titles ? is_section_title?(*reader.peek_lines(2)) : is_section_title?(reader.peek_line)
   end
 
@@ -1516,9 +1666,9 @@ class Lexer
   end
 
   def self.is_single_line_section_title?(line1)
-    first_char = line1.nil? ? nil : line1[0..0]
+    first_char = line1 ? line1.chr : nil
     if (first_char == '=' || (Compliance.markdown_syntax && first_char == '#')) &&
-        (match = line1.match(REGEXP[:section_title]))
+        (match = AtxSectionRx.match(line1))
       single_line_section_level match[1]
     else
       false
@@ -1526,8 +1676,8 @@ class Lexer
   end
 
   def self.is_two_line_section_title?(line1, line2)
-    if !line1.nil? && !line2.nil? && SECTION_LEVELS.has_key?(line2[0..0]) &&
-        line2.match(REGEXP[:section_underline]) && line1.match(REGEXP[:section_name]) &&
+    if line1 && line2 && SECTION_LEVELS.has_key?(line2.chr) &&
+        line2 =~ SetextSectionLineRx && line1 =~ SetextSectionTitleRx &&
         # chomp so that a (non-visible) endline does not impact calculation
         (line_length(line1) - line_length(line2)).abs <= 1
       section_level line2
@@ -1547,9 +1697,9 @@ class Lexer
   # Examples
   #
   #   reader.lines
-  #   # => ["Foo\n", "~~~\n"]
+  #   # => ["Foo", "~~~"]
   #
-  #   title, level, id, single = parse_section_title(reader, document)
+  #   id, reftext, title, level, single = parse_section_title(reader, document)
   #
   #   title
   #   # => "Foo"
@@ -1561,9 +1711,9 @@ class Lexer
   #   # => false
   #
   #   line1
-  #   # => "==== Foo\n"
+  #   # => "==== Foo"
   #
-  #   title, level, id, single = parse_section_title(reader, document)
+  #   id, reftext, title, level, single = parse_section_title(reader, document)
   #
   #   title
   #   # => "Foo"
@@ -1574,8 +1724,8 @@ class Lexer
   #   single
   #   # => true
   #
-  # returns an Array of [String, Integer, String, Boolean], representing the
-  # id, title, level and line count of the Section, or nil.
+  # returns an Array of [String, String, Integer, String, Boolean], representing the
+  # id, reftext, title, level and line count of the Section, or nil.
   #
   #--
   # NOTE for efficiency, we don't reuse methods that check for a section title
@@ -1584,25 +1734,33 @@ class Lexer
     sect_id = nil
     sect_title = nil
     sect_level = -1
+    sect_reftext = nil
     single_line = true
 
-    first_char = line1[0..0]
+    first_char = line1.chr
     if (first_char == '=' || (Compliance.markdown_syntax && first_char == '#')) &&
-        (match = line1.match(REGEXP[:section_title]))
-      sect_id = match[3]
-      sect_title = match[2]
+        (match = AtxSectionRx.match(line1))
       sect_level = single_line_section_level match[1]
+      sect_title = match[2]
+      if sect_title.end_with?(']]') && (anchor_match = InlineSectionAnchorRx.match(sect_title))
+        if anchor_match[2].nil?
+          sect_title = anchor_match[1]
+          sect_id = anchor_match[3]
+          sect_reftext = anchor_match[4]
+        end
+      end
     elsif Compliance.underline_style_section_titles
-      line2 = reader.peek_line true
-      if !line2.nil? && SECTION_LEVELS.has_key?(line2[0..0]) && line2.match(REGEXP[:section_underline]) &&
-        (name_match = line1.match(REGEXP[:section_name])) &&
+      if (line2 = reader.peek_line(true)) && SECTION_LEVELS.has_key?(line2.chr) && line2 =~ SetextSectionLineRx &&
+        (name_match = SetextSectionTitleRx.match(line1)) &&
         # chomp so that a (non-visible) endline does not impact calculation
         (line_length(line1) - line_length(line2)).abs <= 1
-        if anchor_match = name_match[1].match(REGEXP[:anchor_embedded]) 
-          sect_id = anchor_match[2]
-          sect_title = anchor_match[1]
-        else
-          sect_title = name_match[1]
+        sect_title = name_match[1]
+        if sect_title.end_with?(']]') && (anchor_match = InlineSectionAnchorRx.match(sect_title))
+          if anchor_match[2].nil?
+            sect_title = anchor_match[1]
+            sect_id = anchor_match[3]
+            sect_reftext = anchor_match[4]
+          end
         end
         sect_level = section_level line2
         single_line = false
@@ -1612,7 +1770,7 @@ class Lexer
     if sect_level >= 0
       sect_level += document.attr('leveloffset', 0).to_i
     end
-    [sect_id, sect_title, sect_level, single_line]
+    [sect_id, sect_reftext, sect_title, sect_level, single_line]
   end
 
   # Public: Calculate the number of unicode characters in the line, excluding the endline
@@ -1621,7 +1779,7 @@ class Lexer
   #
   # returns the number of unicode characters in the line
   def self.line_length(line)
-    FORCE_UNICODE_LINE_LENGTH ? line.chomp.scan(/./u).length : line.chomp.length
+    FORCE_UNICODE_LINE_LENGTH ? line.scan(UnicodeCharScanRx).length : line.length
   end
 
   # Public: Consume and parse the two header lines (line 1 = author info, line 2 = revision info).
@@ -1634,7 +1792,8 @@ class Lexer
   #
   # Examples
   #
-  #  parse_header_metadata(Reader.new ["Author Name <author@example.org>\n", "v1.0, 2012-12-21: Coincide w/ end of world.\n"])
+  #  data = ["Author Name <author@example.org>\n", "v1.0, 2012-12-21: Coincide w/ end of world.\n"]
+  #  parse_header_metadata(Reader.new data, nil, :normalize => true)
   #  # => {'author' => 'Author Name', 'firstname' => 'Author', 'lastname' => 'Name', 'email' => 'author@example.org',
   #  #       'revnumber' => '1.0', 'revdate' => '2012-12-21', 'revremark' => 'Coincide w/ end of world.'}
   def self.parse_header_metadata(reader, document = nil)
@@ -1649,12 +1808,12 @@ class Lexer
       author_metadata = process_authors reader.read_line
 
       unless author_metadata.empty?
-        # apply header subs and assign to document
-        if !document.nil?
-          author_metadata.map do |key, val|
-            val = val.is_a?(String) ? document.apply_header_subs(val) : val
-            document.attributes[key] = val if !document.attributes.has_key?(key)
-            val
+        if document
+          # apply header subs and assign to document
+          author_metadata.each do |key, val|
+            unless document.attributes.has_key? key
+              document.attributes[key] = ((val.is_a? ::String) ? document.apply_header_subs(val) : val)
+            end
           end
 
           implicit_author = document.attributes['author']
@@ -1671,7 +1830,7 @@ class Lexer
 
       if reader.has_more_lines? && !reader.next_line_empty?
         rev_line = reader.read_line 
-        if match = rev_line.match(REGEXP[:revision_info])
+        if (match = RevisionInfoLineRx.match(rev_line))
           rev_metadata['revdate'] = match[2].strip
           rev_metadata['revnumber'] = match[1].rstrip unless match[1].nil?
           rev_metadata['revremark'] = match[3].rstrip unless match[3].nil?
@@ -1682,12 +1841,12 @@ class Lexer
       end
 
       unless rev_metadata.empty?
-        # apply header subs and assign to document
-        if !document.nil?
-          rev_metadata.map do |key, val|
-            val = document.apply_header_subs(val)
-            document.attributes[key] = val if !document.attributes.has_key?(key)
-            val
+        if document
+          # apply header subs and assign to document
+          rev_metadata.each do |key, val|
+            unless document.attributes.has_key? key
+              document.attributes[key] = document.apply_header_subs(val)
+            end
           end
         end
 
@@ -1700,7 +1859,7 @@ class Lexer
       reader.skip_blank_lines
     end
 
-    if !document.nil?
+    if document
       # process author attribute entries that override (or stand in for) the implicit author line
       author_metadata = nil
       if document.attributes.has_key?('author') &&
@@ -1713,21 +1872,21 @@ class Lexer
         author_metadata = process_authors author_line, true
       else
         authors = []
-        author_key = "author_#{authors.size + 1}"
+        author_key = %(author_#{authors.size + 1})
         while document.attributes.has_key? author_key
           authors << document.attributes[author_key]
-          author_key = "author_#{authors.size + 1}"
+          author_key = %(author_#{authors.size + 1})
         end
         if authors.size == 1
           # do not allow multiple, process as names only
-          author_metadata = process_authors authors.first, true, false
+          author_metadata = process_authors authors[0], true, false
         elsif authors.size > 1
           # allow multiple, process as names only
           author_metadata = process_authors authors.join('; '), true
         end
       end
 
-      unless author_metadata.nil?
+      if author_metadata
         document.attributes.update author_metadata
 
         # special case
@@ -1752,9 +1911,8 @@ class Lexer
   def self.process_authors(author_line, names_only = false, multiple = true)
     author_metadata = {}
     keys = ['author', 'authorinitials', 'firstname', 'middlename', 'lastname', 'email']
-    author_entries = multiple ? author_line.split(';').map(&:strip) : [author_line]
+    author_entries = multiple ? (author_line.split ';').map {|line| line.strip } : [author_line]
     author_entries.each_with_index do |author_entry, idx|
-      author_entry.strip!
       next if author_entry.empty?
       key_map = {}
       if idx.zero?
@@ -1763,7 +1921,7 @@ class Lexer
         end
       else
         keys.each do |key|
-          key_map[key.to_sym] = "#{key}_#{idx + 1}"
+          key_map[key.to_sym] = %(#{key}_#{idx + 1})
         end
       end
 
@@ -1771,7 +1929,7 @@ class Lexer
       if names_only
         # splitting on ' ' will collapse repeating spaces
         segments = author_entry.split(' ', 3)
-      elsif (match = author_entry.match(REGEXP[:author_info]))
+      elsif (match = AuthorInfoLineRx.match(author_entry))
         segments = match.to_a
         segments.shift
       end
@@ -1792,7 +1950,7 @@ class Lexer
         end
         author_metadata[key_map[:email]] = segments[3] unless names_only || segments[3].nil?
       else
-        author_metadata[key_map[:author]] = author_metadata[key_map[:firstname]] = fname = author_entry.strip.squeeze(' ')
+        author_metadata[key_map[:author]] = author_metadata[key_map[:firstname]] = fname = author_entry.strip.tr_s(' ', ' ')
         author_metadata[key_map[:authorinitials]] = fname[0, 1]
       end
 
@@ -1800,13 +1958,13 @@ class Lexer
       # only assign the _1 attributes if there are multiple authors
       if idx == 1
         keys.each do |key|
-          author_metadata["#{key}_1"] = author_metadata[key] if author_metadata.has_key? key
+          author_metadata[%(#{key}_1)] = author_metadata[key] if author_metadata.has_key? key
         end
       end
       if idx.zero?
         author_metadata['authors'] = author_metadata[key_map[:author]]
       else
-        author_metadata['authors'] = "#{author_metadata['authors']}, #{author_metadata[key_map[:author]]}"
+        author_metadata['authors'] = %(#{author_metadata['authors']}, #{author_metadata[key_map[:author]]})
       end
     end
 
@@ -1856,30 +2014,28 @@ class Lexer
   #
   # returns true if the line contains metadata, otherwise false
   def self.parse_block_metadata_line(reader, parent, attributes, options = {})
-    return false if !reader.has_more_lines?
+    return false unless reader.has_more_lines?
     next_line = reader.peek_line
-    if (commentish = next_line.start_with?('//')) && (match = next_line.match(REGEXP[:comment_blk]))
+    if (commentish = next_line.start_with?('//')) && (match = CommentBlockRx.match(next_line))
       terminator = match[0]
       reader.read_lines_until(:skip_first_line => true, :preserve_last_line => true, :terminator => terminator, :skip_processing => true)
-    elsif commentish && next_line.match(REGEXP[:comment])
+    elsif commentish && CommentLineRx =~ next_line
       # do nothing, we'll skip it
-    elsif !options[:text] && (match = next_line.match(REGEXP[:attr_entry]))
+    elsif !options[:text] && next_line.start_with?(':') && (match = AttributeEntryRx.match(next_line))
       process_attribute_entry(reader, parent, attributes, match)
-    elsif match = next_line.match(REGEXP[:anchor])
-      id, reftext = match[1].split(',')
-      attributes['id'] = id
-      # AsciiDoc always uses [id] as the reftext in HTML output,
-      # but I'd like to do better in Asciidoctor
-      #parent.document.register(:ids, id)
-      if reftext
-        attributes['reftext'] = reftext
-        parent.document.register(:ids, [id, reftext])
+    elsif (in_square_brackets = next_line.start_with?('[') && next_line.end_with?(']')) && (match = BlockAnchorRx.match(next_line))
+      unless match[1].nil_or_empty?
+        attributes['id'] = match[1]
+        # AsciiDoc always uses [id] as the reftext in HTML output,
+        # but I'd like to do better in Asciidoctor
+        # registration is deferred until the block or section is processed
+        attributes['reftext'] = match[2] unless match[2].nil?
       end
-    elsif match = next_line.match(REGEXP[:blk_attr_list])
+    elsif in_square_brackets && (match = BlockAttributeListRx.match(next_line))
       parent.document.parse_attributes(match[1], [], :sub_input => true, :into => attributes)
     # NOTE title doesn't apply to section, but we need to stash it for the first block
     # TODO should issue an error if this is found above the document title
-    elsif !options[:text] && (match = next_line.match(REGEXP[:blk_title]))
+    elsif !options[:text] && (match = BlockTitleRx.match(next_line))
       attributes['title'] = match[1]
     else
       return false
@@ -1898,25 +2054,26 @@ class Lexer
   end
 
   def self.process_attribute_entry(reader, parent, attributes = nil, match = nil)
-    match ||= reader.has_more_lines? ? reader.peek_line.match(REGEXP[:attr_entry]) : nil
+    match ||= (reader.has_more_lines? ? AttributeEntryRx.match(reader.peek_line) : nil)
     if match
       name = match[1]
-      value = match[2].nil? ? '' : match[2]
-      if value.end_with? LINE_BREAK
-        value.chop!.rstrip!
-        while reader.advance
-          next_line = reader.peek_line.strip
-          break if next_line.empty?
-          if next_line.end_with? LINE_BREAK
-            value = "#{value} #{next_line.chop.rstrip}"
-          else
-            value = "#{value} #{next_line}"
-            break
+      unless (value = match[2] || '').empty?
+        if value.end_with?(line_continuation = LINE_CONTINUATION) ||
+            value.end_with?(line_continuation = LINE_CONTINUATION_LEGACY)
+          value = value.chop.rstrip
+          while reader.advance
+            break if (next_line = reader.peek_line.strip).empty?
+            if (keep_open = next_line.end_with? line_continuation)
+              next_line = next_line.chop.rstrip
+            end
+            separator = (value.end_with? LINE_BREAK) ? EOL : ' '
+            value = %(#{value}#{separator}#{next_line})
+            break unless keep_open
           end
         end
       end
 
-      store_attribute(name, value, parent.nil? ? nil : parent.document, attributes)
+      store_attribute(name, value, (parent ? parent.document : nil), attributes)
       true
     else
       false
@@ -1932,6 +2089,7 @@ class Lexer
   #
   # returns a 2-element array containing the attribute name and value
   def self.store_attribute(name, value, doc = nil, attrs = nil)
+    # TODO move processing of attribute value to utility method
     if name.end_with?('!')
       # a nil value signals the attribute should be deleted (undefined)
       value = nil
@@ -1944,11 +2102,25 @@ class Lexer
 
     name = sanitize_attribute_name(name)
     accessible = true
-    unless doc.nil?
-      accessible = value.nil? ? doc.delete_attribute(name) : doc.set_attribute(name, value)
+    if doc
+      # alias numbered attribute to sectnums
+      if name == 'numbered'
+        name = 'sectnums'
+      # support relative leveloffset values
+      elsif name == 'leveloffset'
+        if value
+          case value.chr
+          when '+'
+            value = ((doc.attr 'leveloffset', 0).to_i + (value[1..-1] || 0).to_i).to_s
+          when '-'
+            value = ((doc.attr 'leveloffset', 0).to_i - (value[1..-1] || 0).to_i).to_s
+          end
+        end
+      end
+      accessible = value ? doc.set_attribute(name, value) : doc.delete_attribute(name)
     end
 
-    unless !accessible || attrs.nil?
+    if accessible && attrs
       Document::AttributeEntry.new(name, value).save_to(attrs)
     end
 
@@ -1998,12 +2170,12 @@ class Lexer
   # Examples
   #
   #  marker = 'B.'
-  #  Lexer::resolve_ordered_list_marker(marker, 1, true)
+  #  Parser.resolve_ordered_list_marker(marker, 1, true)
   #  # => 'A.'
   #
   # Returns the String of the first marker in this number series 
   def self.resolve_ordered_list_marker(marker, ordinal = 0, validate = false, reader = nil)
-    number_style = ORDERED_LIST_STYLES.detect {|s| marker.match(ORDERED_LIST_MARKER_PATTERNS[s]) }
+    number_style = ORDERED_LIST_STYLES.detect {|s| OrderedListMarkerRxMap[s] =~ marker }
     expected = actual = nil
     case number_style
       when :arabic
@@ -2041,7 +2213,7 @@ class Lexer
     end
 
     if validate && expected != actual
-      warn "asciidoctor: WARNING: #{reader.line_info}: list item index: expected #{expected}, got #{actual}"
+      warn %(asciidoctor: WARNING: #{reader.line_info}: list item index: expected #{expected}, got #{actual})
     end
 
     marker
@@ -2057,15 +2229,15 @@ class Lexer
   # Returns a Boolean indicating whether this line is a sibling list item given
   # the criteria provided
   def self.is_sibling_list_item?(line, list_type, sibling_trait)
-    if sibling_trait.is_a?(Regexp)
+    if sibling_trait.is_a? ::Regexp
       matcher = sibling_trait
       expected_marker = false
     else
-      matcher = REGEXP[list_type]
+      matcher = ListRxMap[list_type]
       expected_marker = sibling_trait
     end
 
-    if m = line.match(matcher)
+    if (m = matcher.match(line))
       if expected_marker
         expected_marker == resolve_list_marker(list_type, m[1])
       else
@@ -2085,8 +2257,10 @@ class Lexer
   # returns an instance of Asciidoctor::Table parsed from the provided reader
   def self.next_table(table_reader, parent, attributes)
     table = Table.new(parent, attributes)
-    table.title = attributes.delete('title') if attributes.has_key?('title')
-    table.assign_caption attributes.delete('caption')
+    if (attributes.has_key? 'title')
+      table.title = attributes.delete 'title'
+      table.assign_caption attributes.delete('caption')
+    end
 
     if attributes.has_key? 'cols'
       table.create_columns(parse_col_specs(attributes['cols']))
@@ -2104,7 +2278,7 @@ class Lexer
       line = table_reader.read_line
 
       if skipped == 0 && loop_idx.zero? && !attributes.has_key?('options') &&
-          !(next_line = table_reader.peek_line).nil? && next_line == ::Asciidoctor::EOL
+          !(next_line = table_reader.peek_line).nil? && next_line.empty?
         table.has_header_option = true
         table.set_option 'header'
       end
@@ -2115,7 +2289,7 @@ class Lexer
           # push an empty cell spec if boundary at start of line
           parser_ctx.close_open_cell
         else
-          next_cell_spec, line = parse_cell_spec(line, :start)
+          next_cell_spec, line = parse_cell_spec(line, :start, parser_ctx.delimiter)
           # if the cell spec is not null, then we're at a cell boundary
           if !next_cell_spec.nil?
             parser_ctx.close_open_cell next_cell_spec
@@ -2125,8 +2299,10 @@ class Lexer
         end
       end
 
-      while !line.empty?
-        if m = parser_ctx.match_delimiter(line)
+      seen = false
+      while !seen || !line.empty?
+        seen = true
+        if (m = parser_ctx.match_delimiter(line))
           if parser_ctx.format == 'csv'
             if parser_ctx.buffer_has_unclosed_quotes?(m.pre_match)
               # throw it back, it's too small
@@ -2153,7 +2329,7 @@ class Lexer
         else
           # no other delimiters to see here
           # suck up this line into the buffer and move on
-          parser_ctx.buffer = %(#{parser_ctx.buffer}#{line})
+          parser_ctx.buffer = %(#{parser_ctx.buffer}#{line}#{EOL})
           # QUESTION make stripping endlines in csv data an option? (unwrap-option?)
           if parser_ctx.format == 'csv'
             parser_ctx.buffer = %(#{parser_ctx.buffer.rstrip} )
@@ -2200,27 +2376,24 @@ class Lexer
   # returns a Hash of attributes that specify how to format
   # and layout the cells in the table.
   def self.parse_col_specs(records)
-    specs = []
-
-    # check for deprecated syntax
-    if m = records.match(REGEXP[:digits])
-      1.upto(m[0].to_i) {
-        specs << {'width' => 1}
-      }
-      return specs
+    # check for deprecated syntax: single number, equal column spread
+    # REVIEW could use records == records.to_i.to_s instead of regexp
+    if DigitsRx =~ records
+      return ::Array.new(records.to_i) { { 'width' => 1 } }
     end
 
+    specs = []
     records.split(',').each {|record|
       # TODO might want to use scan rather than this mega-regexp
-      if m = record.match(REGEXP[:table_colspec])
+      if (m = ColumnSpecRx.match(record))
         spec = {}
         if m[2]
           # make this an operation
           colspec, rowspec = m[2].split '.'
-          if !colspec.to_s.empty? && Table::ALIGNMENTS[:h].has_key?(colspec)
+          if !colspec.nil_or_empty? && Table::ALIGNMENTS[:h].has_key?(colspec)
             spec['halign'] = Table::ALIGNMENTS[:h][colspec]
           end
-          if !rowspec.to_s.empty? && Table::ALIGNMENTS[:v].has_key?(rowspec)
+          if !rowspec.nil_or_empty? && Table::ALIGNMENTS[:v].has_key?(rowspec)
             spec['valign'] = Table::ALIGNMENTS[:v][rowspec]
           end
         end
@@ -2248,47 +2421,65 @@ class Lexer
   #
   # The cell specs dictate the cell's alignments, styles or filters,
   # colspan, rowspan and/or repeating content.
+  #
+  # The default spec when pos == :end is {} since we already know we're at a
+  # delimiter. When pos == :start, we *may* be at a delimiter, nil indicates
+  # we're not.
   # 
   # returns the Hash of attributes that indicate how to layout
   # and style this cell in the table.
-  def self.parse_cell_spec(line, pos = :start)
-    # the default for the end pos it {} since we
-    # know we're at a delimiter; when the pos
-    # is start, we *may* be at a delimiter and
-    # nil indicates we're not
-    spec = (pos == :end ? {} : nil)
-    rest = line
-
-    if m = line.match(REGEXP[:table_cellspec][pos]) 
-      spec = {}
-      return [spec, line] if m[0].chomp.empty?
-      rest = (pos == :start ? m.post_match : m.pre_match)
-      if m[1]
-        colspec, rowspec = m[1].split '.'
-        colspec = colspec.to_s.empty? ? 1 : colspec.to_i
-        rowspec = rowspec.to_s.empty? ? 1 : rowspec.to_i
-        if m[2] == '+'
-          spec['colspan'] = colspec unless colspec == 1
-          spec['rowspan'] = rowspec unless rowspec == 1
-        elsif m[2] == '*'
-          spec['repeatcol'] = colspec unless colspec == 1
+  def self.parse_cell_spec(line, pos = :start, delimiter = nil)
+    m = nil
+    rest = ''
+
+    case pos
+    when :start
+      if line.include? delimiter
+        spec_part, rest = line.split delimiter, 2
+        if (m = CellSpecStartRx.match spec_part)
+          return [{}, rest] if m[0].empty?
+        else
+          return [nil, line]
         end
+      else
+        return [nil, line]
       end
-      
-      if m[3]
-        colspec, rowspec = m[3].split '.'
-        if !colspec.to_s.empty? && Table::ALIGNMENTS[:h].has_key?(colspec)
-          spec['halign'] = Table::ALIGNMENTS[:h][colspec]
-        end
-        if !rowspec.to_s.empty? && Table::ALIGNMENTS[:v].has_key?(rowspec)
-          spec['valign'] = Table::ALIGNMENTS[:v][rowspec]
-        end
+    when :end
+      if (m = CellSpecEndRx.match line)
+        # NOTE return the line stripped of trailing whitespace if no cellspec is found in this case
+        return [{}, line.rstrip] if m[0].lstrip.empty?
+        rest = m.pre_match
+      else
+        return [{}, line]
       end
+    end
 
-      if m[4] && Table::TEXT_STYLES.has_key?(m[4])
-        spec['style'] = Table::TEXT_STYLES[m[4]]
+    spec = {}
+    if m[1]
+      colspec, rowspec = m[1].split '.'
+      colspec = colspec.nil_or_empty? ? 1 : colspec.to_i
+      rowspec = rowspec.nil_or_empty? ? 1 : rowspec.to_i
+      if m[2] == '+'
+        spec['colspan'] = colspec unless colspec == 1
+        spec['rowspan'] = rowspec unless rowspec == 1
+      elsif m[2] == '*'
+        spec['repeatcol'] = colspec unless colspec == 1
+      end
+    end
+    
+    if m[3]
+      colspec, rowspec = m[3].split '.'
+      if !colspec.nil_or_empty? && Table::ALIGNMENTS[:h].has_key?(colspec)
+        spec['halign'] = Table::ALIGNMENTS[:h][colspec]
       end
-    end 
+      if !rowspec.nil_or_empty? && Table::ALIGNMENTS[:v].has_key?(rowspec)
+        spec['valign'] = Table::ALIGNMENTS[:v][rowspec]
+      end
+    end
+
+    if m[4] && Table::TEXT_STYLES.has_key?(m[4])
+      spec['style'] = Table::TEXT_STYLES[m[4]]
+    end
 
     [spec, rest]
   end
@@ -2321,10 +2512,7 @@ class Lexer
     original_style = attributes['style']
     raw_style = attributes[1]
     # NOTE spaces are not allowed in shorthand, so if we find one, this ain't shorthand
-    if !raw_style || raw_style.include?(' ')
-      attributes['style'] = raw_style
-      [raw_style, original_style]
-    else
+    if raw_style && !raw_style.include?(' ') && Compliance.shorthand_property_syntax
       type = :style
       collector = []
       parsed = {}
@@ -2332,7 +2520,7 @@ class Lexer
       save_current = lambda {
         if collector.empty?
           if type != :style
-            warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} invalid empty #{type} detected in style attribute"
+            warn %(asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} invalid empty #{type} detected in style attribute)
           end
         else
           case type
@@ -2341,7 +2529,7 @@ class Lexer
             parsed[type].push collector.join
           when :id
             if parsed.has_key? :id
-              warn "asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} multiple ids detected in style attribute"
+              warn %(asciidoctor: WARNING:#{reader.nil? ? nil : " #{reader.prev_line_info}:"} multiple ids detected in style attribute)
             end
             parsed[type] = collector.join
           else
@@ -2351,7 +2539,7 @@ class Lexer
         end
       }
 
-      raw_style.split('').each do |c|
+      raw_style.each_char do |c|
         if c == '.' || c == '#' || c == '%'
           save_current.call
           case c
@@ -2389,7 +2577,7 @@ class Lexer
 
         if parsed.has_key? :option
           (options = parsed[:option]).each do |option|
-            attributes["#{option}-option"] = ''
+            attributes[%(#{option}-option)] = ''
           end
           if (existing_opts = attributes['options'])
             attributes['options'] = (options + existing_opts.split(',')) * ',' 
@@ -2400,6 +2588,9 @@ class Lexer
       end
 
       [parsed_style, original_style]
+    else
+      attributes['style'] = raw_style
+      [raw_style, original_style]
     end
   end
 
@@ -2427,13 +2618,13 @@ class Lexer
   #       end
   #   EOS
   #
-  #   source.lines.entries
-  #   # => ["    def names\n", "      @names.split ' '\n", "    end\n"]
+  #   source.split("\n")
+  #   # => ["    def names", "      @names.split ' '", "    end"]
   #
-  #   Lexer.reset_block_indent(source.lines.entries)
-  #   # => ["def names\n", "  @names.split ' '\n", "end\n"]
+  #   Parser.reset_block_indent(source.split "\n")
+  #   # => ["def names", "  @names.split ' '", "end"]
   #
-  #   puts Lexer.reset_block_indent(source.lines.entries).join
+  #   puts Parser.reset_block_indent(source.split "\n") * "\n"
   #   # => def names
   #   # =>   @names.split ' '
   #   # => end
@@ -2442,7 +2633,7 @@ class Lexer
   #--
   # FIXME refactor gsub matchers into compiled regex
   def self.reset_block_indent!(lines, indent = 0)
-    return if indent.nil? || lines.empty?
+    return if !indent || lines.empty?
 
     tab_detected = false
     # TODO make tab size configurable
@@ -2450,10 +2641,10 @@ class Lexer
     # strip leading block indent
     offsets = lines.map do |line|
       # break if the first char is non-whitespace
-      break [] unless line.chomp[0..0].lstrip.empty?
-      if line.include? "\t"
+      break [] unless line.chr.lstrip.empty?
+      if line.include? TAB
         tab_detected = true
-        line = line.gsub("\t", tab_expansion)
+        line = line.gsub(TAB_PATTERN, tab_expansion)
       end
       if (flush_line = line.lstrip).empty?
         nil
@@ -2467,8 +2658,8 @@ class Lexer
     unless offsets.empty? || (offsets = offsets.compact).empty?
       if (offset = offsets.min) > 0
         lines.map! {|line|
-          line = line.gsub("\t", tab_expansion) if tab_detected
-          line[offset..-1] || "\n"
+          line = line.gsub(TAB_PATTERN, tab_expansion) if tab_detected
+          line[offset..-1].to_s
         }
       end
     end
@@ -2498,7 +2689,7 @@ class Lexer
   #   sanitize_attribute_name('Foo 3 #-Billy')
   #   => 'foo3-billy'
   def self.sanitize_attribute_name(name)
-    name.gsub(REGEXP[:illegal_attr_name_chars], '').downcase
+    name.gsub(InvalidAttributeNameCharsRx, '').downcase
   end
 
   # Internal: Converts a Roman numeral to an integer value.