asciidoctor 0.1.3 → 0.1.4

data/lib/asciidoctor/{list_item.rb → list.rb}

@@ -1,4 +1,28 @@
 module Asciidoctor
+# Public: Methods for managing AsciiDoc lists (ordered, unordered and labeled lists)
+class List < AbstractBlock
+
+  # Public: Create alias for blocks
+  alias :items :blocks
+  alias :items? :blocks?
+
+  def initialize(parent, context)
+    super(parent, context)
+  end
+
+  # Public: Get the items in this list as an Array
+  def content
+    @blocks
+  end
+
+  def render
+    result = super
+    @document.callouts.next_list if @context == :colist
+    result
+  end
+
+end
+
 # Public: Methods for managing items for AsciiDoc olists, ulist, and dlists.
 class ListItem < AbstractBlock
 
@@ -20,12 +44,7 @@ class ListItem < AbstractBlock
   end
 
   def text
-    # this will allow the text to be processed
-    Block.new(self, nil, [@text]).content
-  end
-
-  def content
-    blocks? ? blocks.map {|b| b.render }.join : nil
+    apply_subs @text
   end
 
   # Public: Fold the first paragraph block into the text
@@ -40,23 +59,24 @@ class ListItem < AbstractBlock
   #
   # Returns nothing
   def fold_first(continuation_connects_first_block = false, content_adjacent = false)
-    if !blocks.empty? && blocks.first.is_a?(Block) &&
-        ((blocks.first.context == :paragraph && !continuation_connects_first_block) ||
-        ((content_adjacent || !continuation_connects_first_block) && blocks.first.context == :literal &&
-            blocks.first.attr('options', []).include?('listparagraph')))
+    if !(first_block = @blocks.first).nil? && first_block.is_a?(Block) &&
+        ((first_block.context == :paragraph && !continuation_connects_first_block) ||
+        ((content_adjacent || !continuation_connects_first_block) && first_block.context == :literal &&
+            first_block.option?('listparagraph')))
 
       block = blocks.shift
       unless @text.to_s.empty?
-        block.buffer.unshift("#@text\n")
+        block.lines.unshift("#@text\n")
       end
 
-      @text = block.buffer.join
+      @text = block.source
     end
     nil
   end
 
   def to_s
-    "#{super.to_s} - #@context [text:#@text, blocks:#{(@blocks || []).size}]"
+    "#@context [text:#@text, blocks:#{(@blocks || []).size}]"
   end
+
 end
 end

data/lib/asciidoctor/path_resolver.rb

@@ -312,7 +312,7 @@ class PathResolver
           elsif !recover
             raise SecurityError, "#{opts[:target_name] || 'path'} #{target} refers to location outside jail: #{jail} (disallowed in safe mode)"
           elsif !warned
-            puts "asciidoctor: WARNING: #{opts[:target_name] || 'path'} has illegal reference to ancestor of jail, auto-recovering"
+            warn "asciidoctor: WARNING: #{opts[:target_name] || 'path'} has illegal reference to ancestor of jail, auto-recovering"
             warned = true
           end
         else
@@ -339,9 +339,14 @@ class PathResolver
   def web_path(target, start = nil)
     target = posixfy(target)
     start = posixfy(start)
+    uri_prefix = nil
 
     unless is_web_root?(target) || start.empty?
       target = "#{start}#{SLASH}#{target}"
+      if target.include?(':') && target.match(Asciidoctor::REGEXP[:uri_sniff])
+        uri_prefix = $~[0]
+        target = target[uri_prefix.length..-1]
+      end
     end
 
     target_segments, target_root, _ = partition_path(target, true)
@@ -360,7 +365,28 @@ class PathResolver
       accum
     end
 
-    join_path resolved_segments, target_root
+    if uri_prefix.nil?
+      join_path resolved_segments, target_root
+    else
+      "#{uri_prefix}#{join_path resolved_segments, target_root}"
+    end
+  end
+
+  # Public: Calculate the relative path to this absolute filename from the specified base directory
+  #
+  # If either the filename or the base_directory are not absolute paths, no work is done.
+  #
+  # filename       - An absolute file name as a String
+  # base_directory - An absolute base directory as a String
+  #
+  # Return the relative path String of the filename calculated from the base directory
+  def relative_path(filename, base_directory)
+    if (is_root? filename) && (is_root? base_directory)
+      offset = base_directory.chomp(@file_separator).length + 1
+      filename[offset..-1]
+    else
+      filename
+    end
   end
 end
 end

data/lib/asciidoctor/reader.rb

@@ -1,90 +1,284 @@
 module Asciidoctor
 # Public: Methods for retrieving lines from AsciiDoc source files
 class Reader
+  class Cursor
+    attr_accessor :file
+    attr_accessor :dir
+    attr_accessor :path
+    attr_accessor :lineno
+  
+    def initialize file, dir = nil, path = nil, lineno = nil
+      @file = file
+      @dir = dir
+      @path = path
+      @lineno = lineno
+    end
 
-  # Public: Get the document source as a String Array of lines.
-  attr_reader :source
+    def line_info
+      %(#{path}: line #{lineno})
+    end
+  end
+
+  attr_reader :file
+  attr_reader :dir
+  attr_reader :path
 
   # Public: Get the 1-based offset of the current line.
   attr_reader :lineno
 
-  # Public: Initialize the Reader object.
+  # Public: Get the document source as a String Array of lines.
+  attr_reader :source_lines
+
+  # Public: Control whether lines are processed using Reader#process_line on first visit (default: true)
+  attr_accessor :process_lines
+
+  # Public: Initialize the Reader object
+  def initialize data = nil, cursor = nil
+    if cursor.nil?
+      @file = @dir = nil
+      @path = '<stdin>'
+      @lineno = 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+    elsif cursor.is_a? String
+      @file = cursor
+      @dir = File.dirname @file
+      @path = File.basename @file
+      @lineno = 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+    else
+      @file = cursor.file
+      @dir = cursor.dir
+      @path = cursor.path || '<stdin>'
+      unless @file.nil?
+        if @dir.nil?
+          # REVIEW might to look at this assignment closer
+          @dir = File.dirname @file
+          @dir = nil if @dir == '.' # right?
+        end
+
+        if cursor.path.nil?
+          @path = File.basename @file
+        end
+      end
+      @lineno = cursor.lineno || 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+    end
+    @lines = data.nil? ? [] : (prepare_lines data)
+    @source_lines = @lines.dup
+    @eof = @lines.empty?
+    @look_ahead = 0
+    @process_lines = true
+    @unescape_next_line = false
+  end
+
+  # Internal: Prepare the lines from the provided data
+  #
+  # This method strips whitespace from the end of every line of
+  # the source data and appends a LF (i.e., Unix endline). This
+  # whitespace substitution is very important to how Asciidoctor
+  # works.
   #
-  # data       - The Array of Strings holding the Asciidoc source document. The
-  #              original instance of this Array is not modified (default: nil)
-  # document   - The document with which this reader is associated. Used to access
-  #              document attributes (default: nil)
-  # preprocess - A flag indicating whether to run the preprocessor on these lines.
-  #              Only enable for the outer-most Reader. If this argument is true,
-  #              a Document object must also be supplied.
-  #              (default: false)
-  # block      - A block that can be used to retrieve external Asciidoc
-  #              data to include in this document.
+  # Any leading or trailing blank lines are also removed.
   #
-  # Examples
+  # The normalized lines are assigned to the @lines instance variable.
   #
-  #   data   = File.readlines(filename)
-  #   reader = Asciidoctor::Reader.new data
-  def initialize(data = nil, document = nil, preprocess = false, &block)
-    data = [] if data.nil?
-    # TODO use Struct to track file/lineno info; track as file changes; offset for sub-readers
-    @lineno = 0
-    if !preprocess
-      @lines = data.is_a?(String) ? data.lines.entries : data.dup
-      @preprocess_source = false
-    elsif !data.empty?
-      # NOTE we assume document is not nil!
-      @document = document
-      @preprocess_source = true
-      @include_block = block_given? ? block : nil
-      normalize_data(data.is_a?(String) ? data.lines.entries : data)
+  # data - A String Array of input data to be normalized
+  # opts - A Hash of options to control what cleansing is done
+  #
+  # Returns The String lines extracted from the data
+  def prepare_lines data, opts = {}
+    data.is_a?(String) ? data.each_line.to_a : data.dup
+  end
+
+  # Internal: Processes a previously unvisited line
+  #
+  # By default, this method marks the line as processed
+  # by incrementing the look_ahead counter and returns
+  # the line unmodified.
+  #
+  # Returns The String line the Reader should make available to the next
+  # invocation of Reader#read_line or nil if the Reader should drop the line,
+  # advance to the next line and process it.
+  def process_line line
+    @look_ahead += 1 if @process_lines
+    line
+  end
+
+  # Public: Check whether there are any lines left to read.
+  #
+  # If a previous call to this method resulted in a value of false,
+  # immediately returned the cached value. Otherwise, delegate to
+  # peek_line to determine if there is a next line available.
+  #
+  # Returns True if there are more lines, False if there are not.
+  def has_more_lines?
+    !(@eof || (@eof = peek_line.nil?))
+  end
+
+  # Public: Peek at the next line and check if it's empty (i.e., whitespace only)
+  #
+  # This method Does not consume the line from the stack.
+  #
+  # Returns True if the there are no more lines or if the next line is empty
+  def next_line_empty?
+    (line = peek_line).nil? || line.chomp.empty?
+  end
+
+  # Public: Peek at the next line of source data. Processes the line, if not
+  # already marked as processed, but does not consume it.
+  #
+  # This method will probe the reader for more lines. If there is a next line
+  # that has not previously been visited, the line is passed to the
+  # Reader#preprocess_line method to be initialized. This call gives
+  # sub-classess the opportunity to do preprocessing. If the return value of
+  # the Reader#process_line is nil, the data is assumed to be changed and
+  # Reader#peek_line is invoked again to perform further processing.
+  #
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: false)
+  #
+  # Returns the next line of the source data as a String if there are lines remaining.
+  # Returns nil if there is no more data.
+  def peek_line direct = false
+    if direct || @look_ahead > 0
+      @unescape_next_line ? @lines.first[1..-1] : @lines.first
+    elsif @eof || @lines.empty?
+      @eof = true
+      @look_ahead = 0
+      nil
     else
-      @lines = []
-      @preprocess_source = false
+      # FIXME the problem with this approach is that we aren't
+      # retaining the modified line (hence the @unescape_next_line tweak)
+      # perhaps we need a stack of proxy lines
+      if (line = process_line @lines.first).nil?
+        peek_line
+      else
+        line
+      end
     end
+  end
 
-    @source = @lines.dup
+  # Public: Peek at the next multiple lines of source data. Processes the lines, if not
+  # already marked as processed, but does not consume them.
+  #
+  # This method delegates to Reader#read_line to process and collect the line, then
+  # restores the lines to the stack before returning them. This allows the lines to
+  # be processed and marked as such so that subsequent reads will not need to process
+  # the lines again.
+  #
+  # num    - The Integer number of lines to peek.
+  # direct - A Boolean indicating whether processing should be disabled when reading lines
+  #
+  # Returns A String Array of the next multiple lines of source data, or an empty Array
+  # if there are no more lines in this Reader.
+  def peek_lines num = 1, direct = true
+    old_look_ahead = @look_ahead
+    result = []
+    (1..num).each do
+      if (line = read_line direct)
+        result << line
+      else
+        break
+      end
+    end
 
-    @next_line_preprocessed = false
-    @unescape_next_line = false
+    unless result.empty?
+      result.reverse_each {|line| unshift line }
+      @look_ahead = old_look_ahead if direct
+    end
 
-    @conditionals_stack = []
-    @skipping = false
-    @eof = false
+    result
   end
 
-  # Public: Get a copy of the remaining Array of String lines parsed from the source
-  def lines
-    @lines.nil? ? nil : @lines.dup
+  # Public: Get the next line of source data. Consumes the line returned.
+  #
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: false)
+  #
+  # Returns the String of the next line of the source data if data is present.
+  # Returns nil if there is no more data.
+  def read_line direct = false
+    if direct || @look_ahead > 0 || has_more_lines?
+      shift
+    else
+      nil
+    end
   end
 
-  # Public: Check whether there are any lines left to read.
+  # Public: Get the remaining lines of source data.
   #
-  # If preprocessing is enabled for this Reader, and there are lines remaining,
-  # the next line is preprocessed before checking whether there are more lines. 
+  # This method calls Reader#read_line repeatedly until all lines are consumed
+  # and returns the lines as a String Array. This method differs from
+  # Reader#lines in that it processes each line in turn, hence triggering
+  # any preprocessors implemented in sub-classes.
   #
-  # Returns true if @lines is empty, or false otherwise.
-  def has_more_lines?
-    if @eof || (@eof = @lines.empty?)
-      false
-    elsif @preprocess_source && !@next_line_preprocessed
-      preprocess_next_line.nil? ? false : !@lines.empty?
-    else
-      true
+  # Returns the lines read as a String Array
+  def read_lines
+    lines = []
+    while has_more_lines?
+      lines << read_line
     end
+    lines
   end
+  alias :readlines :read_lines
 
-  # Public: Check whether this reader is empty (contains no lines)
+  # Public: Get the remaining lines of source data joined as a String.
   #
-  # If preprocessing is enabled for this Reader, and there are lines remaining,
-  # the next line is preprocessed before checking whether there are more lines. 
+  # Delegates to Reader#read_lines, then joins the result.
   #
-  # Returns true if @lines is empty, otherwise false.
-  def empty?
-    !has_more_lines?
+  # Returns the lines read joined as a String
+  def read
+    read_lines.join
+  end
+
+  # Public: Advance to the next line by discarding the line at the front of the stack
+  #
+  # direct  - A Boolean flag to bypasses the check for more lines and immediately
+  #           returns the first element of the internal @lines Array. (default: true)
+  #
+  # returns a Boolean indicating whether there was a line to discard.
+  def advance direct = true
+    !(read_line direct).nil?
+  end
+
+  # Public: Push the String line onto the beginning of the Array of source data.
+  #
+  # Since this line was (assumed to be) previously retrieved through the
+  # reader, it is marked as seen.
+  #
+  # returns nil
+  def unshift_line line_to_restore
+    unshift line_to_restore
+    nil
+  end
+  alias :restore_line :unshift_line
+
+  # Public: Push an Array of lines onto the front of the Array of source data.
+  #
+  # Since these lines were (assumed to be) previously retrieved through the
+  # reader, they are marked as seen.
+  #
+  # Returns nil
+  def unshift_lines lines_to_restore
+    # QUESTION is it faster to use unshift(*lines_to_restore)?
+    lines_to_restore.reverse_each {|line| unshift line }
+    nil
   end
+  alias :restore_lines :unshift_lines
 
-  # Private: Strip off leading blank lines in the Array of lines.
+  # Public: Replace the current line with the specified line.
+  #
+  # Calls Reader#advance to consume the current line, then calls
+  # Reader#unshift to push the replacement onto the top of the
+  # line stack.
+  #
+  # replacement  - The String line to put in place of the line at the cursor.
+  #
+  # Returns nothing.
+  def replace_line replacement
+    advance
+    unshift replacement
+    nil
+  end
+
+  # Public: Strip off leading blank lines in the Array of lines.
   #
   # Examples
   #
@@ -99,76 +293,66 @@ class Reader
   #
   # Returns an Integer of the number of lines skipped
   def skip_blank_lines
-    skipped = 0
+    return 0 if eof?
+
+    num_skipped = 0
     # optimized code for shortest execution path
-    while !(next_line = get_line).nil?
+    while (next_line = peek_line)
       if next_line.chomp.empty?
-        skipped += 1
+        advance
+        num_skipped += 1
       else
-        unshift_line next_line
-        break
+        return num_skipped
       end
-    end 
+    end
 
-    skipped
+    num_skipped
   end
 
-  # Public: Consume consecutive lines containing line- or block-level comments.
-  #
-  # Returns the Array of lines that were consumed
+  # Public: Skip consecutive lines containing line comments and return them.
   #
   # Examples
   #   @lines
-  #   => ["// foo\n", "////\n", "foo bar\n", "////\n", "actual text\n"]
+  #   => ["// foo\n", "bar\n"]
   #
-  #   comment_lines = consume_comments
-  #   => ["// foo\n", "////\n", "foo bar\n", "////\n"]
+  #   comment_lines = skip_comment_lines
+  #   => ["// foo\n"]
   #
   #   @lines
-  #   => ["actual text\n"]
-  def consume_comments(options = {})
+  #   => ["bar\n"]
+  #
+  # Returns the Array of lines that were skipped
+  def skip_comment_lines opts = {}
+    return [] if eof?
+
     comment_lines = []
-    preprocess = options.fetch(:preprocess, true)
-    while !(next_line = get_line(preprocess)).nil?
-      if options[:include_blank_lines] && next_line.chomp.empty?
-        comment_lines << next_line
+    include_blank_lines = opts[:include_blank_lines]
+    while (next_line = peek_line)
+      if include_blank_lines && next_line.chomp.empty?
+        comment_lines << read_line
       elsif (commentish = next_line.start_with?('//')) && (match = next_line.match(REGEXP[:comment_blk]))
-        comment_lines << next_line
-        comment_lines.push(*(grab_lines_until(:terminator => match[0], :grab_last_line => true, :preprocess => false)))
+        comment_lines << read_line
+        comment_lines.push(*(read_lines_until(:terminator => match[0], :read_last_line => true, :skip_processing => true)))
       elsif commentish && next_line.match(REGEXP[:comment])
-        comment_lines << next_line
+        comment_lines << read_line
       else
-        # throw it back
-        unshift_line next_line
         break
       end
     end
 
     comment_lines
   end
-  alias :skip_comment_lines :consume_comments
 
-  # Public: Consume consecutive lines containing line comments.
-  #
-  # Returns the Array of lines that were consumed
-  #
-  # Examples
-  #   @lines
-  #   => ["// foo\n", "bar\n"]
-  #
-  #   comment_lines = consume_comments
-  #   => ["// foo\n"]
-  #
-  #   @lines
-  #   => ["bar\n"]
-  def consume_line_comments
+  # Public: Skip consecutive lines that are line comments and return them.
+  def skip_line_comments
+    return [] if eof?
+
     comment_lines = []
     # optimized code for shortest execution path
-    while !(next_line = get_line).nil?
+    while (next_line = peek_line)
       if next_line.match(REGEXP[:comment])
-        comment_lines << next_line
+        comment_lines << read_line
       else
-        unshift_line next_line
         break
       end
     end
@@ -176,122 +360,302 @@ class Reader
     comment_lines
   end
 
-  # Public: Get the next line of source data. Consumes the line returned.
+  # Public: Advance to the end of the reader, consuming all remaining lines
   #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
+  # Returns nothing.
+  def terminate
+    @lineno += @lines.size
+    @lines.clear
+    @eof = true
+    @look_ahead = 0
+    nil
+  end
+
+  # Public: Check whether this reader is empty (contains no lines)
   #
-  # Returns the String of the next line of the source data if data is present.
-  # Returns nil if there is no more data.
-  def get_line(preprocess = true)
-    if @eof || (@eof = @lines.empty?)
-      @next_line_preprocessed = true
-      nil
-    elsif preprocess && @preprocess_source &&
-        !@next_line_preprocessed && preprocess_next_line.nil?
-      @next_line_preprocessed = true
-      nil
+  # Returns true if there are no more lines to peek, otherwise false.
+  def eof?
+    !has_more_lines?
+  end
+  alias :empty? :eof?
+
+  # Public: Return all the lines from `@lines` until we (1) run out them,
+  #   (2) find a blank line with :break_on_blank_lines => true, or (3) find
+  #   a line for which the given block evals to true.
+  #
+  # options - an optional Hash of processing options:
+  #           * :break_on_blank_lines may be used to specify to break on
+  #               blank lines
+  #           * :skip_first_line may be used to tell the reader to advance
+  #               beyond the first line before beginning the scan
+  #           * :preserve_last_line may be used to specify that the String
+  #               causing the method to stop processing lines should be
+  #               pushed back onto the `lines` Array.
+  #           * :read_last_line may be used to specify that the String
+  #               causing the method to stop processing lines should be
+  #               included in the lines being returned
+  #
+  # Returns the Array of lines forming the next segment.
+  #
+  # Examples
+  #
+  #   reader = Reader.new ["First paragraph\n", "Second paragraph\n",
+  #                        "Open block\n", "\n", "Can have blank lines\n",
+  #                        "--\n", "\n", "In a different segment\n"]
+  #
+  #   reader.read_lines_until
+  #   => ["First paragraph\n", "Second paragraph\n", "Open block\n"]
+  def read_lines_until options = {}
+    result = []
+    advance if options[:skip_first_line]
+    if @process_lines && options[:skip_processing]
+      @process_lines = false
+      restore_process_lines = true
     else
-      @lineno += 1
-      @next_line_preprocessed = false
-      if @unescape_next_line
-        @unescape_next_line = false
-        @lines.shift[1..-1]
-      else
-        @lines.shift
+      restore_process_lines = false
+    end
+
+    has_block = block_given?
+    if (terminator = options[:terminator])
+      break_on_blank_lines = false
+      break_on_list_continuation = false
+      chomp_last_line = options.fetch :chomp_last_line, false
+    else
+      break_on_blank_lines = options[:break_on_blank_lines]
+      break_on_list_continuation = options[:break_on_list_continuation]
+      chomp_last_line = break_on_blank_lines
+    end
+    skip_line_comments = options[:skip_line_comments]
+    line_read = false
+    line_restored = false
+    
+    while (line = read_line)
+      finish = while true
+        break true if terminator && line.chomp == terminator
+        # QUESTION: can we get away with line.chomp.empty? here?
+        break true if break_on_blank_lines && line.chomp.empty?
+        if break_on_list_continuation && line_read && line.chomp == LIST_CONTINUATION
+          options[:preserve_last_line] = true
+          break true
+        end
+        break true if has_block && (yield line)
+        break false
+      end
+
+      if finish
+        if options[:read_last_line]
+          result << line
+          line_read = true
+        end
+        if options[:preserve_last_line]
+          restore_line line
+          line_restored = true
+        end
+        break
+      end
+
+      unless skip_line_comments && line.start_with?('//') && line.match(REGEXP[:comment])
+        result << line
+        line_read = true
       end
     end
+
+    if chomp_last_line && line_read
+      result << result.pop.chomp
+    end
+
+    if restore_process_lines
+      @process_lines = true
+      @look_ahead -= 1 if line_restored && terminator.nil?
+    end
+    result
   end
 
-  # Public: Advance to the next line by discarding the line at the front of the stack
+  # Internal: Shift the line off the stack and increment the lineno
+  def shift
+    @lineno += 1
+    @look_ahead -= 1 unless @look_ahead == 0
+    @lines.shift
+  end
+
+  # Internal: Restore the line to the stack and decrement the lineno
+  def unshift line
+    @lineno -= 1
+    @look_ahead += 1
+    @eof = false
+    @lines.unshift line
+  end
+
+  def cursor
+    Cursor.new @file, @dir, @path, @lineno
+  end
+
+  # Public: Get information about the last line read, including file name and line number.
   #
-  # Removes the line at the front of the stack without any processing.
+  # Returns A String summary of the last line read
+  def line_info
+    %(#{@path}: line #{@lineno})
+  end
+  alias :next_line_info :line_info
+
+  def prev_line_info
+    %(#{@path}: line #{@lineno - 1})
+  end
+
+  # Public: Get a copy of the remaining Array of String lines managed by this Reader
   #
-  # returns a boolean indicating whether there was a line to discard
-  def advance
-    @next_line_preprocessed = false
-    # we assume that we're advancing over a line of known content
-    if @eof || (@eof = @lines.empty?)
-      false
-    else
-      @lineno += 1
-      @lines.shift
-      true
-    end
+  # Returns A copy of the String Array of lines remaining in this Reader
+  def lines
+    @lines.dup
   end
 
-  # Public: Get the next line of source data. Does not consume the line returned.
+  # Public: Get a copy of the remaining lines managed by this Reader joined as a String
+  def string
+    @lines.join
+  end
+
+  # Public: Get the source lines for this Reader joined as a String
+  def source
+    @source_lines.join
+  end
+
+  # Public: Get a summary of this Reader.
   #
-  # preprocess - A Boolean flag indicating whether to evaluate preprocessing
-  #              directives (macros) before reading line (default: true)
   #
-  # Returns a String dup of the next line of the source data if data is present.
-  # Returns nil if there is no more data.
-  def peek_line(preprocess = true)
-    if !preprocess
-      # QUESTION do we need to dup?
-      @eof || (@eof = @lines.empty?) ? nil : @lines.first.dup
-    elsif has_more_lines?
-      # QUESTION do we need to dup?
-      @lines.first.dup
+  # Returns A string summary of this reader, which contains the path and line information
+  def to_s
+    line_info
+  end
+end
+
+# Public: Methods for retrieving lines from AsciiDoc source files, evaluating preprocessor
+# directives as each line is read off the Array of lines.
+class PreprocessorReader < Reader
+  attr_reader :include_stack
+  attr_reader :includes
+
+  # Public: Initialize the PreprocessorReader object
+  def initialize document, data = nil, cursor = nil
+    @document = document
+    super data, cursor
+    include_depth_default = document.attributes.fetch('max-include-depth', 64).to_i
+    include_depth_default = 0 if include_depth_default < 0
+    # track both absolute depth for comparing to size of include stack and relative depth for reporting
+    @maxdepth = {:abs => include_depth_default, :rel => include_depth_default}
+    @include_stack = []
+    @includes = (document.references[:includes] ||= [])
+    @skipping = false
+    @conditional_stack = []
+    @include_processors = nil
+  end
+
+  def prepare_lines data, opts = {}
+    if data.is_a?(String)
+      if ::Asciidoctor::FORCE_ENCODING
+        result = data.each_line.map {|line| "#{line.rstrip.force_encoding ::Encoding::UTF_8}#{::Asciidoctor::EOL}" }
+      else
+        result = data.each_line.map {|line| "#{line.rstrip}#{::Asciidoctor::EOL}" }
+      end
     else
-      nil
+      if ::Asciidoctor::FORCE_ENCODING
+        result = data.map {|line| "#{line.rstrip.force_encoding ::Encoding::UTF_8}#{::Asciidoctor::EOL}" }
+      else
+        result = data.map {|line| "#{line.rstrip}#{::Asciidoctor::EOL}" }
+      end
     end
-  end
 
-  # TODO document & test me!
-  def peek_lines(number = 1)
-    lines = []
-    idx = 0
-    (1..number).each do
-      if @preprocess_source && !@next_line_preprocessed
-        advanced = preprocess_next_line
-        break if advanced.nil? || @eof || (@eof = @lines.empty?)
-        idx = 0 if advanced
+    # QUESTION should this work for AsciiDoc table cell content? Currently it does not.
+    unless @document.nil? || !(@document.attributes.has_key? 'skip-front-matter')
+      if (front_matter = skip_front_matter! result)
+        @document.attributes['front-matter'] = front_matter.join.chomp
       end
-      break if idx >= @lines.size
-      # QUESTION do we need to dup?
-      lines << @lines[idx].dup
-      idx += 1
     end
-    lines
+
+    # QUESTION should we chomp last line? (with or without the condense flag?)
+    if opts.fetch(:condense, true)
+      result.shift && @lineno += 1 while !(first = result.first).nil? && first == ::Asciidoctor::EOL
+      result.pop while !(last = result.last).nil? && last == ::Asciidoctor::EOL
+    end
+
+    if (indent = opts.fetch(:indent, nil))
+      Lexer.reset_block_indent! result, indent.to_i
+    end
+
+    result
   end
 
-  # Internal: Preprocess the next line until the cursor is at a line of content
-  #
-  # Evaluate preprocessor macros on the next line, continuing to do so until
-  # the cursor arrives at a line of included content. That line is marked as
-  # preprocessed so that preprocessing is not performed multiple times.
-  #
-  # returns a Boolean indicating whether the cursor advanced, or nil if there
-  # are no more lines available.
-  def preprocess_next_line
-    # this return could be happening from a recursive call
-    return nil if @eof || (next_line = @lines.first).nil?
-    if next_line.include?('::') && (next_line.include?('if') || next_line.include?('endif')) && (match = next_line.match(REGEXP[:ifdef_macro]))
-      if next_line.start_with? '\\'
-        @next_line_preprocessed = true
+  def process_line line
+    return line unless @process_lines
+
+    if line.chomp.empty?
+      @look_ahead += 1
+      return ''
+    end
+
+    macroish = line.include?('::') && line.include?('[')
+    if macroish && line.include?('if') && (match = line.match(REGEXP[:ifdef_macro]))
+      # if escaped, mark as processed and return line unescaped
+      if line.start_with? '\\'
         @unescape_next_line = true
-        false
+        @look_ahead += 1
+        line[1..-1]
       else
-        preprocess_conditional_inclusion(*match.captures)
+        if preprocess_conditional_inclusion(*match.captures)
+          # move the pointer past the conditional line
+          advance
+          # treat next line as uncharted territory
+          nil
+        else
+          # the line was not a valid conditional line
+          # mark it as visited and return it
+          @look_ahead += 1
+          line
+        end
       end
     elsif @skipping
       advance
-      # skip over comment blocks, we don't want to process directives in them
-      skip_comment_lines :include_blank_lines => true, :preprocess => false
-      preprocess_next_line.nil? ? nil : true
-    elsif next_line.include?('include::') && match = next_line.match(REGEXP[:include_macro])
-      if next_line.start_with? '\\'
-        @next_line_preprocessed = true
+      nil 
+    elsif macroish && line.include?('include::') && (match = line.match(REGEXP[:include_macro]))
+      # if escaped, mark as processed and return line unescaped
+      if line.start_with? '\\'
         @unescape_next_line = true
-        false
+        @look_ahead += 1
+        line[1..-1]
       else
-        preprocess_include(match[1], match[2].strip)
+        # QUESTION should we strip whitespace from raw attributes in Substituters#parse_attributes? (check perf)
+        if preprocess_include match[1], match[2].strip
+          # peek again since the content has changed
+          nil
+        else
+          # the line was not a valid include line and is unchanged
+          # mark it as visited and return it
+          @look_ahead += 1
+          line
+        end
       end
     else
-      @next_line_preprocessed = true
-      false
+      # optimization to inline super
+      #super
+      @look_ahead += 1
+      line
+    end
+  end
+
+  # Public: Override the Reader#peek_line method to pop the include
+  # stack if the last line has been reached and there's at least
+  # one include on the stack.
+  #
+  # Returns the next line of the source data as a String if there are lines remaining
+  # in the current include context or a parent include context.
+  # Returns nil if there are no more lines remaining and the include stack is empty.
+  def peek_line direct = false
+    if (line = super)
+      line
+    elsif @include_stack.empty?
+      nil
+    else
+      pop_include
+      peek_line direct
     end
   end
 
@@ -314,37 +678,36 @@ class Reader
   #              Used for a single-line conditional block in the case of the ifdef or
   #              ifndef directives, and for the conditional expression for the ifeval directive.
   #
-  # returns a Boolean indicating whether the cursor advanced, or nil if there
-  # are no more lines available.
-  def preprocess_conditional_inclusion(directive, target, delimiter, text)
+  # returns a Boolean indicating whether the cursor should be advanced
+  def preprocess_conditional_inclusion directive, target, delimiter, text
     # must have a target before brackets if ifdef or ifndef
     # must not have text between brackets if endif
     # don't honor match if it doesn't meet this criteria
+    # QUESTION should we warn for these bogus declarations?
     if ((directive == 'ifdef' || directive == 'ifndef') && target.empty?) ||
         (directive == 'endif' && !text.nil?)
-      @next_line_preprocessed = true
       return false
     end
 
     if directive == 'endif'
-      stack_size = @conditionals_stack.size
+      stack_size = @conditional_stack.size
       if stack_size > 0
-        pair = @conditionals_stack.last
+        pair = @conditional_stack.last
         if target.empty? || target == pair[:target]
-          @conditionals_stack.pop
-          @skipping = @conditionals_stack.empty? ? false : @conditionals_stack.last[:skipping]
+          @conditional_stack.pop
+          @skipping = @conditional_stack.empty? ? false : @conditional_stack.last[:skipping]
         else
-          puts "asciidoctor: ERROR: line #{@lineno + 1}: mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[]"
+          warn "asciidoctor: ERROR: #{line_info}: mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[]"
         end
       else
-        puts "asciidoctor: ERROR: line #{@lineno + 1}: unmatched macro: endif::#{target}[]"
+        warn "asciidoctor: ERROR: #{line_info}: unmatched macro: endif::#{target}[]"
       end
-      advance
-      return preprocess_next_line.nil? ? nil : true
+      return true
     end
 
     skip = false
-    if !@skipping
+    unless @skipping
+      # QUESTION any way to wrap ifdef & ifndef logic up together?
       case directive
       when 'ifdef'
         case delimiter
@@ -374,32 +737,35 @@ class Reader
         # the text in brackets must match an expression
         # don't honor match if it doesn't meet this criteria
         if !target.empty? || !(expr_match = text.strip.match(REGEXP[:eval_expr]))
-          @next_line_preprocessed = true
           return false
         end
 
-        lhs = resolve_expr_val(expr_match[1])
+        lhs = resolve_expr_val expr_match[1]
+        # regex enforces a restrict set of math-related operations
         op = expr_match[2]
-        rhs = resolve_expr_val(expr_match[3])
+        rhs = resolve_expr_val expr_match[3]
 
-        skip = !lhs.send(op.to_sym, rhs)
+        skip = !(lhs.send op.to_sym, rhs)
       end
     end
-    advance
-    # single line conditional inclusion
-    if directive != 'ifeval' && !text.nil?
-      if !@skipping && !skip
-        unshift_line "#{text.rstrip}\n"
-        return true
-      end
+
     # conditional inclusion block
+    if directive == 'ifeval' || text.nil?
+      @skipping = true if skip
+      @conditional_stack << {:target => target, :skip => skip, :skipping => @skipping}
+    # single line conditional inclusion
     else
-      if !@skipping && skip
-        @skipping = true
+      unless @skipping || skip
+        # FIXME slight hack to skip past conditional line
+        # but keep our synthetic line marked as processed
+        conditional_line = peek_line true
+        replace_line "#{text.rstrip}#{::Asciidoctor::EOL}"
+        unshift conditional_line
+        return true
       end
-      @conditionals_stack << {:target => target, :skip => skip, :skipping => @skipping}
     end
-    return preprocess_next_line.nil? ? nil : true
+
+    true
   end
 
   # Internal: Preprocess the directive (macro) to include the target document.
@@ -410,13 +776,12 @@ class Reader
   # If SafeMode is SECURE or greater, the directive is ignore and the include
   # directive line is emitted verbatim.
   #
-  # Otherwise, if an include handler is specified (currently controlled by a
-  # closure block), pass the target to that block and expect an Array of String
-  # lines in return.
+  # Otherwise, if an include processor is specified pass the target and
+  # attributes to that processor and expect an Array of String lines in return.
   #
-  # Otherwise, if the include-depth attribute is greater than 0, normalize the
-  # target path and read the lines onto the beginning of the Array of source
-  # data.
+  # Otherwise, if the max depth is greater than 0, and is not exceeded by the
+  # stack size, normalize the target path and read the lines onto the beginning
+  # of the Array of source data.
   #
   # If none of the above apply, emit the include directive line verbatim.
   #
@@ -424,39 +789,68 @@ class Reader
   #          target slot of the include::[] macro
   #
   # returns a Boolean indicating whether the line under the cursor has changed.
-  def preprocess_include(target, raw_attributes)
-    target = @document.sub_attributes target
+  def preprocess_include target, raw_attributes
+    target = @document.sub_attributes target, :attribute_missing => 'drop-line'
     if target.empty?
+      if @document.attributes.fetch('attribute-missing', COMPLIANCE[:attribute_missing]) == 'skip'
+        false
+      else
+        advance
+        true
+      end
+    # assume that if an include processor is given, the developer wants
+    # to handle when and how to process the include
+    elsif include_processors? &&
+        (processor = @include_processors.find {|candidate| candidate.handles? target })
       advance
-      @next_line_preprocessed = false
-      false
+      # QUESTION should we use @document.parse_attribues?
+      processor.process self, target, AttributeList.new(raw_attributes).parse
+      true
     # if running in SafeMode::SECURE or greater, don't process this directive
     # however, be friendly and at least make it a link to the source document
     elsif @document.safe >= SafeMode::SECURE
-      @lines[0] = "link:#{target}[#{target}]\n"
-      @next_line_preprocessed = true
+      replace_line "link:#{target}[]#{::Asciidoctor::EOL}"
+      # TODO make creating the output target a helper method
+      #output_target = %(#{File.join(File.dirname(target), File.basename(target, File.extname(target)))}#{@document.attributes['outfilesuffix']})
+      #unshift "link:#{output_target}[]#{::Asciidoctor::EOL}"
+      true
+    elsif (abs_maxdepth = @maxdepth[:abs]) > 0 && @include_stack.size >= abs_maxdepth
+      warn %(asciidoctor: ERROR: #{line_info}: maximum include depth of #{@maxdepth[:rel]} exceeded)
       false
-    # assume that if a block is given, the developer wants
-    # to handle when and how to process the include, even
-    # if the include-depth attribute is 0
-    elsif @include_block
-      advance
-      # FIXME this borks line numbers
-      @lines.unshift(*normalize_include_data(@include_block.call(target)))
-    # FIXME currently we're not checking the upper bound of the include depth
-    elsif @document.attributes.fetch('include-depth', 0).to_i > 0
-      advance
-      # FIXME this borks line numbers
-      include_file = @document.normalize_system_path(target, nil, nil, :target_name => 'include file')
-      if !File.file?(include_file)
-        puts "asciidoctor: WARNING: line #{@lineno}: include file not found: #{include_file}"
-        return true
+    elsif abs_maxdepth > 0
+      if target.include?(':') && target.match(REGEXP[:uri_sniff])
+        unless @document.attributes.has_key? 'allow-uri-read'
+          replace_line "link:#{target}[]#{::Asciidoctor::EOL}"
+          return true
+        end
+
+        target_type = :uri
+        include_file = path = target
+        if @document.attributes.has_key? 'cache-uri'
+          # caching requires the open-uri-cached gem to be installed
+          # processing will be automatically aborted if these libraries can't be opened
+          Helpers.require_library 'open-uri/cached', 'open-uri-cached'
+        else
+          Helpers.require_library 'open-uri'
+        end
+      else
+        target_type = :file
+        # include file is resolved relative to dir of current include, or base_dir if within original docfile
+        include_file = @document.normalize_system_path(target, @dir, nil, :target_name => 'include file')
+        if !File.file?(include_file)
+          warn "asciidoctor: WARNING: #{line_info}: include file not found: #{include_file}"
+          advance
+          return true
+        end
+        #path = @document.relative_path include_file
+        path = PathResolver.new.relative_path include_file, @document.base_dir
       end
 
       inc_lines = nil
       tags = nil
       attributes = {}
       if !raw_attributes.empty?
+        # QUESTION should we use @document.parse_attribues?
         attributes = AttributeList.new(raw_attributes).parse
         if attributes.has_key? 'lines'
           inc_lines = []
@@ -474,6 +868,8 @@ class Reader
             end
           end
           inc_lines = inc_lines.sort.uniq
+        elsif attributes.has_key? 'tag'
+          tags = [attributes['tag']]
         elsif attributes.has_key? 'tags'
           tags = attributes['tags'].split(REGEXP[:ssv_or_csv_delim]).uniq
         end
@@ -481,213 +877,227 @@ class Reader
       if !inc_lines.nil?
         if !inc_lines.empty?
           selected = []
-          f = File.new(include_file)
-          f.each_line do |l|
-            take = inc_lines.first
-            if take.is_a?(Float) && take.infinite?
-              selected.push l
-            else
-              if f.lineno == take
-                selected.push l
-                inc_lines.shift 
+          inc_line_offset = 0
+          inc_lineno = 0
+          begin
+            open(include_file) do |f|
+              f.each_line do |l|
+                inc_lineno += 1
+                take = inc_lines.first
+                if take.is_a?(Float) && take.infinite?
+                  selected.push l
+                  inc_line_offset = inc_lineno if inc_line_offset == 0
+                else
+                  if f.lineno == take
+                    selected.push l
+                    inc_line_offset = inc_lineno if inc_line_offset == 0
+                    inc_lines.shift
+                  end
+                  break if inc_lines.empty?
+                end
               end
-              break if inc_lines.empty?
             end
+          rescue
+            warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+            advance
+            return true
           end
-          @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
+          advance
+          # FIXME not accounting for skipped lines in reader line numbering
+          push_include selected, include_file, path, inc_line_offset, attributes
         end
       elsif !tags.nil?
         if !tags.empty?
           selected = []
+          inc_line_offset = 0
+          inc_lineno = 0
           active_tag = nil
-          f = File.new(include_file)
-          f.each_line do |l|
-            l.force_encoding(::Encoding::UTF_8) if ::Asciidoctor::FORCE_ENCODING
-            if !active_tag.nil?
-              if l.include?("end::#{active_tag}[]")
-                active_tag = nil
-              else
-                selected.push "#{l.rstrip}\n"
-              end
-            else
-              tags.each do |tag|
-                if l.include?("tag::#{tag}[]")
-                  active_tag = tag
-                  break
+          begin
+            open(include_file) do |f|
+              f.each_line do |l|
+                inc_lineno += 1
+                # must force encoding here since we're performing String operations on line
+                l.force_encoding(::Encoding::UTF_8) if ::Asciidoctor::FORCE_ENCODING
+                if !active_tag.nil?
+                  if l.include?("end::#{active_tag}[]")
+                    active_tag = nil
+                  else
+                    selected.push l
+                    inc_line_offset = inc_lineno if inc_line_offset == 0
+                  end
+                else
+                  tags.each do |tag|
+                    if l.include?("tag::#{tag}[]")
+                      active_tag = tag
+                      break
+                    end
+                  end
                 end
               end
             end
+          rescue
+            warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+            advance
+            return true
           end
-          #@lines.unshift(*selected) unless selected.empty?
-          @lines.unshift(*normalize_include_data(selected, attributes['indent'])) unless selected.empty?
+          advance
+          # FIXME not accounting for skipped lines in reader line numbering
+          push_include selected, include_file, path, inc_line_offset, attributes
         end
       else
-        @lines.unshift(*normalize_include_data(File.readlines(include_file), attributes['indent']))
+        begin
+          advance
+          push_include open(include_file) {|f| f.read }, include_file, path, 1, attributes
+        rescue
+          warn "asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{include_file}"
+          advance
+          return true
+        end
       end
       true
     else
-      @next_line_preprocessed = true
       false
     end
   end
 
-  # Public: Push the String line onto the beginning of the Array of source data.
-  #
-  # Since this line was (assumed to be) previously retrieved through the
-  # reader, it is marked as preprocessed.
-  #
-  # returns nil
-  def unshift_line(line)
-    @lines.unshift line
-    @next_line_preprocessed = true
-    @eof = false
-    @lineno -= 1
-    nil
+  def push_include data, file = nil, path = nil, lineno = 1, attributes = {}
+    @include_stack << [@lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines]
+    @includes << Helpers.rootname(path)
+    @file = file
+    @dir = File.dirname file
+    @path = path
+    @lineno = lineno
+    # NOTE only process lines in AsciiDoc files
+    @process_lines = ASCIIDOC_EXTENSIONS[File.extname(@file)]
+    if attributes.has_key? 'depth'
+      depth = attributes['depth'].to_i
+      depth = 1 if depth <= 0
+      @maxdepth = {:abs => (@include_stack.size - 1) + depth, :rel => depth}
+    end
+    # effectively fill the buffer
+    @lines = prepare_lines data, :condense => false, :indent => attributes['indent']
+    # FIXME kind of a hack
+    #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
+    #Document::AttributeEntry.new('indir', File.dirname(@file)).save_to_next_block @document
+    if @lines.empty?
+      pop_include
+    else
+      @eof = false
+      @look_ahead = 0
+    end
   end
 
-  # Public: Push Array of lines onto the front of the Array of source data, unless `lines` has no non-nil values.
-  #
-  # Returns nil
-  def unshift(*new_lines)
-    size = new_lines.size
-    if size > 0
-      @lines.unshift(*new_lines)
-      # assume that what we are putting back on is already processed for directives
-      @next_line_preprocessed = true
-      @eof = false
-      @lineno -= size
+  def pop_include
+    if @include_stack.size > 0
+      @lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines = @include_stack.pop
+      # FIXME kind of a hack
+      #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
+      #Document::AttributeEntry.new('indir', File.dirname(@file)).save_to_next_block @document
+      @eof = @lines.empty?
+      @look_ahead = 0
     end
-    nil
   end
 
-  # Public: Chomp the String on the last line if this reader contains at least one line
-  #
-  # Delegates to chomp!
-  #
-  # Returns nil
-  def chomp_last!
-    @lines.last.chomp! unless @eof || (@eof = @lines.empty?)
-    nil
+  def include_depth
+    @include_stack.size 
   end
 
-  # Public: Return all the lines from `@lines` until we (1) run out them,
-  #   (2) find a blank line with :break_on_blank_lines => true, or (3) find
-  #   a line for which the given block evals to true.
-  #
-  # options - an optional Hash of processing options:
-  #           * :break_on_blank_lines may be used to specify to break on
-  #               blank lines
-  #           * :skip_first_line may be used to tell the reader to advance
-  #               beyond the first line before beginning the scan
-  #           * :preserve_last_line may be used to specify that the String
-  #               causing the method to stop processing lines should be
-  #               pushed back onto the `lines` Array.
-  #           * :grab_last_line may be used to specify that the String
-  #               causing the method to stop processing lines should be
-  #               included in the lines being returned
-  #
-  # Returns the Array of lines forming the next segment.
-  #
-  # Examples
-  #
-  #   reader = Reader.new ["First paragraph\n", "Second paragraph\n",
-  #                        "Open block\n", "\n", "Can have blank lines\n",
-  #                        "--\n", "\n", "In a different segment\n"]
-  #
-  #   reader.grab_lines_until
-  #   => ["First paragraph\n", "Second paragraph\n", "Open block\n"]
-  def grab_lines_until(options = {}, &block)
-    buffer = []
+  def exceeded_max_depth?
+    if (abs_maxdepth = @maxdepth[:abs]) > 0 && @include_stack.size >= abs_maxdepth
+      @maxdepth[:rel]
+    else
+      false
+    end
+  end
 
-    advance if options[:skip_first_line]
-    # very hot code
-    # save options to locals for minor optimizations
-    if options.has_key? :terminator
-      terminator = options[:terminator]
-      break_on_blank_lines = false
-      break_on_list_continuation = false
-      chomp_last_line = options[:chomp_last_line] || false
+  # TODO Document this override
+  # also, we now have the field in the super class, so perhaps
+  # just implement the logic there?
+  def shift
+    if @unescape_next_line
+      @unescape_next_line = false
+      super[1..-1]
     else
-      terminator = nil
-      break_on_blank_lines = options[:break_on_blank_lines]
-      break_on_list_continuation = options[:break_on_list_continuation]
-      chomp_last_line = break_on_blank_lines
+      super
     end
-    skip_line_comments = options[:skip_line_comments]
-    preprocess = options.fetch(:preprocess, true)
-    buffer_empty = true
-    while !(this_line = get_line(preprocess)).nil?
-      # effectively a no-args lamba, but much faster
-      finish = while true
-        break true if terminator && this_line.chomp == terminator
-        break true if break_on_blank_lines && this_line.strip.empty?
-        if break_on_list_continuation && !buffer_empty && this_line.chomp == LIST_CONTINUATION
-          options[:preserve_last_line] = true
-          break true
-        end
-        break true if block && yield(this_line)
-        break false
-      end
+  end
 
-      if finish
-        if options[:grab_last_line]
-          buffer << this_line
-          buffer_empty = false
-        end
-        # QUESTION should we dup this_line when restoring??
-        unshift_line this_line if options[:preserve_last_line]
-        break
+  # Private: Ignore front-matter, commonly used in static site generators
+  def skip_front_matter! data, increment_linenos = true
+    front_matter = nil
+    if data.size > 0 && data.first.chomp == '---'
+      original_data = data.dup
+      front_matter = []
+      data.shift
+      @lineno += 1 if increment_linenos
+      while !data.empty? && data.first.chomp != '---'
+        front_matter.push data.shift
+        @lineno += 1 if increment_linenos
       end
 
-      unless skip_line_comments && this_line.match(REGEXP[:comment])
-        buffer << this_line
-        buffer_empty = false
+      if data.empty?
+        data.unshift(*original_data)
+        @lineno = 0 if increment_linenos
+        front_matter = nil
+      else
+        data.shift
+        @lineno += 1 if increment_linenos
       end
     end
 
-    # should we dup the line before chopping?
-    buffer.last.chomp! if chomp_last_line && !buffer_empty
-    buffer
+    front_matter
   end
 
-  # Public: Convert a string to a legal attribute name.
+  # Private: Resolve the value of one side of the expression
   #
-  # name  - The String holding the Asciidoc attribute name.
+  # Examples
   #
-  # Returns a String with the legal name.
+  #   expr = '"value"'
+  #   resolve_expr_val(expr)
+  #   # => "value"
   #
-  # Examples
+  #   expr = '"value'
+  #   resolve_expr_val(expr)
+  #   # => "\"value"
   #
-  #   sanitize_attribute_name('Foo Bar')
-  #   => 'foobar'
+  #   expr = '"{undefined}"'
+  #   resolve_expr_val(expr)
+  #   # => ""
   #
-  #   sanitize_attribute_name('foo')
-  #   => 'foo'
+  #   expr = '{undefined}'
+  #   resolve_expr_val(expr)
+  #   # => nil
   #
-  #   sanitize_attribute_name('Foo 3 #-Billy')
-  #   => 'foo3-billy'
-  def sanitize_attribute_name(name)
-    Lexer.sanitize_attribute_name(name)
-  end
-
-  # Private: Resolve the value of one side of the expression
+  #   expr = '2'
+  #   resolve_expr_val(expr)
+  #   # => 2
+  #
+  #   @document.attributes['name'] = 'value'
+  #   expr = '"{name}"'
+  #   resolve_expr_val(expr)
+  #   # => "value"
+  #
+  # Returns The value of the expression, coerced to the appropriate type
   def resolve_expr_val(str)
     val = str
     type = nil
 
     if val.start_with?('"') && val.end_with?('"') ||
         val.start_with?('\'') && val.end_with?('\'')
-      type = :s
-      val = val[1..-2]
+      type = :string
+      val = val[1...-1]
     end
 
+    # QUESTION should we substitute first?
     if val.include? '{'
       val = @document.sub_attributes val
     end
 
-    if type != :s
+    unless type == :string
       if val.empty?
         val = nil
+      elsif val.strip.empty?
+        val = ' '
       elsif val == 'true'
         val = true
       elsif val == 'false'
@@ -695,6 +1105,8 @@ class Reader
       elsif val.include?('.')
         val = val.to_f
       else
+        # fallback to coercing to integer, since we
+        # require string values to be explicitly quoted
         val = val.to_i
       end
     end
@@ -702,75 +1114,22 @@ class Reader
     val
   end
 
-  # Private: Normalize raw input read from an include directive
-  #
-  # This method strips whitespace from the end of every line of
-  # the source data and appends a LF (i.e., Unix endline). This
-  # whitespace substitution is very important to how Asciidoctor
-  # works.
-  #
-  # Any leading or trailing blank lines are also removed. (DISABLED)
-  #
-  # data - A String Array of input data to be normalized
-  #
-  # returns the processed lines
-  #-
-  # FIXME this shares too much in common w/ normalize_data; combine
-  # in a shared function
-  def normalize_include_data(data, indent = nil)
-    if ::Asciidoctor::FORCE_ENCODING
-      result = data.map {|line| "#{line.rstrip.force_encoding(::Encoding::UTF_8)}\n" }
+  def include_processors?
+    if @include_processors.nil?
+      if @document.extensions? && @document.extensions.include_processors?
+        @include_processors = @document.extensions.load_include_processors(@document)
+        true
+      else
+        @include_processors = false
+        false
+      end
     else
-      result = data.map {|line| "#{line.rstrip}\n" }
+      @include_processors != false
     end
-
-    unless indent.nil?
-      Lexer.reset_block_indent! result, indent.to_i
-    end
-
-    result
-
-    #data.shift while !data.first.nil? && data.first.chomp.empty?
-    #data.pop while !data.last.nil? && data.last.chomp.empty?
-    #data
   end
 
-  # Private: Normalize raw input, used for the outermost Reader.
-  #
-  # This method strips whitespace from the end of every line of
-  # the source data and appends a LF (i.e., Unix endline). This
-  # whitespace substitution is very important to how Asciidoctor
-  # works.
-  #
-  # Any leading or trailing blank lines are also removed.
-  #
-  # The normalized lines are assigned to the @lines instance variable.
-  #
-  # data - A String Array of input data to be normalized
-  #
-  # returns nothing
-  def normalize_data(data)
-    # normalize line ending to LF (purging occurrences of CRLF)
-    # this rstrip is *very* important to how Asciidoctor works
-
-    if ::Asciidoctor::FORCE_ENCODING
-      @lines = data.map {|line| "#{line.rstrip.force_encoding(::Encoding::UTF_8)}\n" }
-    else
-      @lines = data.map {|line| "#{line.rstrip}\n" }
-    end
-
-    @lines.shift && @lineno += 1 while !@lines.first.nil? && @lines.first.chomp.empty?
-    @lines.pop while !@lines.last.nil? && @lines.last.chomp.empty?
-
-    # Process bibliography references, so they're available when text
-    # before the reference is being rendered.
-    # FIXME reenable whereever it belongs
-    #@lines.each do |line|
-    #  if biblio = line.match(REGEXP[:biblio])
-    #    @document.register(:ids, biblio[1])
-    #  end
-    #end
-    nil
+  def to_s
+    %(#{self.class.name} [path: #{@path}, line #: #{@lineno}, include depth: #{@include_stack.size}, include stack: [#{@include_stack.map {|inc| inc.to_s}.join ', '}]])
   end
 end
 end