asciidoctor 0.1.3 → 0.1.4

data/lib/asciidoctor/renderer.rb

@@ -2,7 +2,15 @@ module Asciidoctor
 # Public: Methods for rendering Asciidoc Documents, Sections, and Blocks
 # using eRuby templates.
 class Renderer
+  RE_ASCIIDOCTOR_NAMESPACE = /^Asciidoctor::/
+  RE_TEMPLATE_CLASS_SUFFIX = /Template$/
+  RE_CAMELCASE_BOUNDARY_1 = /([[:upper:]]+)([[:upper:]][[:alpha:]])/
+  RE_CAMELCASE_BOUNDARY_2 = /([[:lower:]])([[:upper:]])/
+
   attr_reader :compact
+  attr_reader :cache
+
+  @@global_cache = nil
 
   # Public: Initialize an Asciidoctor::Renderer object.
   #
@@ -11,10 +19,11 @@ class Renderer
 
     @views = {}
     @compact = options[:compact]
+    @cache = nil
 
     backend = options[:backend]
     case backend
-    when 'html5', 'docbook45'
+    when 'html5', 'docbook45', 'docbook5'
       eruby = load_eruby options[:eruby]
       #Helpers.require_library 'asciidoctor/backends/' + backend
       require 'asciidoctor/backends/' + backend
@@ -23,7 +32,7 @@ class Renderer
         if tc.to_s.downcase.include?('::' + backend + '::') # optimization
           view_name, view_backend = self.class.extract_view_mapping(tc)
           if view_backend == backend
-            @views[view_name] = tc.new(view_name, eruby)
+            @views[view_name] = tc.new(view_name, backend, eruby)
           end
         end
       end
@@ -32,26 +41,19 @@ class Renderer
     end
 
     # If user passed in a template dir, let them override our base templates
-    if template_dir = options.delete(:template_dir)
-      Helpers.require_library 'tilt'
-
-      template_glob = '*'
-      if (engine = options[:template_engine])
-        template_glob = "*.#{engine}"
-        # example: templates/haml
-        if File.directory? File.join(template_dir, engine)
-          template_dir = File.join template_dir, engine
-        end
-      end
+    if (template_dirs = options.delete(:template_dirs))
+      Helpers.require_library 'tilt', true
 
-      # example: templates/html5 or templates/haml/html5
-      if File.directory? File.join(template_dir, options[:backend])
-        template_dir = File.join template_dir, options[:backend]
+      if (template_cache = options[:template_cache]) === true
+        # FIXME probably want to use our own cache object for more control
+        @cache = (@@global_cache ||= TemplateCache.new)
+      elsif template_cache
+        @cache = template_cache
       end
 
       view_opts = {
         :erb =>  { :trim => '<>' },
-        :haml => { :attr_wrapper => '"', :ugly => true, :escape_attrs => false },
+        :haml => { :format => :xhtml, :attr_wrapper => '"', :ugly => true, :escape_attrs => false },
         :slim => { :disable_escape => true, :sort_attrs => false, :pretty => false }
       }
 
@@ -61,29 +63,64 @@ class Renderer
       end
 
       slim_loaded = false
-      helpers = nil
-      
-      # Grab the files in the top level of the directory (we're not traversing)
-      Dir.glob(File.join(template_dir, template_glob)).
-          select{|f| File.file? f }.each do |template|
-        basename = File.basename(template)
-        if basename == 'helpers.rb'
-          helpers = template
+      path_resolver = PathResolver.new
+      engine = options[:template_engine]
+
+      template_dirs.each do |template_dir|
+        # TODO need to think about safe mode restrictions here
+        template_dir = path_resolver.system_path template_dir, nil
+        template_glob = '*'
+        if engine
+          template_glob = "*.#{engine}"
+          # example: templates/haml
+          if File.directory? File.join(template_dir, engine)
+            template_dir = File.join template_dir, engine
+          end
+        end
+
+        # example: templates/html5 or templates/haml/html5
+        if File.directory? File.join(template_dir, backend)
+          template_dir = File.join template_dir, backend
+        end
+
+        # skip scanning folder if we've already done it for same backend/engine
+        if @cache && @cache.cached?(:scan, template_dir, template_glob)
+          @views.update(@cache.fetch :scan, template_dir, template_glob)
           next
         end
-        name_parts = basename.split('.')
-        next if name_parts.size < 2
-        view_name = name_parts.first 
-        ext_name = name_parts.last
-        if ext_name == 'slim' && !slim_loaded
-          # slim doesn't get loaded by Tilt
-          Helpers.require_library 'slim'
+
+        helpers = nil
+        scan_result = {}
+        # Grab the files in the top level of the directory (we're not traversing)
+        Dir.glob(File.join(template_dir, template_glob)).
+            select{|f| File.file? f }.each do |template|
+          basename = File.basename(template)
+          if basename == 'helpers.rb'
+            helpers = template
+            next
+          end
+          name_parts = basename.split('.')
+          next if name_parts.size < 2
+          view_name = name_parts.first 
+          ext_name = name_parts.last
+          if ext_name == 'slim' && !slim_loaded
+            # slim doesn't get loaded by Tilt
+            Helpers.require_library 'slim', true
+          end
+          next unless Tilt.registered? ext_name
+          opts = view_opts[ext_name.to_sym]
+          if @cache
+            @views[view_name] = scan_result[view_name] = @cache.fetch(:view, template) {
+              Tilt.new(template, nil, opts)
+            }
+          else
+            @views[view_name] = Tilt.new template, nil, opts
+          end
         end
-        next unless Tilt.registered? ext_name
-        @views[view_name] = Tilt.new(template, nil, view_opts[ext_name.to_sym])
-      end
 
-      require helpers unless helpers.nil?
+        require helpers unless helpers.nil?
+        @cache.store(scan_result, :scan, template_dir, template_glob) if @cache
+      end
     end
   end
 
@@ -106,6 +143,11 @@ class Renderer
     readonly_views
   end
 
+  def register_view(view_name, tilt_template)
+    # TODO need to figure out how to cache this
+    @views[view_name] = tilt_template
+  end
+
   # Internal: Load the eRuby implementation
   #
   # name - the String name of the eRuby implementation (default: 'erb')
@@ -116,15 +158,25 @@ class Renderer
       name = 'erb'
     end
 
-    Helpers.require_library name
-
     if name == 'erb'
+      Helpers.require_library 'erb'
       ::ERB
     elsif name == 'erubis'
+      Helpers.require_library 'erubis', true
       ::Erubis::FastEruby
     end
   end
 
+  # TODO better name for this method (and/or field)
+  def self.global_cache
+    @@global_cache
+  end
+
+  # TODO better name for this method (and/or field)
+  def self.reset_global_cache
+    @@global_cache.clear if @@global_cache
+  end
+
   # Internal: Extracts the view name and backend from a qualified Ruby class
   #
   # The purpose of this method is to determine the view name and backend to
@@ -145,8 +197,8 @@ class Renderer
   # Returns A two-element String Array mapped as [view_name, backend], where backend may be nil
   def self.extract_view_mapping(qualified_class)
     view_name, backend = qualified_class.to_s.
-        gsub(/^Asciidoctor::/, '').
-        gsub(/Template$/, '').
+        sub(RE_ASCIIDOCTOR_NAMESPACE, '').
+        sub(RE_TEMPLATE_CLASS_SUFFIX, '').
         split('::').reverse
     view_name = camelcase_to_underscore(view_name)
     backend = backend.downcase unless backend.nil?
@@ -165,9 +217,43 @@ class Renderer
   #
   # Returns the String converted from CamelCase to underscore-delimited
   def self.camelcase_to_underscore(str)
-    str.gsub(/([[:upper:]]+)([[:upper:]][[:alpha:]])/, '\1_\2').
-        gsub(/([[:lower:]])([[:upper:]])/, '\1_\2').downcase
+    str.gsub(RE_CAMELCASE_BOUNDARY_1, '\1_\2').
+        gsub(RE_CAMELCASE_BOUNDARY_2, '\1_\2').downcase
+  end
+
+end
+
+class TemplateCache
+  attr_reader :cache
+
+  def initialize
+    @cache = {}
   end
 
+  # check if a key is available in the cache
+  def cached? *key
+    @cache.has_key? key
+  end
+
+  # retrieves an item from the cache stored in the cache key
+  # if a block is given, the block is called and the return
+  # value stored in the cache under the specified key
+  def fetch(*key)
+    if block_given?
+      @cache[key] ||= yield
+    else
+      @cache[key]
+    end
+  end
+
+  # stores an item in the cache under the specified key
+  def store(value, *key)
+    @cache[key] = value
+  end
+
+  # Clears the cache
+  def clear
+    @cache = {}
+  end
 end
 end

data/lib/asciidoctor/section.rb

@@ -20,29 +20,41 @@ module Asciidoctor
 #   => 1
 class Section < AbstractBlock
 
-  # Public: Get/Set the Integer index of this section within the parent block
+  # Public: Get/Set the 0-based index order of this section within the parent block
   attr_accessor :index
 
+  # Public: Get/Set the number of this section within the parent block
+  # Only relevant if the attribute numbered is true
+  attr_accessor :number
+
   # Public: Get/Set the section name of this section
   attr_accessor :sectname
 
   # Public: Get/Set the flag to indicate whether this is a special section or a child of one
   attr_accessor :special
 
+  # Public: Get the state of the numbered attribute at this section (need to preserve for creating TOC)
+  attr_accessor :numbered
+
   # Public: Initialize an Asciidoctor::Section object.
   #
   # parent - The parent Asciidoc Object.
-  def initialize(parent = nil, level = nil)
+  def initialize(parent = nil, level = nil, numbered = true)
     super(parent, :section)
-    if level.nil? && !parent.nil?
-      @level = parent.level + 1
-    end
-    if parent.is_a?(Section) && parent.special
-      @special = true
+    @template_name = 'section'
+    if level.nil?
+      if !parent.nil?
+        @level = parent.level + 1
+      elsif @level.nil?
+        @level = 1
+      end
     else
-      @special = false
+      @level = level
     end
+    @numbered = numbered && @level > 0 && @level < 4
+    @special = parent.is_a?(Section) && parent.special
     @index = 0
+    @number = 1
   end
 
   # Public: The name of this section, an alias of the section title
@@ -69,16 +81,25 @@ class Section < AbstractBlock
   #   another_section.title = "Foo"
   #   another_section.generate_id
   #   => "_foo_1"
+  #
+  #   yet_another_section = Section.new(parent)
+  #   yet_another_section.title = "Ben & Jerry"
+  #   yet_another_section.generate_id
+  #   => "_ben_jerry"
   def generate_id
-    if @document.attr? 'sectids'
-      separator = @document.attr('idseparator', '_')
-      # FIXME define constants for these regexps
-      base_id = @document.attr('idprefix', '_') + title.downcase.gsub(/&#[0-9]+;/, separator).
-          gsub(/\W+/, separator).tr_s(separator, separator).chomp(separator)
+    if @document.attributes.has_key? 'sectids'
+      sep = @document.attributes['idseparator'] || '_'
+      pre = @document.attributes['idprefix'] || '_'
+      base_id = %(#{pre}#{title.downcase.gsub(REGEXP[:illegal_sectid_chars], sep).tr_s(sep, sep).chomp(sep)})
+      # ensure id doesn't begin with idprefix if requested it doesn't
+      if pre.empty? && base_id.start_with?(sep)
+        base_id = base_id[1..-1]
+        base_id = base_id[1..-1] while base_id.start_with?(sep)
+      end
       gen_id = base_id
       cnt = 2
-      while @document.references[:ids].has_key? gen_id 
-        gen_id = "#{base_id}#{separator}#{cnt}" 
+      while @document.references[:ids].has_key? gen_id
+        gen_id = "#{base_id}#{sep}#{cnt}"
         cnt += 1
       end 
       gen_id
@@ -87,33 +108,12 @@ class Section < AbstractBlock
     end
   end
 
-  # Public: Get the rendered String content for this Section and all its child
-  # Blocks.
-  def render
-    @document.playback_attributes @attributes
-    renderer.render('section', self)
-  end
-
-  # Public: Get the String section content by aggregating rendered section blocks.
-  #
-  # Examples
-  #
-  #   section = Section.new
-  #   section << 'foo'
-  #   section << 'bar'
-  #   section << 'baz'
-  #   section.content
-  #   "<div class=\"paragraph\"><p>foo</p></div>\n<div class=\"paragraph\"><p>bar</p></div>\n<div class=\"paragraph\"><p>baz</p></div>"
-  def content
-    @blocks.map {|b| b.render }.join
-  end
-
   # Public: Get the section number for the current Section
   #
   # The section number is a unique, dot separated String
   # where each entry represents one level of nesting and
-  # the value of each entry is the 1-based index of
-  # the Section amongst its sibling Sections
+  # the value of each entry is the 1-based outline number
+  # of the Section amongst its numbered sibling Sections
   #
   # delimiter - the delimiter to separate the number for each level
   # append    - the String to append at the end of the section number
@@ -154,15 +154,29 @@ class Section < AbstractBlock
   def sectnum(delimiter = '.', append = nil)
     append ||= (append == false ? '' : delimiter)
     if !@level.nil? && @level > 1 && @parent.is_a?(Section)
-      "#{@parent.sectnum(delimiter)}#{@index + 1}#{append}"
+      "#{@parent.sectnum(delimiter)}#{@number}#{append}"
     else
-      "#{@index + 1}#{append}"
+      "#{@number}#{append}"
+    end
+  end
+
+  # Public: Append a content block to this block's list of blocks.
+  #
+  # If the child block is a Section, assign an index to it.
+  #
+  # block - The child Block to append to this parent Block
+  #
+  # Returns nothing.
+  def <<(block)
+    super
+    if block.context == :section
+      assign_index block
     end
   end
 
   def to_s
     if @title
-      if @level && @index
+      if @numbered
         %[#{super.to_s} - #{sectnum} #@title [blocks:#{@blocks.size}]]
       else
         %[#{super.to_s} - #@title [blocks:#{@blocks.size}]]

data/lib/asciidoctor/substituters.rb

@@ -4,47 +4,68 @@ module Asciidoctor
 # the necessary substitutions.
 module Substituters
 
+  SUBS = {
+    :basic    => [:specialcharacters],
+    :normal   => [:specialcharacters, :quotes, :attributes, :replacements, :macros, :post_replacements],
+    :verbatim => [:specialcharacters, :callouts],
+    :title    => [:specialcharacters, :quotes, :replacements, :macros, :attributes, :post_replacements],
+    :header   => [:specialcharacters, :attributes],
+    :pass     => [:attributes, :macros]
+  }
+
   COMPOSITE_SUBS = {
     :none => [],
-    :normal => [:specialcharacters, :quotes, :attributes, :replacements, :macros, :post_replacements],
-    :verbatim => [:specialcharacters, :callouts]
+    :normal => SUBS[:normal],
+    :verbatim => SUBS[:verbatim]
   }
 
-  SUB_OPTIONS = COMPOSITE_SUBS.keys + COMPOSITE_SUBS[:normal]
+  SUB_OPTIONS = {
+    :block  => COMPOSITE_SUBS.keys + SUBS[:normal] + [:callouts],
+    :inline => COMPOSITE_SUBS.keys + SUBS[:normal]
+  }
 
   # Internal: A String Array of passthough (unprocessed) text captured from this block
   attr_reader :passthroughs
 
   # Public: Apply the specified substitutions to the lines of text
   #
-  # lines   - The lines of text to process. Can be a String or a String Array 
-  # subs    - The substitutions to perform. Can be a Symbol or a Symbol Array (default: COMPOSITE_SUBS[:normal])
-  # 
+  # source  - The String or String Array of text to process
+  # subs    - The substitutions to perform. Can be a Symbol or a Symbol Array (default: :normal)
+  # expand -  A Boolean to control whether sub aliases are expanded (default: true)
+  #
   # returns Either a String or String Array, whichever matches the type of the first argument
-  def apply_subs(lines, subs = COMPOSITE_SUBS[:normal])
-    if subs.nil?
-      subs = []
-    elsif subs.is_a? Symbol
-      subs = [subs]
-    end
+  def apply_subs source, subs = :normal, expand = false
+    if subs == :normal
+      subs = SUBS[:normal]
+    elsif subs.nil?
+      return source
+    elsif expand
+      if subs.is_a? Symbol
+        subs = COMPOSITE_SUBS[subs] || [subs]
+      else
+        effective_subs = []
+        subs.each do |key|
+          if COMPOSITE_SUBS.has_key? key
+            effective_subs.push(*COMPOSITE_SUBS[key])
+          else
+            effective_subs << key
+          end
+        end
 
-    if !subs.empty?
-      # QUESTION is this most efficient operation?
-      subs = subs.map {|key|
-        COMPOSITE_SUBS.has_key?(key) ? COMPOSITE_SUBS[key] : key
-      }.flatten
+        subs = effective_subs
+      end
     end
 
-    return lines if subs.empty?
+    return source if subs.empty?
 
-    multiline = lines.is_a?(Array)
-    text = multiline ? lines.join : lines
+    multiline = source.is_a?(Array)
+    text = multiline ? source.join : source
 
     if (has_passthroughs = subs.include?(:macros))
       text = extract_passthroughs(text)
     end
-    
-    subs.each {|type|
+
+    subs.each do |type|
       case type
       when :specialcharacters
         text = sub_specialcharacters(text)
@@ -56,14 +77,16 @@ module Substituters
         text = sub_replacements(text)
       when :macros
         text = sub_macros(text)
+      when :highlight
+        text = highlight_source(text, subs.include?(:callouts))
       when :callouts
-        text = sub_callouts(text)
+        text = sub_callouts(text) unless subs.include?(:highlight)
       when :post_replacements
         text = sub_post_replacements(text)
       else
-        puts "asciidoctor: WARNING: unknown substitution type #{type}"
+        warn "asciidoctor: WARNING: unknown substitution type #{type}"
       end
-    }
+    end
     text = restore_passthroughs(text) if has_passthroughs
 
     multiline ? text.lines.entries : text
@@ -71,11 +94,11 @@ module Substituters
 
   # Public: Apply normal substitutions.
   #
-  # lines  - The lines of text to process. Can be a String or a String Array 
+  # lines  - The lines of text to process. Can be a String or a String Array
   #
   # returns - A String with normal substitutions performed
   def apply_normal_subs(lines)
-    apply_subs(lines.is_a?(Array) ? lines.join : lines)
+    apply_subs lines.is_a?(Array) ? lines.join : lines
   end
 
   # Public: Apply substitutions for titles.
@@ -84,23 +107,7 @@ module Substituters
   #
   # returns - A String with title substitutions performed
   def apply_title_subs(title)
-    apply_subs(title, [:specialcharacters, :quotes, :replacements, :macros, :attributes, :post_replacements])
-  end
-
-  # Public: Apply substitutions for titles
-  #
-  # lines  - A String Array containing the lines of text process
-  #
-  # returns - A String with literal (verbatim) substitutions performed
-  def apply_literal_subs(lines)
-    if attr? 'subs'
-      apply_subs(lines.join, resolve_subs(attr 'subs'))
-    elsif @document.attributes['basebackend'] == 'html' && attr('style') == 'source' &&
-      @document.attributes['source-highlighter'] == 'coderay' && attr?('language')
-      sub_callouts(highlight_source(lines.join))
-    else
-      apply_subs(lines.join, COMPOSITE_SUBS[:verbatim])
-    end
+    apply_subs title, SUBS[:title]
   end
 
   # Public: Apply substitutions for header metadata and attribute assignments
@@ -109,19 +116,37 @@ module Substituters
   #
   # returns - A String with header substitutions performed
   def apply_header_subs(text)
-    apply_subs(text, [:specialcharacters, :attributes])
+    apply_subs text, SUBS[:header]
   end
 
+=begin
   # Public: Apply explicit substitutions, if specified, otherwise normal substitutions.
   #
-  # lines  - The lines of text to process. Can be a String or a String Array 
+  # lines  - The lines of text to process. Can be a String or a String Array
   #
   # returns - A String with substitutions applied
   def apply_para_subs(lines)
-    if attr? 'subs'
-      apply_subs(lines.join, resolve_subs(attr 'subs'))
+    if (subs = attr('subs', nil, false))
+      apply_subs lines.join, resolve_subs(subs)
     else
-      apply_subs(lines.join)
+      apply_subs lines.join
+    end
+  end
+
+  # Public: Apply substitutions for titles
+  #
+  # lines  - A String Array containing the lines of text process
+  #
+  # returns - A String with literal (verbatim) substitutions performed
+  def apply_literal_subs(lines)
+    if (subs = attr('subs', nil, false))
+      apply_subs lines.join, resolve_subs(subs)
+    elsif @style == 'source' && @document.attributes['basebackend'] == 'html' &&
+      ((highlighter = @document.attributes['source-highlighter']) == 'coderay' ||
+      highlighter == 'pygments') && attr?('language')
+      highlight_source lines.join, highlighter, callouts
+    else
+      apply_subs lines.join, SUBS[:verbatim]
     end
   end
 
@@ -131,13 +156,14 @@ module Substituters
   #
   # returns - A String with passthrough substitutions performed
   def apply_passthrough_subs(lines)
-    if attr? 'subs'
-      subs = resolve_subs(attr('subs'))
+    if (subs = attr('subs', nil, false))
+      subs = resolve_subs(subs)
     else
-      subs = [:attributes, :macros]
+      subs = SUBS[:pass]
     end
-    apply_subs(lines.join, subs)
+    apply_subs lines.join, subs
   end
+=end
 
   # Internal: Extract the passthrough text from the document for reinsertion after processing.
   #
@@ -157,10 +183,10 @@ module Substituters
 
       if m[1] == '$$'
         subs = [:specialcharacters]
-      elsif !m[3].nil? && !m[3].empty?
-        subs = resolve_subs(m[3])
-      else
+      elsif m[3].nil? || m[3].empty?
         subs = []
+      else
+        subs = resolve_pass_subs m[3]
       end
 
       # TODO move unescaping closing square bracket to an operation
@@ -173,14 +199,23 @@ module Substituters
       # alias match for Ruby 1.8.7 compat
       m = $~
 
+      unescaped_attrs = nil
       # honor the escape
-      if m[2].start_with? '\\'
-        next "#{m[1]}#{m[2][1..-1]}"
+      if m[3].start_with? '\\'
+        next m[2].nil? ? "#{m[1]}#{m[3][1..-1]}" : "#{m[1]}[#{m[2]}]#{m[3][1..-1]}"
+      elsif m[1] == '\\' && !m[2].nil?
+        unescaped_attrs = "[#{m[2]}]"
+      end
+
+      if unescaped_attrs.nil? && !m[2].nil?
+        attributes = parse_attributes(m[2])
+      else
+        attributes = {}
       end
-      
-      @passthroughs << {:text => m[3], :subs => [:specialcharacters], :literal => true}
+
+      @passthroughs << {:text => m[4], :subs => [:specialcharacters], :attributes => attributes, :literal => true}
       index = @passthroughs.size - 1
-      "#{m[1]}\e#{index}\e"
+      "#{unescaped_attrs || m[1]}\e#{index}\e"
     } unless !result.include?('`')
 
     result
@@ -193,11 +228,11 @@ module Substituters
   # returns The String text with the passthrough text restored
   def restore_passthroughs(text)
     return text if @passthroughs.nil? || @passthroughs.empty? || !text.include?("\e")
-    
+
     text.gsub(REGEXP[:pass_placeholder]) {
       pass = @passthroughs[$1.to_i];
       text = apply_subs(pass[:text], pass.fetch(:subs, []))
-      pass[:literal] ? Inline.new(self, :quoted, text, :type => :monospaced).render : text
+      pass[:literal] ? Inline.new(self, :quoted, text, :type => :monospaced, :attributes => pass.fetch(:attributes, {})).render : text
     }
   end
 
@@ -214,6 +249,7 @@ module Substituters
 
     text.gsub(SPECIAL_CHARS_PATTERN) { SPECIAL_CHARS[$&] }
   end
+  alias :sub_specialchars :sub_specialcharacters
 
   # Public: Substitute quoted text (includes emphasis, strong, monospaced, etc)
   #
@@ -226,7 +262,7 @@ module Substituters
     QUOTE_SUBS.each {|type, scope, pattern|
       result.gsub!(pattern) { transform_quoted_text($~, type, scope) }
     }
-    
+
     result
   end
 
@@ -252,12 +288,12 @@ module Substituters
           when :leading
             "#{head}#{replacement}"
           when :bounding
-            "#{head}#{replacement}#{tail}" 
+            "#{head}#{replacement}#{tail}"
           end
         end
       }
     }
-    
+
     result
   end
 
@@ -273,7 +309,7 @@ module Substituters
   #--
   # NOTE it's necessary to perform this substitution line-by-line
   # so that a missing key doesn't wipe out the whole block of data
-  def sub_attributes(data)
+  def sub_attributes(data, opts = {})
     return data if data.nil? || data.empty?
 
     string_data = data.is_a? String
@@ -297,8 +333,11 @@ module Substituters
             args = expr.split(':')
             _, value = Lexer::store_attribute(args[0], args[1] || '', @document)
             if value.nil?
-              reject = true
-              break '{undefined}'
+              # since this is an assignment, only drop-line applies here (skip and drop imply the same result)
+              if @document.attributes.fetch('attribute-undefined', COMPLIANCE[:attribute_undefined]) == 'drop-line'
+                Debug.debug { "Undefining attribute: #{key}, line marked for removal" }
+                break ''
+              end
             end
             ''
           when 'counter', 'counter2'
@@ -307,7 +346,7 @@ module Substituters
             directive == 'counter2' ? '' : val
           else
             # if we get here, our attr_ref regex is too loose
-            puts "asciidoctor: WARNING: illegal attribute directive: #{m[2]}"
+            warn "asciidoctor: WARNING: illegal attribute directive: #{m[2]}"
             ''
           end
         elsif (key = m[2].downcase) && @document.attributes.has_key?(key)
@@ -315,11 +354,17 @@ module Substituters
         elsif INTRINSICS.has_key? key
           INTRINSICS[key]
         else
-          Debug.debug { "Missing attribute: #{key}, line marked for removal" }
-          reject = true
-          break '{undefined}'
+          case (opts[:attribute_missing] || @document.attributes.fetch('attribute-missing', COMPLIANCE[:attribute_missing]))
+          when 'skip'
+            "{#{key}}"
+          when 'drop-line'
+            Debug.debug { "Missing attribute: #{key}, line marked for removal" }
+            break ''
+          else # 'drop'
+            ''
+          end
         end
-      } if line.include? '{' 
+      } if line.include? '{'
 
       result << line unless reject
     }
@@ -378,14 +423,14 @@ module Substituters
                 c
               }
             end
-            Inline.new(self, :kbd, nil, :attributes => {'keys' => keys}).render 
+            Inline.new(self, :kbd, nil, :attributes => {'keys' => keys}).render
           elsif captured.start_with?('btn')
             label = unescape_bracketed_text m[1]
             Inline.new(self, :button, label).render
           end
         }
       end
-      
+
       if found[:macroish] && result.include?('menu:')
         result.gsub!(REGEXP[:menu_macro]) {
           # alias match for Ruby 1.8.7 compat
@@ -427,13 +472,37 @@ module Substituters
           input = m[1]
 
           menu, *submenus = input.split('&gt;').map(&:strip)
-          menuitem = submenus.pop 
+          menuitem = submenus.pop
           Inline.new(self, :menu, nil, :attributes => {'menu' => menu, 'submenus' => submenus, 'menuitem' => menuitem}).render
         }
       end
     end
 
-    if found[:macroish] && result.include?('image:')
+    # FIXME this location is somewhat arbitrary, probably need to be able to control ordering
+    # TODO this handling needs some cleanup
+    if (extensions = @document.extensions) && extensions.inline_macros? && found[:macroish]
+      extensions.load_inline_macro_processors(@document).each do |processor|
+        result.gsub!(processor.regexp) {
+          # alias match for Ruby 1.8.7 compat
+          m = $~
+          # honor the escape
+          if m[0].start_with? '\\'
+            next m[0][1..-1]
+          end
+
+          target = m[1]
+          if processor.options[:short_form]
+            attributes = {}
+          else
+            posattrs = processor.options.fetch(:pos_attrs, [])
+            attributes = parse_attributes(m[2], posattrs, :sub_input => true, :unescape_input => true)
+          end
+          processor.process self, target, attributes
+        }
+      end
+    end
+
+    if found[:macroish] && (result.include?('image:') || result.include?('icon:'))
       # image:filename.png[Alt Text]
       result.gsub!(REGEXP[:image_macro]) {
         # alias match for Ruby 1.8.7 compat
@@ -442,13 +511,24 @@ module Substituters
         if m[0].start_with? '\\'
           next m[0][1..-1]
         end
+
+        raw_attrs = unescape_bracketed_text m[2]
+        if m[0].start_with? 'icon:'
+          type = 'icon'
+          posattrs = ['size']
+        else
+          type = 'image'
+          posattrs = ['alt', 'width', 'height']
+        end
         target = sub_attributes(m[1])
-        @document.register(:images, target)
-        attrs = parse_attributes(unescape_bracketed_text(m[2]), ['alt', 'width', 'height'])
+        unless type == 'icon'
+          @document.register(:images, target)
+        end
+        attrs = parse_attributes(raw_attrs, posattrs)
         if !attrs['alt']
           attrs['alt'] = File.basename(target, File.extname(target))
         end
-        Inline.new(self, :image, nil, :target => target, :attributes => attrs).render
+        Inline.new(self, :image, nil, :type => type, :target => target, :attributes => attrs).render
       }
     end
 
@@ -464,10 +544,10 @@ module Substituters
         end
 
         terms = unescape_bracketed_text(m[1] || m[2]).split(',').map(&:strip)
-        document.register(:indexterms, [*terms])
+        @document.register(:indexterms, [*terms])
         Inline.new(self, :indexterm, text, :attributes => {'terms' => terms}).render
       }
-    
+
       # indexterm2:[Tigers]
       # ((Tigers))
       result.gsub!(REGEXP[:indexterm2_macro]) {
@@ -479,7 +559,7 @@ module Substituters
         end
 
         text = unescape_bracketed_text(m[1] || m[2])
-        document.register(:indexterms, [text])
+        @document.register(:indexterms, [text])
         Inline.new(self, :indexterm, text, :type => :visible).render
       }
     end
@@ -654,7 +734,33 @@ module Substituters
           id = m[2]
           reftext = !m[3].empty? ? m[3] : nil
         end
-        Inline.new(self, :anchor, reftext, :type => :xref, :target => id).render
+
+        if id.include? '#'
+          path, fragment = id.split('#')
+        else
+          path = nil
+          fragment = id
+        end
+
+        # handles form: id
+        if path.nil?
+          refid = fragment
+          target = "##{fragment}"
+        # handles forms: doc#, doc.adoc#, doc#id and doc.adoc#id
+        else
+          path = Helpers.rootname(path)
+          # the referenced path is this document, or its contents has been included in this document
+          if @document.attr?('docname', path) || @document.references[:includes].include?(path)
+            refid = fragment
+            path = nil
+            target = "##{fragment}"
+          else
+            refid = fragment.nil? ? path : "#{path}##{fragment}"
+            path = "#{path}#{@document.attr 'outfilesuffix', '.html'}"
+            target = fragment.nil? ? path : "#{path}##{fragment}"
+          end
+        end
+        Inline.new(self, :anchor, reftext, :type => :xref, :target => target, :attributes => {'path' => path, 'fragment' => fragment, 'refid' => refid}).render
       }
     end
 
@@ -707,10 +813,11 @@ module Substituters
       # alias match for Ruby 1.8.7 compat
       m = $~
       # honor the escape
-      if m[0].start_with? '\\'
-        next "&lt;#{m[1]}&gt;"
+      if m[1] == '\\'
+        # we have to do a sub since we aren't sure it's the first char
+        next m[0].sub('\\', '')
       end
-      Inline.new(self, :callout, m[1], :id => document.callouts.read_next_id).render
+      Inline.new(self, :callout, m[3], :id => @document.callouts.read_next_id).render
     }
   end
 
@@ -720,11 +827,11 @@ module Substituters
   #
   # returns The String with the post replacements rendered using the backend templates
   def sub_post_replacements(text)
-    if @document.attr? 'hardbreaks'
+    if @document.attributes['hardbreaks']
       lines = text.lines.entries
       return text if lines.size == 1
       last = lines.pop
-      "#{lines.map {|line| Inline.new(self, :break, line.rstrip.chomp(LINE_BREAK), :type => :line).render } * "\n"}\n#{last}"
+      lines.map {|line| Inline.new(self, :break, line.rstrip.chomp(LINE_BREAK), :type => :line).render }.push(last) * EOL
     else
       text.gsub(REGEXP[:line_break]) { Inline.new(self, :break, $1, :type => :line).render }
     end
@@ -738,12 +845,70 @@ module Substituters
   #
   # returns The rendered text for the quoted text region
   def transform_quoted_text(match, type, scope)
+    unescaped_attrs = nil
     if match[0].start_with? '\\'
-      match[0][1..-1]
-    elsif scope == :constrained
-      "#{match[1]}#{Inline.new(self, :quoted, match[3], :type => type, :attributes => parse_attributes(match[2])).render}"
+      if scope == :constrained && !match[2].nil?
+        unescaped_attrs = "[#{match[2]}]"
+      else
+        return match[0][1..-1]
+      end
+    end
+
+    if scope == :constrained
+      if unescaped_attrs.nil?
+        attributes = parse_quoted_text_attributes(match[2])
+        id = attributes.nil? ? nil : attributes.delete('id')
+        "#{match[1]}#{Inline.new(self, :quoted, match[3], :type => type, :id => id, :attributes => attributes).render}"
+      else
+        "#{unescaped_attrs}#{Inline.new(self, :quoted, match[3], :type => type, :attributes => {}).render}"
+      end
     else
-      Inline.new(self, :quoted, match[2], :type => type, :attributes => parse_attributes(match[1])).render
+      attributes = parse_quoted_text_attributes(match[1])
+      id = attributes.nil? ? nil : attributes.delete('id')
+      Inline.new(self, :quoted, match[2], :type => type, :id => id, :attributes => attributes).render
+    end
+  end
+
+  # Internal: Parse the attributes that are defined on quoted text
+  #
+  # str       - A String of unprocessed attributes (space-separated roles or the id/role shorthand syntax)
+  #
+  # returns nil if str is nil, an empty Hash if str is empty, otherwise a Hash of attributes (role and id only)
+  def parse_quoted_text_attributes(str)
+    return nil if str.nil?
+    return {} if str.empty?
+    str = sub_attributes(str) if str.include?('{')
+    str = str.strip
+    # for compliance, only consider first positional attribute
+    str, _ = str.split(',', 2) if str.include?(',')
+
+    if str.empty?
+      {}
+    elsif str.start_with?('.') || str.start_with?('#')
+      segments = str.split('#', 2)
+
+      if segments.length > 1
+        id, *more_roles = segments[1].split('.')
+      else
+        id = nil
+        more_roles = []
+      end
+
+      roles = segments[0].empty? ? [] : segments[0].split('.')
+      if roles.length > 1
+        roles.shift
+      end
+
+      if more_roles.length > 0
+        roles.concat more_roles
+      end
+
+      attrs = {}
+      attrs['id'] = id unless id.nil?
+      attrs['role'] = roles.empty? ? nil : (roles * ' ')
+      attrs
+    else
+      {'role' => str}
     end
   end
 
@@ -763,7 +928,7 @@ module Substituters
       # substitutions are only performed on attribute values if block is not nil
       block = self
     end
-    
+
     if opts.has_key?(:into)
       AttributeList.new(attrline, block).parse_into(opts[:into], posattrs)
     else
@@ -775,7 +940,7 @@ module Substituters
   # square brackets from text extracted from brackets
   def unescape_bracketed_text(text)
     return '' if text.empty?
-    text.strip.tr("\n", ' ').gsub('\]', ']')
+    text.strip.tr(EOL, ' ').gsub('\]', ']')
   end
 
   # Internal: Resolve the list of comma-delimited subs against the possible options.
@@ -783,28 +948,136 @@ module Substituters
   # subs - A comma-delimited String of substitution aliases
   #
   # returns An Array of Symbols representing the substitution operation
-  def resolve_subs(subs)
-    candidates = subs.split(',').map {|sub| sub.strip.to_sym}
-    resolved = candidates & SUB_OPTIONS 
+  def resolve_subs subs, type = :block, subject = nil
+    return [] if subs.nil? || subs.empty?
+    candidates = []
+    subs.split(',').each do |val|
+      key = val.strip.to_sym
+      # special case to disable callouts for inline subs
+      if key == :verbatim && type == :inline
+        candidates << :specialcharacters
+      elsif COMPOSITE_SUBS.has_key? key
+        candidates.push(*COMPOSITE_SUBS[key])
+      else
+        candidates << key
+      end
+    end
+    # weed out invalid options and remove duplicates (first wins)
+    resolved = candidates & SUB_OPTIONS[type]
     if (invalid = candidates - resolved).size > 0
-      puts "asciidoctor: WARNING: invalid passthrough macro substitution operation#{invalid.size > 1 ? 's' : ''}: #{invalid * ', '}"
-    end 
+      warn "asciidoctor: WARNING: invalid substitution type#{invalid.size > 1 ? 's' : ''}#{subject ? ' for ' : nil}#{subject}: #{invalid * ', '}"
+    end
     resolved
   end
 
+  def resolve_block_subs subs, subject
+    resolve_subs subs, :block, subject
+  end
+
+  def resolve_pass_subs subs
+    resolve_subs subs, :inline, 'passthrough macro'
+  end
+
   # Public: Highlight the source code if a source highlighter is defined
   # on the document, otherwise return the text unprocessed
   #
+  # Callout marks are stripped from the source prior to passing it to the
+  # highlighter, then later restored in rendered form, so they are not
+  # incorrectly processed by the source highlighter.
+  #
   # source - the source code String to highlight
+  # sub_callouts - a Boolean flag indicating whether callout marks should be substituted
   #
   # returns the highlighted source code, if a source highlighter is defined
   # on the document, otherwise the unprocessed text
-  def highlight_source(source)
-    Helpers.require_library 'coderay'
-    ::CodeRay::Duo[attr('language', 'text').to_sym, :html, {
-        :css => @document.attributes.fetch('coderay-css', 'class').to_sym,
-        :line_numbers => (attr?('linenums') ? @document.attributes.fetch('coderay-linenums-mode', 'table').to_sym : nil),
-        :line_number_anchors => false}].highlight(source).chomp
+  def highlight_source(source, sub_callouts, highlighter = nil)
+    highlighter ||= @document.attributes['source-highlighter']
+    Helpers.require_library highlighter, (highlighter == 'pygments' ? 'pygments.rb' : highlighter)
+    callout_marks = {}
+    lineno = 0
+    callout_on_last = false
+    if sub_callouts
+      last = -1
+      # extract callout marks, indexed by line number
+      source = source.split(EOL).map {|line|
+        lineno = lineno + 1
+        line.gsub(REGEXP[:callout_scan]) {
+          # alias match for Ruby 1.8.7 compat
+          m = $~
+          # honor the escape
+          if m[1] == '\\'
+            m[0].sub('\\', '')
+          else
+            (callout_marks[lineno] ||= []) << m[3]
+            last = lineno
+            nil
+          end
+        }
+      } * EOL
+      callout_on_last = (last == lineno)
+    end
+
+    linenums_mode = nil
+
+    case highlighter
+      when 'coderay'
+        result = ::CodeRay::Duo[attr('language', 'text').to_sym, :html, {
+            :css => @document.attributes.fetch('coderay-css', 'class').to_sym,
+            :line_numbers => (linenums_mode = (attr?('linenums') ? @document.attributes.fetch('coderay-linenums-mode', 'table').to_sym : nil)),
+            :line_number_anchors => false}].highlight(source)
+      when 'pygments'
+        lexer = ::Pygments::Lexer[attr('language')]
+        if lexer
+          opts = { :cssclass => 'pyhl', :classprefix => 'tok-', :nobackground => true }
+          opts[:noclasses] = true unless @document.attributes.fetch('pygments-css', 'class') == 'class'
+          if attr? 'linenums'
+            opts[:linenos] = (linenums_mode = @document.attributes.fetch('pygments-linenums-mode', 'table').to_sym).to_s
+          end
+
+          # FIXME stick these regexs into constants
+          if linenums_mode == :table
+            result = lexer.highlight(source, :options => opts).
+                sub(/<div class="pyhl">(.*)<\/div>/m, '\1').
+                gsub(/<pre[^>]*>(.*?)<\/pre>\s*/m, '\1')
+          else
+            result = lexer.highlight(source, :options => opts).
+                sub(/<div class="pyhl"><pre[^>]*>(.*?)<\/pre><\/div>/m, '\1')
+          end
+        else
+          result = source
+        end
+    end
+
+    if !sub_callouts || callout_marks.empty?
+      result
+    else
+      lineno = 0
+      reached_code = linenums_mode != :table
+      result.split(EOL).map {|line|
+        unless reached_code
+          unless line.include?('<td class="code">')
+            next line
+          end
+          reached_code = true
+        end
+        lineno = lineno + 1
+        if (conums = callout_marks.delete(lineno))
+          tail = nil
+          if callout_on_last && callout_marks.empty? && (pos = line.index '</pre>')
+            tail = line[pos..-1]
+            line = line[0...pos]
+          end
+          if conums.size == 1
+            %(#{line}#{Inline.new(self, :callout, conums.first, :id => @document.callouts.read_next_id).render }#{tail})
+          else
+            conums_markup = conums.map {|conum| Inline.new(self, :callout, conum, :id => @document.callouts.read_next_id).render } * ' '
+            %(#{line}#{conums_markup}#{tail})
+          end
+        else
+          line
+        end
+      } * EOL
+    end
   end
 end
 end