asciidoctor 2.0.9 → 2.0.14

data/lib/asciidoctor/parser.rb

@@ -119,7 +119,7 @@ class Parser
   # returns the Hash of orphan block attributes captured above the header
   def self.parse_document_header(reader, document)
     # capture lines of block-level metadata and plow away comment lines that precede first block
-    block_attrs = parse_block_metadata_lines reader, document
+    block_attrs = reader.skip_blank_lines ? (parse_block_metadata_lines reader, document) : {}
     doc_attrs = document.attributes
 
     # special case, block title is not allowed above document title,
@@ -144,7 +144,10 @@ class Parser
         l0_section_title = nil
       else
         document.title = l0_section_title
-        doc_attrs['doctitle'] = doctitle_attr_val = document.apply_header_subs l0_section_title
+        if (doc_attrs['doctitle'] = doctitle_attr_val = document.sub_specialchars l0_section_title).include? ATTR_REF_HEAD
+          # QUESTION should we defer substituting attributes until the end of the header? or should we substitute again if necessary?
+          doc_attrs['doctitle'] = doctitle_attr_val = document.sub_attributes doctitle_attr_val, attribute_missing: 'skip'
+        end
       end
       document.header.source_location = source_location if source_location
       # default to compat-mode if document has setext doctitle
@@ -215,9 +218,6 @@ class Parser
           name_section = initialize_section reader, document, {}
           name_section_buffer = (reader.read_lines_until break_on_blank_lines: true, skip_line_comments: true).map {|l| l.lstrip }.join ' '
           if ManpageNamePurposeRx =~ name_section_buffer
-            doc_attrs['manname-title'] ||= name_section.title
-            doc_attrs['manname-id'] = name_section.id if name_section.id
-            doc_attrs['manpurpose'] = $2
             if (manname = $1).include? ATTR_REF_HEAD
               manname = document.sub_attributes manname
             end
@@ -226,8 +226,14 @@ class Parser
             else
               mannames = [manname]
             end
+            if (manpurpose = $2).include? ATTR_REF_HEAD
+              manpurpose = document.sub_attributes manpurpose
+            end
+            doc_attrs['manname-title'] ||= name_section.title
+            doc_attrs['manname-id'] = name_section.id if name_section.id
             doc_attrs['manname'] = manname
             doc_attrs['mannames'] = mannames
+            doc_attrs['manpurpose'] = manpurpose
             if document.backend == 'manpage'
               doc_attrs['docname'] = manname
               doc_attrs['outfilesuffix'] = %(.#{manvolnum})
@@ -327,7 +333,7 @@ class Parser
       if current_level == 0
         part = book
       elsif current_level == 1 && section.special
-        # NOTE technically preface and abstract sections are only permitted in the book doctype
+        # NOTE technically preface sections are only permitted in the book doctype
         unless (sectname = section.sectname) == 'appendix' || sectname == 'preface' || sectname == 'abstract'
           expected_next_level = nil
         end
@@ -433,8 +439,10 @@ class Parser
     # is treated like an untitled section
     elsif preamble # implies parent == document
       if preamble.blocks?
+        if book || document.blocks[1] || !Compliance.unwrap_standalone_preamble
+          preamble.source_location = preamble.blocks[0].source_location if document.sourcemap
         # unwrap standalone preamble (i.e., document has no sections) except for books, if permissible
-        unless book || document.blocks[1] || !Compliance.unwrap_standalone_preamble
+        else
           document.blocks.shift
           while (child_block = preamble.blocks.shift)
             document << child_block
@@ -858,6 +866,7 @@ class Parser
       when :literal
         block = build_block(block_context, :verbatim, terminator, parent, reader, attributes)
       when :example
+        attributes['caption'] = '' if attributes['collapsible-option']
         block = build_block(block_context, :compound, terminator, parent, reader, attributes)
       when :quote, :verse
         AttributeList.rekey(attributes, [nil, 'attribution', 'citetitle'])
@@ -899,9 +908,7 @@ class Parser
     # FIXME title and caption should be assigned when block is constructed (though we need to handle all cases)
     if attributes['title']
       block.title = block_title = attributes.delete 'title'
-      if (caption_attr_name = CAPTION_ATTR_NAMES[block.context]) && document.attributes[caption_attr_name]
-        block.assign_caption (attributes.delete 'caption')
-      end
+      block.assign_caption (attributes.delete 'caption') if CAPTION_ATTRIBUTE_NAMES[block.context]
     end
     # TODO eventually remove the style attribute from the attributes hash
     #block.style = attributes.delete 'style'
@@ -1148,14 +1155,16 @@ class Parser
   def self.catalog_inline_anchors text, block, document, reader
     text.scan InlineAnchorScanRx do
       if (id = $1)
-        if (reftext = $2)
-          next if (reftext.include? ATTR_REF_HEAD) && (reftext = document.sub_attributes reftext).empty?
-        end
+        next if (reftext = $2) && (reftext.include? ATTR_REF_HEAD) && (reftext = document.sub_attributes reftext).empty?
       else
         id = $3
         if (reftext = $4)
-          reftext = reftext.gsub '\]', ']' if reftext.include? ']'
-          next if (reftext.include? ATTR_REF_HEAD) && (reftext = document.sub_attributes reftext).empty?
+          if reftext.include? ']'
+            reftext = reftext.gsub '\]', ']'
+            reftext = document.sub_attributes reftext if reftext.include? ATTR_REF_HEAD
+          elsif (reftext.include? ATTR_REF_HEAD) && (reftext = document.sub_attributes reftext).empty?
+            next
+          end
         end
       end
       unless document.register :refs, [id, (Inline.new block, :anchor, reftext, type: :ref, id: id)]
@@ -1311,7 +1320,7 @@ class Parser
         list_item.marker = sibling_trait
         if ordinal == 0 && !style
           # using list level makes more sense, but we don't track it
-          # basing style on marker level is compliant with AsciiDoc Python
+          # basing style on marker level is compliant with AsciiDoc.py
           list_block.style = implicit_style || ((ORDERED_LIST_STYLES[sibling_trait.length - 1] || 'arabic').to_s)
         end
         if item_text.start_with?('[[') && LeadingInlineAnchorRx =~ item_text
@@ -2119,6 +2128,8 @@ class Parser
       name = 'sectnums'
     elsif name == 'hardbreaks'
       name = 'hardbreaks-option'
+    elsif name == 'showtitle'
+      store_attribute 'notitle', (value ? nil : ''), doc, attrs
     end
 
     if doc
@@ -2273,9 +2284,15 @@ class Parser
     end
 
     skipped = table_reader.skip_blank_lines || 0
+    if attributes['header-option']
+      table.has_header_option = true
+    elsif skipped == 0 && !attributes['noheader-option']
+      # NOTE: assume table has header until we know otherwise; if it doesn't (nil), cells in first row get reprocessed
+      table.has_header_option = :implicit
+      implicit_header = true
+    end
     parser_ctx = Table::ParserContext.new table_reader, table, attributes
     format, loop_idx, implicit_header_boundary = parser_ctx.format, -1, nil
-    implicit_header = true unless skipped > 0 || attributes['header-option'] || attributes['noheader-option']
 
     while (line = table_reader.read_line)
       if (beyond_first = (loop_idx += 1) > 0) && line.empty?
@@ -2295,7 +2312,7 @@ class Parser
             implicit_header_boundary = nil if implicit_header_boundary
           # otherwise, the cell continues from previous line
           elsif implicit_header_boundary && implicit_header_boundary == loop_idx
-            implicit_header, implicit_header_boundary = false, nil
+            table.has_header_option = implicit_header = implicit_header_boundary = nil
           end
         end
       end
@@ -2307,7 +2324,7 @@ class Parser
           if table_reader.has_more_lines? && table_reader.peek_line.empty?
             implicit_header_boundary = 1
           else
-            implicit_header = false
+            table.has_header_option = implicit_header = nil
           end
         end
       end
@@ -2358,7 +2375,7 @@ class Parser
           case format
           when 'csv'
             if parser_ctx.buffer_has_unclosed_quotes?
-              implicit_header, implicit_header_boundary = false, nil if implicit_header_boundary && loop_idx == 0
+              table.has_header_option = implicit_header = implicit_header_boundary = nil if implicit_header_boundary && loop_idx == 0
               parser_ctx.keep_cell_open
             else
               parser_ctx.close_cell true
@@ -2380,15 +2397,8 @@ class Parser
       end
     end
 
-    unless (table.attributes['colcount'] ||= table.columns.size) == 0 || explicit_colspecs
-      table.assign_column_widths
-    end
-
-    if implicit_header
-      table.has_header_option = true
-      attributes['header-option'] = ''
-    end
-
+    table.assign_column_widths unless (table.attributes['colcount'] ||= table.columns.size) == 0 || explicit_colspecs
+    table.has_header_option = true if implicit_header
     table.partition_header_footer attributes
 
     table
@@ -2579,9 +2589,7 @@ class Parser
           attributes['role'] = (existing_role = attributes['role']).nil_or_empty? ? (parsed_attrs[:role].join ' ') : %(#{existing_role} #{parsed_attrs[:role].join ' '})
         end
 
-        if parsed_attrs.key? :option
-          (opts = parsed_attrs[:option]).each {|opt| attributes[%(#{opt}-option)] = '' }
-        end
+        parsed_attrs[:option].each {|opt| attributes[%(#{opt}-option)] = '' } if parsed_attrs.key? :option
 
         parsed_style
       else

data/lib/asciidoctor/path_resolver.rb

@@ -331,11 +331,12 @@ class PathResolver
 
   # Public: Securely resolve a system path
   #
-  # Resolve a system path from the target relative to the start path, jail path, or working
-  # directory (specified in the constructor), in that order. If a jail path is specified, enforce
-  # that the resolved path descends from the jail path. If a jail path is not provided, the resolved
-  # path may be any location on the system. If the resolved path is absolute, use it as is (unless
-  # it breaches the jail path). Expand all parent and self references in the resolved path.
+  # Resolves the target to an absolute path on the current filesystem. The target is assumed to be
+  # relative to the start path, jail path, or working directory (specified in the constructor), in
+  # that order. If a jail path is specified, the resolved path is forced to descend from the jail
+  # path. If a jail path is not provided, the resolved path may be any location on the system. If
+  # the target is an absolute path, use it as is (unless it breaches the jail path). Expands all
+  # parent and self references in the resolved path.
   #
   # target - the String target path
   # start  - the String start path from which to resolve a relative target; falls back to jail, if
@@ -347,8 +348,9 @@ class PathResolver
   #            automatically recover when an illegal path is encountered
   #          * :target_name is used in messages to refer to the path being resolved
   #
-  # returns a String path relative to the start path, if specified, and confined to the jail path,
-  # if specified. The path is posixified and all parent and self references in the path are expanded.
+  # Returns an absolute String path relative to the start path, if specified, and confined to the
+  # jail path, if specified. The path is posixified and all parent and self references in the path
+  # are expanded.
   def system_path target, start = nil, jail = nil, opts = {}
     if jail
       raise ::SecurityError, %(Jail is not an absolute path: #{jail}) unless root? jail
@@ -362,7 +364,7 @@ class PathResolver
         if jail && !(descends_from? target_path, jail)
           if opts.fetch :recover, true
             logger.warn %(#{opts[:target_name] || 'path'} is outside of jail; recovering automatically)
-            target_segments, _ = partition_path target_path
+            target_segments, = partition_path target_path
             jail_segments, jail_root = partition_path jail
             return join_path jail_segments + target_segments, jail_root
           else
@@ -371,7 +373,7 @@ class PathResolver
         end
         return target_path
       else
-        target_segments, _ = partition_path target
+        target_segments, = partition_path target
       end
     else
       target_segments = []
@@ -387,7 +389,7 @@ class PathResolver
           return expand_path start
         end
       else
-        target_segments, _ = partition_path start
+        target_segments, = partition_path start
         start = jail || @working_dir
       end
     elsif start.nil_or_empty?
@@ -419,7 +421,7 @@ class PathResolver
     if (resolved_segments = start_segments + target_segments).include? DOT_DOT
       unresolved_segments, resolved_segments = resolved_segments, []
       if jail
-        jail_segments, _ = partition_path jail unless jail_segments
+        jail_segments, = partition_path jail unless jail_segments
         warned = false
         unresolved_segments.each do |segment|
           if segment == DOT_DOT
@@ -450,7 +452,7 @@ class PathResolver
         target_path
       elsif opts.fetch :recover, true
         logger.warn %(#{opts[:target_name] || 'path'} is outside of jail; recovering automatically)
-        jail_segments, _ = partition_path jail unless jail_segments
+        jail_segments, = partition_path jail unless jail_segments
         join_path jail_segments + target_segments, jail_root
       else
         raise ::SecurityError, %(#{opts[:target_name] || 'path'} #{target} is outside of jail: #{jail} (disallowed in safe mode))

data/lib/asciidoctor/reader.rb

@@ -44,11 +44,11 @@ class Reader
       @file = nil
       @dir = '.'
       @path = '<stdin>'
-      @lineno = 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+      @lineno = 1
     elsif ::String === cursor
       @file = cursor
       @dir, @path = ::File.split @file
-      @lineno = 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+      @lineno = 1
     else
       if (@file = cursor.file)
         @dir = cursor.dir || (::File.dirname @file)
@@ -57,7 +57,7 @@ class Reader
         @dir = cursor.dir || '.'
         @path = cursor.path || '<stdin>'
       end
-      @lineno = cursor.lineno || 1 # IMPORTANT lineno assignment must proceed prepare_lines call!
+      @lineno = cursor.lineno || 1
     end
     @lines = prepare_lines data, opts
     @source_lines = @lines.drop 0
@@ -570,17 +570,18 @@ class Reader
   #
   # data - A String Array or String of source data to be normalized.
   # opts - A Hash of options to control how lines are prepared.
-  #        :normalize - Enables line normalization, which coerces the encoding to UTF-8 and removes trailing whitespace
-  #        (optional, default: false).
+  #        :normalize - Enables line normalization, which coerces the encoding to UTF-8 and removes trailing whitespace;
+  #        :rstrip removes all trailing whitespace; :chomp removes trailing newline only (optional, not set).
   #
   # Returns A String Array of source lines. If the source data is an Array, this method returns a copy.
   def prepare_lines data, opts = {}
-    if opts[:normalize]
-      ::Array === data ? (Helpers.prepare_source_array data) : (Helpers.prepare_source_string data)
+    if (normalize = opts[:normalize])
+      trim_end = normalize == :chomp ? false : true
+      ::Array === data ? (Helpers.prepare_source_array data, trim_end) : (Helpers.prepare_source_string data, trim_end)
     elsif ::Array === data
       data.drop 0
     elsif data
-      data.split LF, -1
+      data.chomp.split LF, -1
     else
       []
     end
@@ -717,7 +718,7 @@ class PreprocessorReader < Reader
     end
 
     # effectively fill the buffer
-    if (@lines = prepare_lines data, normalize: true, condense: false, indent: attributes['indent']).empty?
+    if (@lines = prepare_lines data, normalize: @process_lines || :chomp, condense: false, indent: attributes['indent']).empty?
       pop_include
     else
       # FIXME we eventually want to handle leveloffset without affecting the lines
@@ -808,7 +809,6 @@ class PreprocessorReader < Reader
     end
 
     if opts.fetch :condense, true
-      result.shift && @lineno += 1 while (first = result[0]) && first.empty?
       result.pop while (last = result[-1]) && last.empty?
     end
 
@@ -1060,7 +1060,13 @@ class PreprocessorReader < Reader
         return inc_path
       end
 
+      if (enc = parsed_attrs['encoding']) && (::Encoding.find enc rescue nil)
+        (read_mode_params = read_mode.split ':')[1] = enc
+        read_mode = read_mode_params.join ':'
+      end unless RUBY_ENGINE_OPAL
+
       inc_linenos = inc_tags = nil
+      # NOTE attrlist is nil if missing from include directive
       if attrlist
         if parsed_attrs.key? 'lines'
           inc_linenos = []
@@ -1131,9 +1137,10 @@ class PreprocessorReader < Reader
           else
             select = base_select = wildcard = inc_tags.delete '**'
           end
+        elsif inc_tags.key? '*'
+          select = base_select = !(wildcard = inc_tags.delete '*')
         else
-          select = base_select = !(inc_tags.value? true)
-          wildcard = inc_tags.delete '*'
+          select = base_select = false
         end
         begin
           reader.call inc_path, read_mode do |f|
@@ -1148,7 +1155,7 @@ class PreprocessorReader < Reader
                     active_tag, select = tag_stack.empty? ? [nil, base_select] : tag_stack[-1]
                   elsif inc_tags.key? this_tag
                     include_cursor = create_include_cursor inc_path, expanded_target, inc_lineno
-                    if (idx = tag_stack.rindex {|key, _| key == this_tag })
+                    if (idx = tag_stack.rindex {|key,| key == this_tag })
                       idx == 0 ? tag_stack.shift : (tag_stack.delete_at idx)
                       logger.warn message_with_context %(mismatched end tag (expected '#{active_tag}' but found '#{this_tag}') at line #{inc_lineno} of include #{target_type}: #{inc_path}), source_location: cursor, include_location: include_cursor
                     else

data/lib/asciidoctor/rx.rb

@@ -269,7 +269,7 @@ module Asciidoctor
   #
   # NOTE we only have to check as far as the blank character because we know it means non-whitespace follows.
   # IMPORTANT if this regexp does not agree with the regexp for each list type, the parser will hang.
-  AnyListRx = %r(^(?:[ \t]*(?:-|\*\**|\.\.*|\u2022|\d+\.|[a-zA-Z]\.|[IVXivx]+\))[ \t]|(?!//[^/])[ \t]*[^ \t]#{CC_ANY}*?(?::::{0,2}|;;)(?:$|[ \t])|<?\d+>[ \t]))
+  AnyListRx = %r(^(?:[ \t]*(?:-|\*\**|\.\.*|\u2022|\d+\.|[a-zA-Z]\.|[IVXivx]+\))[ \t]|(?!//[^/])[ \t]*[^ \t]#{CC_ANY}*?(?::::{0,2}|;;)(?:$|[ \t])|<(?:\d+|\.)>[ \t]))
 
   # Matches an unordered list item (one level for hyphens, up to 5 levels for asterisks).
   #
@@ -469,7 +469,7 @@ module Asciidoctor
   #   footnoteref:[id,text] (legacy)
   #   footnoteref:[id] (legacy)
   #
-  InlineFootnoteMacroRx = /\\?footnote(?:(ref):|:([#{CC_WORD}-]+)?)\[(?:|(#{CC_ALL}*?[^\\]))\]/m
+  InlineFootnoteMacroRx = /\\?footnote(?:(ref):|:([#{CC_WORD}-]+)?)\[(?:|(#{CC_ALL}*?[^\\]))\](?!<\/a>)/m
 
   # Matches an image or icon inline macro.
   #
@@ -514,9 +514,10 @@ module Asciidoctor
   #   https://github.com[GitHub]
   #   <https://github.com>
   #   link:https://github.com[]
+  #   "https://github.com[]"
   #
   # FIXME revisit! the main issue is we need different rules for implicit vs explicit
-  InlineLinkRx = %r((^|link:|#{CG_BLANK}|&lt;|[>\(\)\[\];])(\\?(?:https?|file|ftp|irc)://[^\s\[\]<]*([^\s.,\[\]<]))(?:\[(|#{CC_ALL}*?[^\\])\])?)m
+  InlineLinkRx = %r((^|link:|#{CG_BLANK}|&lt;|[>\(\)\[\];"'])(\\?(?:https?|file|ftp|irc)://[^\s\[\]<]*([^\s.,\[\]<]))(?:\[(|#{CC_ALL}*?[^\\])\])?)m
 
   # Match a link or e-mail inline macro.
   #
@@ -551,7 +552,7 @@ module Asciidoctor
   #   menu:View[Page Style > No Style]
   #   menu:View[Page Style, No Style]
   #
-  InlineMenuMacroRx = /\\?menu:(#{CG_WORD}|[#{CC_WORD}&][^\n\[]*[^\s\[])\[ *(?:|(#{CC_ALL}*?[^\\]))?\]/m
+  InlineMenuMacroRx = /\\?menu:(#{CG_WORD}|[#{CC_WORD}&][^\n\[]*[^\s\[])\[ *(?:|(#{CC_ALL}*?[^\\]))\]/m
 
   # Matches an implicit menu inline macro.
   #
@@ -591,7 +592,7 @@ module Asciidoctor
   #   $$text$$
   #   pass:quotes[text]
   #
-  # NOTE we have to support an empty pass:[] for compatibility with AsciiDoc Python
+  # NOTE we have to support an empty pass:[] for compatibility with AsciiDoc.py
   InlinePassMacroRx = /(?:(?:(\\?)\[([^\]]+)\])?(\\{0,2})(\+\+\+?|\$\$)(#{CC_ALL}*?)\4|(\\?)pass:([a-z]+(?:,[a-z-]+)*)?\[(|#{CC_ALL}*?[^\\])\])/m
 
   # Matches an xref (i.e., cross-reference) inline macro, which may span multiple lines.
@@ -610,7 +611,7 @@ module Asciidoctor
   # Matches a trailing + preceded by at least one space character,
   # which forces a hard line break (<br> tag in HTML output).
   #
-  # NOTE AsciiDoc Python allows + to be preceded by TAB; Asciidoctor does not
+  # NOTE AsciiDoc.py allows + to be preceded by TAB; Asciidoctor does not
   #
   # Examples
   #

data/lib/asciidoctor/substitutors.rb

@@ -331,8 +331,8 @@ module Substitutors
             target ||= ext_config[:format] == :short ? content : target
           end
           if (Inline === (replacement = extension.process_method[self, target, attributes]))
-            if (inline_subs = replacement.attributes.delete 'subs')
-              replacement.text = apply_subs replacement.text, (expand_subs inline_subs)
+            if (inline_subs = replacement.attributes.delete 'subs') && (inline_subs = expand_subs inline_subs, 'custom inline macro')
+              replacement.text = apply_subs replacement.text, inline_subs
             end
             replacement.convert
           elsif replacement
@@ -542,14 +542,17 @@ module Substitutors
         end
 
         prefix, suffix = $1, ''
-        # NOTE if $4 is set, then we're looking at a formal macro
+        # NOTE if $4 is set, we're looking at a formal macro (e.g., https://example.org[])
         if $4
           prefix = '' if prefix == 'link:'
           text = $4
         else
-          # invalid macro syntax (link: prefix w/o trailing square brackets)
-          # FIXME we probably shouldn't even get here...our regex is doing too much
-          next $& if prefix == 'link:'
+          # invalid macro syntax (link: prefix w/o trailing square brackets or enclosed in double quotes)
+          # FIXME we probably shouldn't even get here when the link: prefix is present; the regex is doing too much
+          case prefix
+          when 'link:', ?", ?'
+            next $&
+          end
           text = ''
           case $3
           when ')'
@@ -591,7 +594,8 @@ module Substitutors
         unless text.empty?
           text = text.gsub ESC_R_SB, R_SB if text.include? R_SB
           if !doc.compat_mode && (text.include? '=')
-            text = (attrs = (AttributeList.new text, self).parse)[1] || ''
+            # NOTE if an equals sign (=) is present, extract attributes from text
+            text, attrs = extract_attributes_from_text text, ''
             link_opts[:id] = attrs['id']
           end
 
@@ -637,7 +641,8 @@ module Substitutors
           text = text.gsub ESC_R_SB, R_SB if text.include? R_SB
           if mailto
             if !doc.compat_mode && (text.include? ',')
-              text = (attrs = (AttributeList.new text, self).parse)[1] || ''
+              # NOTE if a comma (,) is present, extract attributes from text
+              text, attrs = extract_attributes_from_text text, ''
               link_opts[:id] = attrs['id']
               if attrs.key? 2
                 if attrs.key? 3
@@ -648,7 +653,8 @@ module Substitutors
               end
             end
           elsif !doc.compat_mode && (text.include? '=')
-            text = (attrs = (AttributeList.new text, self).parse)[1] || ''
+            # NOTE if an equals sign (=) is present, extract attributes from text
+            text, attrs = extract_attributes_from_text text, ''
             link_opts[:id] = attrs['id']
           end
 
@@ -739,8 +745,8 @@ module Substitutors
           refid = $2
           if (text = $3)
             text = text.gsub ESC_R_SB, R_SB if text.include? R_SB
-            # NOTE if an equal sign (=) is present, parse text as attributes
-            text = ((AttributeList.new text, self).parse_into attrs)[1] if !doc.compat_mode && (text.include? '=')
+            # NOTE if an equals sign (=) is present, extract attributes from text
+            text, attrs = extract_attributes_from_text text if !doc.compat_mode && (text.include? '=')
           end
         end
 
@@ -804,7 +810,7 @@ module Substitutors
         # handles: id (in compat mode or when natural xrefs are disabled)
         elsif doc.compat_mode || !Compliance.natural_xrefs
           refid, target = fragment, %(##{fragment})
-          logger.info %(possible invalid reference: #{refid}) if logger.info? && doc.catalog[:refs][refid]
+          logger.info %(possible invalid reference: #{refid}) if logger.info? && !doc.catalog[:refs][refid]
         # handles: id
         elsif doc.catalog[:refs][fragment]
           refid, target = fragment, %(##{fragment})
@@ -843,19 +849,17 @@ module Substitutors
         end
 
         if id
-          if text
+          if (footnote = doc.footnotes.find {|candidate| candidate.id == id })
+            index, text = footnote.index, footnote.text
+            type, target, id = :xref, id, nil
+          elsif text
             text = restore_passthroughs(normalize_text text, true, true)
             index = doc.counter('footnote-number')
             doc.register(:footnotes, Document::Footnote.new(index, id, text))
             type, target = :ref, nil
           else
-            if (footnote = doc.footnotes.find {|candidate| candidate.id == id })
-              index, text = footnote.index, footnote.text
-            else
-              logger.warn %(invalid footnote reference: #{id})
-              index, text = nil, id
-            end
-            type, target, id = :xref, id, nil
+            logger.warn %(invalid footnote reference: #{id})
+            type, target, text, id = :xref, id, id, nil
           end
         elsif text
           text = restore_passthroughs(normalize_text text, true, true)
@@ -917,7 +921,7 @@ module Substitutors
         # use sub since it might be behind a line comment
         $&.sub RS, ''
       else
-        Inline.new(self, :callout, $4 == '.' ? (autonum += 1).to_s : $4, id: @document.callouts.read_next_id, attributes: { 'guard' => $1 }).convert
+        Inline.new(self, :callout, $4 == '.' ? (autonum += 1).to_s : $4, id: @document.callouts.read_next_id, attributes: { 'guard' => $1 || ($3 == '--' ? ['<!--', '-->'] : nil) }).convert
       end
     end
   end
@@ -945,7 +949,7 @@ module Substitutors
     if (linenums_mode = (attr? 'linenums') ? (doc_attrs[%(#{syntax_hl_name}-linenums-mode)] || :table).to_sym : nil)
       start_line_number = 1 if (start_line_number = (attr 'start', 1).to_i) < 1
     end
-    highlight_lines = resolve_lines_to_highlight source, (attr 'highlight') if attr? 'highlight'
+    highlight_lines = resolve_lines_to_highlight source, (attr 'highlight'), start_line_number if attr? 'highlight'
 
     highlighted, source_offset = syntax_hl.highlight self, source, (attr 'language'),
       callouts: callout_marks,
@@ -968,9 +972,10 @@ module Substitutors
   #
   # source - The String source.
   # spec   - The lines specifier (e.g., "1-5, !2, 10" or "1..5;!2;10")
+  # start  - The line number of the first line (optional, default: false)
   #
   # Returns an [Array] of unique, sorted line numbers.
-  def resolve_lines_to_highlight source, spec
+  def resolve_lines_to_highlight source, spec, start = nil
     lines = []
     spec = spec.delete ' ' if spec.include? ' '
     ((spec.include? ',') ? (spec.split ',') : (spec.split ';')).map do |entry|
@@ -981,21 +986,22 @@ module Substitutors
       if (delim = (entry.include? '..') ? '..' : ((entry.include? '-') ? '-' : nil))
         from, delim, to = entry.partition delim
         to = (source.count LF) + 1 if to.empty? || (to = to.to_i) < 0
-        line_nums = (from.to_i..to).to_a
-        if negate
-          lines -= line_nums
-        else
-          lines.concat line_nums
-        end
-      else
         if negate
-          lines.delete entry.to_i
+          lines -= (from.to_i..to).to_a
         else
-          lines << entry.to_i
+          lines |= (from.to_i..to).to_a
         end
+      elsif negate
+        lines.delete entry.to_i
+      elsif !lines.include?(line = entry.to_i)
+        lines << line
       end
     end
-    lines.sort.uniq
+    # If the start attribute is defined, then the lines to highlight specified by the provided spec should be relative to the start value.
+    unless (shift = start ? start - 1 : 0) == 0
+      lines = lines.map {|it| it - shift }
+    end
+    lines.sort
   end
 
   # Public: Extract the passthrough text from the document for reinsertion after processing.
@@ -1112,7 +1118,7 @@ module Substitutors
       end
       subs = $2
       content = normalize_text $3, nil, true
-      # NOTE drop enclosing $ signs around latexmath for backwards compatibility with AsciiDoc Python
+      # NOTE drop enclosing $ signs around latexmath for backwards compatibility with AsciiDoc.py
       content = content.slice 1, content.length - 2 if type == :latexmath && (content.start_with? '$') && (content.end_with? '$')
       subs = subs ? (resolve_pass_subs subs) : ((@document.basebackend? 'html') ? BASIC_SUBS : nil)
       passthrus[passthru_key = passthrus.size] = { text: content, subs: subs, type: type }
@@ -1226,17 +1232,16 @@ module Substitutors
     resolve_subs subs, :inline, nil, 'passthrough macro'
   end
 
-  # Public: Expand all groups in the subs list and return. If no subs are resolve, return nil.
+  # Public: Expand all groups in the subs list and return. If no subs are resolved, return nil.
   #
-  # subs - The substitutions to expand; can be a Symbol, Symbol Array or nil
+  # subs - The substitutions to expand; can be a Symbol, Symbol Array, or String
+  # subject - The String to use in log messages to communicate the subject for which subs are being resolved (default: nil)
   #
   # Returns a Symbol Array of substitutions to pass to apply_subs or nil if no substitutions were resolved.
-  def expand_subs subs
+  def expand_subs subs, subject = nil
     if ::Symbol === subs
-      unless subs == :none
-        SUB_GROUPS[subs] || [subs]
-      end
-    else
+      subs == :none ? nil : SUB_GROUPS[subs] || [subs]
+    elsif ::Array === subs
       expanded_subs = []
       subs.each do |key|
         unless key == :none
@@ -1247,8 +1252,9 @@ module Substitutors
           end
         end
       end
-
       expanded_subs.empty? ? nil : expanded_subs
+    else
+      resolve_subs subs, :inline, nil, subject
     end
   end
 
@@ -1270,7 +1276,7 @@ module Substitutors
         # NOTE :literal with listparagraph-option gets folded into text of list item later
         default_subs = @context == :verse ? NORMAL_SUBS : VERBATIM_SUBS
       when :raw
-        # TODO make pass subs a compliance setting; AsciiDoc Python performs :attributes and :macros on a pass block
+        # TODO make pass subs a compliance setting; AsciiDoc.py performs :attributes and :macros on a pass block
         default_subs = @context == :stem ? BASIC_SUBS : NO_SUBS
       else
         return @subs
@@ -1321,10 +1327,23 @@ module Substitutors
 
   private
 
+  # This method is used in cases when the attrlist can be mixed with the text of a macro.
+  # If no attributes are detected aside from the first positional attribute, and the first positional
+  # attribute matches the attrlist, then the original text is returned.
+  def extract_attributes_from_text text, default_text = nil
+    attrlist = (text.include? LF) ? (text.tr LF, ' ') : text
+    if (resolved_text = (attrs = (AttributeList.new attrlist, self).parse)[1])
+      # NOTE if resolved text remains unchanged, clear attributes and return unparsed text
+      resolved_text == attrlist ? [text, attrs.clear] : [resolved_text, attrs]
+    else
+      [default_text, attrs]
+    end
+  end
+
   # Internal: Extract the callout numbers from the source to prepare it for syntax highlighting.
   def extract_callouts source
     callout_marks = {}
-    lineno = 0
+    autonum = lineno = 0
     last_lineno = nil
     callout_rx = (attr? 'line-comment') ? CalloutExtractRxMap[attr 'line-comment'] : CalloutExtractRx
     # extract callout marks, indexed by line number
@@ -1336,7 +1355,7 @@ module Substitutors
           # use sub since it might be behind a line comment
           $&.sub RS, ''
         else
-          (callout_marks[lineno] ||= []) << [$1, $4]
+          (callout_marks[lineno] ||= []) << [$1 || ($3 == '--' ? ['<!--', '-->'] : nil), $4 == '.' ? (autonum += 1).to_s : $4]
           last_lineno = lineno
           ''
         end
@@ -1358,15 +1377,15 @@ module Substitutors
     else
       preamble = ''
     end
-    autonum = lineno = 0
+    lineno = 0
     preamble + ((source.split LF, -1).map do |line|
       if (conums = callout_marks.delete lineno += 1)
         if conums.size == 1
-          guard, conum = conums[0]
-          %(#{line}#{Inline.new(self, :callout, conum == '.' ? (autonum += 1).to_s : conum, id: @document.callouts.read_next_id, attributes: { 'guard' => guard }).convert})
+          guard, numeral = conums[0]
+          %(#{line}#{Inline.new(self, :callout, numeral, id: @document.callouts.read_next_id, attributes: { 'guard' => guard }).convert})
         else
-          %(#{line}#{conums.map do |guard_it, conum_it|
-            Inline.new(self, :callout, conum_it == '.' ? (autonum += 1).to_s : conum_it, id: @document.callouts.read_next_id, attributes: { 'guard' => guard_it }).convert
+          %(#{line}#{conums.map do |guard_it, numeral_it|
+            Inline.new(self, :callout, numeral_it, id: @document.callouts.read_next_id, attributes: { 'guard' => guard_it }).convert
           end.join ' '})
         end
       else