coradoc 2.0.21 → 2.0.22

data/lib/coradoc/include_resolver.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Coradoc
+  # Include resolver protocol.
+  #
+  # A resolver is anything that responds to +#call(target:, base_dir:,
+  # options:, context:)+ and returns the raw bytes of the included
+  # target, BEFORE tag/line/indent selectors are applied. Selectors
+  # live in the processor; the resolver only fetches bytes.
+  #
+  # The default resolver is {IncludeResolver::Filesystem}. Custom
+  # resolvers (HTTP, database, generated) plug in here without changes
+  # to the processor (OCP).
+  #
+  # Contract:
+  #   call(target:, base_dir:, options:, context:) -> String
+  #     target   String              path/URL as authored
+  #     base_dir String              absolute path to the including file's dir
+  #     options  CoreModel::IncludeOptions
+  #     context  Hash                recursion state (depth, parent_chain, ...)
+  #
+  #   Raises Coradoc::IncludeNotFoundError if the target cannot be located.
+  #   The processor's missing-file policy decides what to do with that.
+  #
+  # This base class is provided for documentation and for is_a? checks.
+  # Custom resolvers do NOT need to inherit — duck typing on the call
+  # signature is sufficient. ( SPEC 13 uses a bare Object with
+  # define_singleton_method, which we support.)
+  class IncludeResolver
+    autoload :Filesystem, "#{__dir__}/include_resolver/filesystem"
+
+    def call(target:, base_dir:, options:, context:)
+      raise NotImplementedError,
+            "#{self.class} must implement #call(target:, base_dir:, options:, context:)"
+    end
+
+    class << self
+      # Coerce +value+ into something that quacks like an IncludeResolver.
+      # - Already-callable objects (respond to :call) are returned as-is.
+      # - Symbols are interpreted as built-in names: +:filesystem+,
+      #   +:filesystem_strict+ (path-traversal protection on).
+      #
+      # @param value [Object, nil] the resolver or built-in name
+      # @param base_dir [String] required for built-in filesystem resolvers
+      # @param allow_unsafe [Boolean] opt-out of path-traversal protection
+      # @return [Object] something callable as a resolver
+      def coerce(value, base_dir:, allow_unsafe: false)
+        return Filesystem.new(base_dir: base_dir, allow_unsafe: allow_unsafe) if value.nil?
+
+        case value
+        when Symbol then coerce_symbol(value, base_dir: base_dir, allow_unsafe: allow_unsafe)
+        else value
+        end
+      end
+
+      private
+
+      def coerce_symbol(name, base_dir:, allow_unsafe:)
+        case name
+        when :filesystem then Filesystem.new(base_dir: base_dir, allow_unsafe: allow_unsafe)
+        else
+          raise ArgumentError, "Unknown include resolver: #{name.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/include_selectors/indent.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module IncludeSelectors
+    # Indent normalization. Two modes per asciidoctor:
+    #
+    #   indent=0   strip all leading whitespace from every line
+    #   indent=N   normalize leading whitespace to exactly N spaces
+    #   nil        pass through unchanged
+    module Indent
+      # @param text [String]
+      # @param options [Coradoc::CoreModel::IncludeOptions]
+      # @return [String]
+      def self.call(text, options:)
+        return text if options.indent.nil?
+
+        if options.indent.zero?
+          strip_all(text)
+        else
+          reindent(text, options.indent)
+        end
+      end
+
+      class << self
+        private
+
+        def strip_all(text)
+          text.lines.map { |line| line.sub(/\A[[:space:]]+/, '') }.join
+        end
+
+        def reindent(text, target)
+          min_indent = text.lines
+                           .reject { |l| l.strip.empty? }
+                           .map { |l| l.length - l.lstrip.length }
+                           .min || 0
+
+          pad = ' ' * target
+          text.lines.map do |line|
+            stripped = strip_common_prefix(line, min_indent)
+            if stripped.strip.empty?
+              stripped.strip + "\n"
+            else
+              pad + stripped
+            end
+          end.join
+        end
+
+        def strip_common_prefix(line, count)
+          line.sub(/\A[[:space:]]{0,#{count}}/, '')
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/include_selectors/level_offset.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module IncludeSelectors
+    # Shifts section heading levels in a parsed CoreModel subtree.
+    #
+    # Applied AFTER parsing — works on SectionElement instances. Two modes:
+    #
+    #   relative (+N / -N)   every SectionElement#level += delta
+    #   absolute (N)         the FIRST section's level becomes N, and
+    #                         descendants shift by the same delta so their
+    #                         relative structure is preserved (asciidoctor
+    #                         behavior).
+    #
+    # The processor passes a freshly-parsed subtree to this selector, so
+    # there are no external references and in-place mutation is safe.
+    module LevelOffset
+      # @param core [Coradoc::CoreModel::Base] freshly parsed — mutated
+      # @param options [Coradoc::CoreModel::IncludeOptions]
+      # @return [Coradoc::CoreModel::Base] the same core (mutated)
+      def self.call(core, options:)
+        offset = options.leveloffset
+        return core if offset.nil?
+
+        first_level = find_first_level(core)
+        actual_delta = compute_actual_delta(offset, first_level)
+        return core if actual_delta.zero?
+
+        walk_and_shift(core, actual_delta)
+        core
+      end
+
+      class << self
+        private
+
+        def compute_actual_delta(offset, first_level)
+          case offset.mode
+          when 'relative' then offset.delta
+          when 'absolute'
+            return 0 if first_level.nil?
+
+            offset.delta - first_level
+          else 0
+          end
+        end
+
+        def find_first_level(node)
+          case node
+          when Coradoc::CoreModel::SectionElement
+            node.level || 1
+          when Coradoc::CoreModel::StructuralElement, Coradoc::CoreModel::Block
+            walk_for_first_level(node.children)
+          else
+            nil
+          end
+        end
+
+        def walk_for_first_level(children)
+          return nil if children.nil?
+
+          children.each do |child|
+            lvl = find_first_level(child)
+            return lvl unless lvl.nil?
+          end
+          nil
+        end
+
+        def walk_and_shift(node, delta)
+          case node
+          when Coradoc::CoreModel::SectionElement
+            node.level = [(node.level || 1) + delta, 0].max
+            walk_children(node, delta)
+          when Coradoc::CoreModel::StructuralElement, Coradoc::CoreModel::Block
+            walk_children(node, delta)
+          end
+        end
+
+        def walk_children(node, delta)
+          return if node.children.nil?
+
+          node.children.each { |c| walk_and_shift(c, delta) }
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/include_selectors/lines.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module IncludeSelectors
+    # Line-range selection. Parses asciidoctor-style specs:
+    #
+    #   N            single line
+    #   A..B         inclusive range
+    #   A..B;C;D..E  multiple, semicolon-separated
+    #
+    # Out-of-bounds clamps gracefully (SPEC 3.4). One-based indexing
+    # (asciidoctor convention).
+    module Lines
+      SPEC_PART = %r{
+        \A
+        (?<start>\d+)
+        (?:\.\.(?<finish>\d+))?
+        \z
+      }x.freeze
+
+      # @param text [String]
+      # @param options [Coradoc::CoreModel::IncludeOptions]
+      # @return [String]
+      def self.call(text, options:)
+        return text unless options.lines?
+
+        ranges = parse_spec(options.lines_spec, max: text.lines.length)
+        return '' if ranges.empty?
+
+        indices = ranges.flat_map { |start, finish| (start..finish).to_a }.uniq.sort
+        text.lines.values_at(*indices.map { |i| i - 1 }).join
+      end
+
+      class << self
+        private
+
+        def parse_spec(spec, max:)
+          spec.split(';').map(&:strip).filter_map do |part|
+            parse_part(part, max: max)
+          end
+        end
+
+        def parse_part(part, max:)
+          SPEC_PART.match(part) do |m|
+            start = m[:start].to_i
+            finish = m[:finish] ? m[:finish].to_i : start
+            [start, finish].minmax.map { |n| clamp(n, max: max) }
+          end
+        end
+
+        def clamp(n, max:)
+          return nil if n < 1
+          return max if n > max
+
+          n
+        end
+      end
+    end
+  end
+end

data/lib/coradoc/include_selectors/tags.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Coradoc
+  module IncludeSelectors
+    # Extracts regions delimited by +// tag::name[]+ / +// end::name[]+
+    # markers from included content. Supports single, multiple,
+    # wildcard (*), and inverted (**) selection.
+    #
+    # Tag markers themselves are never emitted as text (SPEC 2.8).
+    # Unknown tag names yield empty content (SPEC 2.6). Nested tag
+    # regions are included when an outer tag is selected (SPEC 2.7).
+    #
+    # Marker forms recognized:
+    #   // tag::name[]      (AsciiDoc line comment)
+    #   ## tag::name[]      (Markdown line comment, permissive)
+    #
+    # Markers may appear on their own line; they must be the first
+    # non-whitespace token on that line (asciidoctor convention).
+    module Tags
+      MARKER_OPEN  = /\A[[:space:]]*(?:\/\/+|#+)[[:space:]]*tag::([^\[\]]+)\[[[:space:]]*\]/
+      MARKER_CLOSE = /\A[[:space:]]*(?:\/\/+|#+)[[:space:]]*end::([^\[\]]+)\[[[:space:]]*\]/
+
+      # @param text [String] raw included file content
+      # @param options [Coradoc::CoreModel::IncludeOptions]
+      # @return [String] filtered content
+      def self.call(text, options:)
+        return text unless options.tags?
+
+        if options.tags_inverted
+          inverted(text)
+        elsif options.tags_wildcard
+          wildcard(text)
+        else
+          named(text, options.tags)
+        end
+      end
+
+      class << self
+        private
+
+        def scan_markers(text)
+          markers = []
+          text.each_line.with_index do |line, idx|
+            if (m = MARKER_OPEN.match(line))
+              markers << [:open, m[1].strip, idx]
+            elsif (m = MARKER_CLOSE.match(line))
+              markers << [:close, m[1].strip, idx]
+            end
+          end
+          markers
+        end
+
+        def named(text, names)
+          wanted_indices = selected_line_indices(text, names).to_set
+          pick_lines(text, wanted_indices)
+        end
+
+        def selected_line_indices(text, wanted_names)
+          markers = scan_markers(text)
+          wanted = wanted_names.to_set
+          open_stack = []
+          emit = {}
+
+          markers.each do |kind, name, idx|
+            case kind
+            when :open
+              next unless wanted.include?(name)
+
+              open_stack.push([name, idx])
+            when :close
+              next unless wanted.include?(name)
+
+              open_idx = open_stack.rindex { |n, _| n == name }
+              next unless open_idx
+
+              _open_name, open_line = open_stack.delete_at(open_idx)
+              (open_line + 1...idx).each { |i| emit[i] = true }
+            end
+          end
+
+          emit.keys
+        end
+
+        def wildcard(text)
+          markers = scan_markers(text)
+          open_stack = []
+          emit = {}
+
+          markers.each do |kind, _name, idx|
+            case kind
+            when :open
+              open_stack.push(idx)
+            when :close
+              next if open_stack.empty?
+
+              open_line = open_stack.pop
+              (open_line + 1...idx).each { |i| emit[i] = true }
+            end
+          end
+
+          pick_lines(text, emit.keys.to_set)
+        end
+
+        def inverted(text)
+          markers = scan_markers(text)
+          open_stack = []
+          excluded = {}
+
+          markers.each do |kind, _name, idx|
+            case kind
+            when :open
+              open_stack.push(idx)
+            when :close
+              next if open_stack.empty?
+
+              open_line = open_stack.pop
+              (open_line..idx).each { |i| excluded[i] = true }
+            end
+          end
+
+          markers.each { |_kind, _name, idx| excluded[idx] = true }
+
+          lines = text.lines
+          kept = lines.each_with_index.reject { |_line, idx| excluded[idx] }
+          kept.map(&:first).join
+        end
+
+        def pick_lines(text, wanted_indices)
+          lines = text.lines
+          lines.each_with_index.select { |_line, idx| wanted_indices.include?(idx) }
+               .map(&:first).join
+        end
+      end
+    end
+  end
+end
+
+require 'set'

data/lib/coradoc/include_selectors.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Coradoc
+  # Pure-function selectors applied to resolved include content.
+  #
+  # Each selector owns exactly one transformation (MECE):
+  #   Tags         // tag::X[] ... // end::X[] region extraction
+  #   Lines        line-range selection (1..N;2;3..4)
+  #   Indent       leading-whitespace normalization
+  #   LevelOffset  section-level shift (applied AFTER parsing)
+  #
+  # Tags, Lines, and Indent take a String and return a String.
+  # LevelOffset takes a parsed CoreModel and returns a new CoreModel.
+  #
+  # The processor orchestrates the order:
+  #   1. Tags   (or Lines; Lines wins if both specified — SPEC 3.5)
+  #   2. Indent
+  #   3. parse → CoreModel
+  #   4. LevelOffset
+  module IncludeSelectors
+    autoload :Tags, "#{__dir__}/include_selectors/tags"
+    autoload :Lines, "#{__dir__}/include_selectors/lines"
+    autoload :Indent, "#{__dir__}/include_selectors/indent"
+    autoload :LevelOffset, "#{__dir__}/include_selectors/level_offset"
+  end
+end

data/lib/coradoc/resolve_includes.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module Coradoc
+  # Flat-mode include processor — the explicit "flatten" step.
+  #
+  # Walks a parsed CoreModel and expands every {CoreModel::Include} link
+  # node into the parsed content of its target, recursing into the result.
+  # The original CoreModel is NOT modified — a new subtree is constructed
+  # and spliced into place.
+  #
+  # Invoked via the public API +Coradoc.resolve_includes(doc, base_dir:)+.
+  # Callers control resolution strategy (filesystem, HTTP, custom),
+  # missing-include policy, recursion depth, and path-traversal safety.
+  #
+  # Honors:
+  #   - +missing_include+ policy: :error (default) | :warn | :silent | :passthrough
+  #   - +max_depth+ limit (raises Coradoc::IncludeDepthExceededError)
+  #   - circular detection (raises Coradoc::CircularIncludeError)
+  #   - tags/lines/indent selectors (applied to raw text before parse)
+  #   - leveloffset selector (applied to parsed CoreModel)
+  #   - +base_dir+ re-rooting (recursive includes resolve relative to
+  #     the including file — SPEC 7.2)
+  class ResolveIncludes
+    DEFAULT_MAX_DEPTH = 64
+
+    class << self
+      def call(core, resolver:, base_dir:, **opts)
+        new(resolver: resolver, base_dir: base_dir, **opts).call(core)
+      end
+    end
+
+    # @param resolver [#call] anything responding to +#call(target:, base_dir:, options:, context:)+
+    # @param base_dir [String] absolute path to the document root directory
+    # @param missing_include [Symbol] :error | :warn | :silent | :passthrough
+    # @param max_depth [Integer] recursion cap
+    # @param parse_format [Symbol] format to use when re-parsing included content
+    def initialize(resolver:, base_dir:,
+                   missing_include: :error,
+                   max_depth: DEFAULT_MAX_DEPTH,
+                   parse_format: :asciidoc)
+      @resolver = Coradoc::IncludeResolver.coerce(resolver, base_dir: base_dir)
+      @base_dir = base_dir
+      @missing_policy = missing_include
+      @max_depth = max_depth
+      @parse_format = parse_format
+    end
+
+    # Walk + transform. Returns a NEW CoreModel with includes expanded.
+    def call(core)
+      expand_node(core, base_dir: File.expand_path(@base_dir), chain: [], depth: 0)
+    end
+
+    private
+
+    def expand_node(node, base_dir:, chain:, depth:)
+      return node unless node.is_a?(Coradoc::CoreModel::Base)
+
+      case node
+      when Coradoc::CoreModel::Include
+        expand_include(node, base_dir: base_dir, chain: chain, depth: depth)
+      when Coradoc::CoreModel::StructuralElement, Coradoc::CoreModel::Block
+        expand_container(node, base_dir: base_dir, chain: chain, depth: depth)
+      else
+        node
+      end
+    end
+
+    def expand_container(node, base_dir:, chain:, depth:)
+      return node if node.children.nil? || node.children.empty?
+
+      expanded_children = node.children.flat_map do |child|
+        expanded = expand_node(child, base_dir: base_dir, chain: chain, depth: depth)
+        Array(expanded)
+      end
+
+      return node if expanded_children.equal?(node.children) || same_children?(expanded_children, node.children)
+
+      duplicate_with_children(node, expanded_children)
+    end
+
+    # The processor must not mutate its input. Each container that has
+    # expanded includes is replaced by a shallow copy with a new
+    # +children+ array — the original document tree stays intact so the
+    # caller can re-resolve with different options.
+    def duplicate_with_children(node, new_children)
+      duplicate = node.dup
+      duplicate.children = new_children
+      duplicate
+    end
+
+    def same_children?(expanded, original)
+      return false unless expanded.length == original.length
+
+      expanded.each_with_index.all? { |node, i| node.equal?(original[i]) }
+    end
+
+    def expand_include(include_node, base_dir:, chain:, depth:)
+      enforce_depth!(include_node, depth)
+      enforce_cycle!(include_node, base_dir: base_dir, chain: chain)
+
+      target = include_node.target
+      new_chain = chain + [resolve_target_path(target, base_dir)]
+
+      content = fetch_content(include_node, base_dir: base_dir)
+      return replacement_for_missing(include_node) if missing_content?(content)
+
+      applied = apply_text_selectors(content, include_node.options)
+      parsed = parse_included(applied)
+
+      shifted = Coradoc::IncludeSelectors::LevelOffset.call(parsed, options: include_node.options)
+
+      new_base_dir = File.dirname(resolve_target_path(target, base_dir))
+      expand_subtree(shifted, base_dir: new_base_dir, chain: new_chain, depth: depth + 1)
+    end
+
+    def missing_content?(content)
+      content.nil? || content == :passthrough
+    end
+
+    def fetch_content(include_node, base_dir:)
+      @resolver.call(
+        target: include_node.target,
+        base_dir: base_dir,
+        options: include_node.options,
+        context: {}
+      )
+    rescue Coradoc::IncludeNotFoundError => e
+      handle_missing(include_node, e)
+    end
+
+    def handle_missing(include_node, error)
+      case @missing_policy
+      when :error then raise error
+      when :warn
+        Coradoc::Logger.warn("Include target not found: #{include_node.target}")
+        nil
+      when :silent then nil
+      when :passthrough then :passthrough
+      else raise error
+      end
+    end
+
+    def replacement_for_missing(include_node)
+      return [include_node] if @missing_policy == :passthrough
+
+      []
+    end
+
+    def apply_text_selectors(text, options)
+      text = apply_lines_or_tags(text, options)
+      Coradoc::IncludeSelectors::Indent.call(text, options: options)
+    end
+
+    def apply_lines_or_tags(text, options)
+      # lines wins when both specified (SPEC 3.5)
+      return Coradoc::IncludeSelectors::Lines.call(text, options: options) if options.lines?
+
+      Coradoc::IncludeSelectors::Tags.call(text, options: options)
+    end
+
+    def parse_included(text)
+      return empty_core if text.nil? || text.empty?
+
+      Coradoc.parse(text, format: @parse_format)
+    end
+
+    def empty_core
+      Coradoc::CoreModel::DocumentElement.new
+    end
+
+    def expand_subtree(core, base_dir:, chain:, depth:)
+      expanded = expand_node(core, base_dir: base_dir, chain: chain, depth: depth)
+      return [expanded] unless expanded.is_a?(Coradoc::CoreModel::StructuralElement)
+
+      expanded.children || []
+    end
+
+    def enforce_depth!(include_node, depth)
+      return if depth < @max_depth
+
+      raise Coradoc::IncludeDepthExceededError.new(
+        target: include_node.target,
+        depth: depth,
+        max: @max_depth
+      )
+    end
+
+    def enforce_cycle!(include_node, base_dir:, chain:)
+      full = resolve_target_path(include_node.target, base_dir)
+      return unless chain.include?(full)
+
+      raise Coradoc::CircularIncludeError.new(
+        target: include_node.target,
+        chain: chain
+      )
+    end
+
+    def resolve_target_path(target, base_dir)
+      File.expand_path(target, base_dir)
+    end
+  end
+end

data/lib/coradoc/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coradoc
-  VERSION = '2.0.21'
+  VERSION = '2.0.22'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coradoc
 version: !ruby/object:Gem::Version
-  version: 2.0.21
+  version: 2.0.22
 platform: ruby
 authors:
 - Ribose Inc.
@@ -73,11 +73,16 @@ files:
 - lib/coradoc/core_model/frontmatter.rb
 - lib/coradoc/core_model/frontmatter/codec.rb
 - lib/coradoc/core_model/frontmatter/field_transform.rb
+- lib/coradoc/core_model/frontmatter/frontmatter_value.rb
 - lib/coradoc/core_model/frontmatter/schema_resolver.rb
 - lib/coradoc/core_model/frontmatter/text_splitter.rb
+- lib/coradoc/core_model/has_children.rb
 - lib/coradoc/core_model/horizontal_rule_block.rb
 - lib/coradoc/core_model/id_generator.rb
 - lib/coradoc/core_model/image.rb
+- lib/coradoc/core_model/include.rb
+- lib/coradoc/core_model/include_level_offset.rb
+- lib/coradoc/core_model/include_options.rb
 - lib/coradoc/core_model/inline_element.rb
 - lib/coradoc/core_model/list_block.rb
 - lib/coradoc/core_model/list_item.rb
@@ -104,6 +109,13 @@ files:
 - lib/coradoc/errors.rb
 - lib/coradoc/format_module.rb
 - lib/coradoc/hooks.rb
+- lib/coradoc/include_resolver.rb
+- lib/coradoc/include_resolver/filesystem.rb
+- lib/coradoc/include_selectors.rb
+- lib/coradoc/include_selectors/indent.rb
+- lib/coradoc/include_selectors/level_offset.rb
+- lib/coradoc/include_selectors/lines.rb
+- lib/coradoc/include_selectors/tags.rb
 - lib/coradoc/input.rb
 - lib/coradoc/logger.rb
 - lib/coradoc/output.rb
@@ -111,6 +123,7 @@ files:
 - lib/coradoc/processor_registry.rb
 - lib/coradoc/query.rb
 - lib/coradoc/registry.rb
+- lib/coradoc/resolve_includes.rb
 - lib/coradoc/serializer/registry.rb
 - lib/coradoc/transform.rb
 - lib/coradoc/transform/base.rb