asciisourcerer 0.1.0 → 0.2.1

data/lib/sourcerer/asciidoc.rb

@@ -208,7 +208,8 @@ module Sourcerer
         source_path,
         conversion_source_text,
         backend: selected_backend,
-        header_footer: options[:header_footer])
+        header_footer: options[:header_footer],
+        attributes: options[:attributes])
 
       frontmatter = options[:include_frontmatter] ? extract_frontmatter(source_text, document.attributes) : {}
       html_body = document.convert
@@ -239,7 +240,7 @@ module Sourcerer
 
     # @api private
     def self.normalize_mark_down_grade_options options
-      supported_option_keys = %i[html_output_path backend header_footer include_frontmatter markdown_options]
+      supported_option_keys = %i[html_output_path backend header_footer include_frontmatter markdown_options attributes]
       unknown_option_keys = options.keys - supported_option_keys
       raise ArgumentError, "unknown option(s): #{unknown_option_keys.join(', ')}" unless unknown_option_keys.empty?
 
@@ -248,7 +249,8 @@ module Sourcerer
         backend: options.fetch(:backend, 'asciidoctor-html5s'),
         header_footer: options.fetch(:header_footer, false),
         include_frontmatter: options.fetch(:include_frontmatter, true),
-        markdown_options: options.fetch(:markdown_options, { github_flavored: true })
+        markdown_options: options.fetch(:markdown_options, { github_flavored: true }),
+        attributes: options.fetch(:attributes, {})
       }
     end
 
@@ -375,8 +377,9 @@ module Sourcerer
     # @param source_text [String]
     # @param backend [String]
     # @param header_footer [Boolean]
+    # @param attributes [Hash]
     # @return [Asciidoctor::Document]
-    def self.load_document_for_markdown_grade source_path, source_text, backend:, header_footer:
+    def self.load_document_for_markdown_grade source_path, source_text, backend:, header_footer:, attributes: {}
       expanded_source_path = File.expand_path(source_path)
       Asciidoctor.load(
         source_text,
@@ -385,11 +388,12 @@ module Sourcerer
         backend: backend,
         header_footer: header_footer,
         base_dir: File.dirname(source_path),
-        attributes: {
-          'docfile' => expanded_source_path,
-          'docdir' => File.dirname(expanded_source_path),
-          'docname' => File.basename(source_path, File.extname(source_path))
-        })
+        attributes: attributes.merge(
+          {
+            'docfile' => expanded_source_path,
+                    'docdir' => File.dirname(expanded_source_path),
+                    'docname' => File.basename(source_path, File.extname(source_path))
+          }))
     end
 
     # Extracts commands from listing and literal blocks with a specific role.

data/lib/sourcerer/attributes_filter.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'asciidoctor'
+
+module Sourcerer
+  # Utilities for filtering and partitioning Asciidoctor document attributes.
+  #
+  # The primary use case is separating user-defined ("custom") attributes from
+  # those injected by Asciidoctor at parse time ("built-in"). This distinction
+  # matters when a skim consumer needs to inspect only the attributes an author
+  # explicitly set in their source.
+  #
+  # Additional attribute manipulation helpers may be added here over time.
+  #
+  # @example
+  #   custom = Sourcerer::AttributesFilter.user_attributes(doc)
+  #   builtin = Sourcerer::AttributesFilter.builtin_attributes(doc)
+  module AttributesFilter
+    # Attribute keys injected by Asciidoctor at parse time rather than defined
+    # by the document author.
+    BUILTIN_ATTR_KEYS = (Asciidoctor::DEFAULT_ATTRIBUTES.keys + %w[
+      asciidoctor asciidoctor-version
+      attribute-missing attribute-undefined
+      authorcount
+      docdate docdatetime docdir docfile docfilesuffix docname doctime doctitle doctype docyear
+      embedded
+      htmlsyntax
+      iconsdir
+      localdate localdatetime localtime localyear
+      max-include-depth
+      notitle
+      outfilesuffix
+      stylesdir
+      toc-position
+      user-home
+    ]).freeze
+
+    BUILTIN_ATTR_PATTERNS = [
+      /^backend(-|$)/,
+      /^basebackend(-|$)/,
+      /^doctype-/,
+      /^filetype(-|$)/,
+      /^safe-mode-/
+    ].freeze
+
+    module_function
+
+    # Returns a hash of user-defined attributes, excluding any key that belongs
+    # to Asciidoctor's built-in set.
+    #
+    # @param doc [Asciidoctor::Document]
+    # @return [Hash{String => String}]
+    def user_attributes doc
+      doc.attributes.reject do |k, _|
+        BUILTIN_ATTR_KEYS.include?(k) ||
+          BUILTIN_ATTR_PATTERNS.any? { |pat| pat.match?(k) }
+      end
+    end
+
+    # Returns a hash of built-in Asciidoctor attributes, i.e., those injected at
+    # parse time rather than authored in the document.
+    #
+    # @param doc [Asciidoctor::Document]
+    # @return [Hash{String => String}]
+    def builtin_attributes doc
+      doc.attributes.select do |k, _|
+        BUILTIN_ATTR_KEYS.include?(k) ||
+          BUILTIN_ATTR_PATTERNS.any? { |pat| pat.match?(k) }
+      end
+    end
+  end
+end

data/lib/sourcerer/rendering.rb

@@ -178,6 +178,35 @@ module Sourcerer
       }
       template.render(context, options)
     end
+
+    # Render a Liquid template string directly with a data hash.
+    #
+    # Unlike {.render_template}, this method accepts an in-memory string and
+    # a plain Ruby Hash rather than paths to data files.  Suitable for
+    # rendering individual block content (e.g. in Sync/Cast) without setting
+    # up a full template pipeline.
+    #
+    # Keys in `data` are stringified to satisfy Liquid's string-key contract.
+    # Nested key stringification is shallow; callee is responsible for deeper
+    # transformations if required.
+    #
+    # The Jekyll/Liquid runtime is initialized before rendering so that any
+    # custom filters or tags registered elsewhere in Sourcerer are available.
+    #
+    # @param content [String] Liquid template source.
+    # @param data [Hash] Variables available to the template.
+    # @return [String] Rendered output.
+    def self.render_liquid_string content, data
+      require_relative 'jekyll'
+      require_relative 'jekyll/liquid/filters'
+      require_relative 'jekyll/liquid/tags'
+      require 'liquid' unless defined?(Liquid::Template)
+      Sourcerer::Jekyll.initialize_liquid_runtime
+
+      template = Liquid::Template.parse(content)
+      template.render(data.transform_keys(&:to_s))
+    end
+
     private_class_method :load_render_data,
                          :resolve_converter,
                          :render_erb,

data/lib/sourcerer/source_skim/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sourcerer
+  module SourceSkim
+    # All recognized element categories. Sections are shape-controlled via +forms+
+    # rather than listed here.
+    ALL_CATEGORIES = %i[
+      attributes_custom
+      attributes_builtin
+      definition_lists
+      code_blocks
+      literal_blocks
+      examples
+      sidebars
+      tables
+      admonitions
+      quotes
+      images
+    ].freeze
+
+    # Categories included when a caller passes +categories: nil+ (the default).
+    # +attributes_builtin+, +admonitions+, and +quotes+ are opt-in only.
+    DEFAULT_CATEGORIES = (ALL_CATEGORIES - %i[attributes_builtin admonitions quotes]).freeze
+
+    # Configuration profile for a single SourceSkim pass.
+    #
+    # Controls which section shapes and element categories are emitted.
+    # Callers pass a +Config+ instance to {Skimmer#process}; it is not part of
+    # the public-facing module API and should be constructed via the keyword
+    # arguments on {Sourcerer::SourceSkim.skim_file} and friends.
+    # @api private
+    class Config
+      attr_reader :forms, :categories
+
+      def initialize forms: [:tree], categories: nil
+        @forms = Array(forms).map(&:to_sym)
+        @categories = categories ? Array(categories).map(&:to_sym) : DEFAULT_CATEGORIES.dup
+      end
+
+      def include? category
+        @categories.include?(category.to_sym)
+      end
+
+      def tree?
+        @forms.include?(:tree)
+      end
+
+      def flat?
+        @forms.include?(:flat)
+      end
+    end
+  end
+end

data/lib/sourcerer/source_skim/skimmer.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+module Sourcerer
+  module SourceSkim
+    # Traverses a parsed Asciidoctor document and produces a JSON-ready skim hash.
+    #
+    # A new instance should be created per-document call; instance variables
+    # accumulate state during a single +process+ pass.
+    #
+    # This class is an internal implementation detail. External callers should
+    # use the {Sourcerer::SourceSkim} module-level methods rather than
+    # instantiating +Skimmer+ directly.
+    # @api private
+    class Skimmer
+      # Scan listing/literal block source for include directives that remain as
+      # raw text (i.e., were not resolved by the parser).
+      INCLUDE_DIRECTIVE_PATTERN = /include::[^\[]+\[[^\]]*\]/
+
+      def process document, config: Config.new
+        @config = config
+        @main_file = document.attr('docfile')
+        @definition_lists = []
+        @code_blocks       = []
+        @literal_blocks    = []
+        @examples          = []
+        @sidebars          = []
+        @tables            = []
+        @admonitions       = []
+        @quotes            = []
+        @images            = []
+
+        process_blocks(document.blocks, 0, nil)
+
+        tree    = build_sections_tree(document.sections)
+        doc_end = line_count_for(@main_file)
+        assign_line_ends(tree, doc_end)
+
+        result = {
+          title: document.doctitle,
+          lines: doc_end
+        }
+
+        if @config.include?(:attributes_custom)
+          result[:attributes_custom] =
+            Sourcerer::AttributesFilter.user_attributes(document)
+        end
+        if @config.include?(:attributes_builtin)
+          result[:attributes_builtin] =
+            Sourcerer::AttributesFilter.builtin_attributes(document)
+        end
+
+        result[:sections_tree] = tree if @config.tree?
+        result[:sections_flat] = flatten_sections(tree) if @config.flat?
+
+        result[:definition_lists] = @definition_lists if @config.include?(:definition_lists)
+        result[:code_blocks]      = @code_blocks       if @config.include?(:code_blocks)
+        result[:literal_blocks]   = @literal_blocks    if @config.include?(:literal_blocks)
+        result[:examples]         = @examples          if @config.include?(:examples)
+        result[:sidebars]         = @sidebars          if @config.include?(:sidebars)
+        result[:tables]           = @tables            if @config.include?(:tables)
+        result[:admonitions]      = @admonitions       if @config.include?(:admonitions)
+        result[:quotes]           = @quotes            if @config.include?(:quotes)
+        result[:images]           = @images            if @config.include?(:images)
+
+        result
+      end
+
+      private
+
+      def line_count_for file_path
+        return nil unless file_path && File.exist?(file_path)
+
+        File.foreach(file_path).inject(0) { |c, _| c + 1 }
+      end
+
+      # Returns the relative filename when +loc+ originates from an included file,
+      # nil otherwise (meaning the block came from the main document).
+      def file_for loc
+        return nil unless loc
+
+        f = loc.file
+        return nil unless f
+        return nil if @main_file && f == @main_file
+
+        loc.path || File.basename(f)
+      end
+
+      def build_sections_tree sections, level = 0
+        sections.map do |section|
+          loc = section.source_location
+          f   = file_for(loc)
+          record = {}
+          record[:file] = f if f
+          record.merge!(
+            id:        section.id,
+            text:      section.title,
+            level:     level + 1,
+            starts_at: loc&.lineno,
+            sections:  build_sections_tree(section.sections, level + 1))
+          record
+        end
+      end
+
+      # Assigns +ends_near+ to each section in-place. +parent_end_line+ is the
+      # last line of the enclosing scope (document total or parent section end).
+      def assign_line_ends sections, parent_end_line
+        sections.each_with_index do |rec, i|
+          next_start = sections[i + 1]&.dig(:starts_at)
+          end_line   = next_start ? next_start - 1 : parent_end_line
+          # Omit ends_near from include-sourced nodes: their starts_at is a line
+          # number in the included file, so mixing it with a main-file end bound
+          # would produce a misleading range.
+          rec[:ends_near] = end_line unless rec.key?(:file)
+          assign_line_ends(rec[:sections], end_line)
+        end
+      end
+
+      # Returns a pre-order flat array from the annotated tree. Each record
+      # carries +parent_id+ (nil for root) and +sections+ as an array of child IDs.
+      def flatten_sections sections, acc = [], parent_id = nil
+        sections.each do |rec|
+          children  = rec[:sections]
+          flat_rec  = rec.except(:sections)
+          flat_rec[:parent_id] = parent_id
+          flat_rec[:sections]  = children.map { |c| c[:id] }
+          acc << flat_rec
+          flatten_sections(children, acc, rec[:id])
+        end
+        acc
+      end
+
+      def detect_includes source
+        return [] unless source
+
+        source.scan(INCLUDE_DIRECTIVE_PATTERN)
+      end
+
+      def process_blocks blocks, level, section_id
+        # rubocop:disable Metrics/BlockLength
+        blocks.each do |block|
+          case block.context
+          when :section
+            process_blocks(block.blocks, level + 1, block.id)
+
+          when :dlist
+            next unless @config.include?(:definition_lists)
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file]  = f if f
+            entry[:title] = block.title
+            entry[:role]  = block.style if block.style && !block.style.empty?
+            entry.merge!(
+              starts_at:        loc&.lineno,
+              section_id:       section_id,
+              definition_terms: block.items.flat_map do |terms, _|
+                terms.map do |term|
+                  tloc = term.source_location
+                  { text: term.text, starts_at: tloc&.lineno }
+                end
+              end)
+            @definition_lists << entry
+
+          when :listing
+            next unless @config.include?(:code_blocks)
+            next unless block.title
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file] = f if f
+            entry.merge!(title: block.title, starts_at: loc&.lineno)
+            entry[:language]   = block.attr('language') if block.style == 'source'
+            entry[:section_id] = section_id
+            entry[:includes]   = detect_includes(block.source)
+            @code_blocks << entry
+
+          when :literal
+            next unless @config.include?(:literal_blocks)
+            next unless block.title
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file] = f if f
+            entry.merge!(
+              title:      block.title,
+              starts_at:  loc&.lineno,
+              section_id: section_id,
+              includes:   detect_includes(block.source))
+            @literal_blocks << entry
+
+          when :example
+            next unless @config.include?(:examples)
+            next unless block.title
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file] = f if f
+            entry.merge!(
+              title:      block.title,
+              starts_at:  loc&.lineno,
+              section_id: section_id,
+              includes:   [])
+            @examples << entry
+            process_blocks(block.blocks, level, section_id) if block.blocks.any?
+
+          when :sidebar
+            next unless @config.include?(:sidebars)
+            next unless block.title
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file] = f if f
+            entry.merge!(
+              title:      block.title,
+              starts_at:  loc&.lineno,
+              section_id: section_id,
+              includes:   [])
+            @sidebars << entry
+            process_blocks(block.blocks, level, section_id) if block.blocks.any?
+
+          when :table
+            next unless @config.include?(:tables)
+
+            header_row = block.rows.head.first
+            headers    = header_row&.map(&:text) if header_row && !header_row.empty?
+            next unless block.title || headers
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file]    = f if f
+            entry[:title]   = block.title
+            entry[:headers] = headers if headers
+            entry.merge!(starts_at: loc&.lineno, section_id: section_id)
+            @tables << entry
+
+          when :admonition
+            next unless @config.include?(:admonitions)
+            next unless block.title
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file] = f if f
+            entry.merge!(
+              type:       block.style,
+              title:      block.title,
+              starts_at:  loc&.lineno,
+              section_id: section_id)
+            @admonitions << entry
+            process_blocks(block.blocks, level, section_id) if block.respond_to?(:blocks) && block.blocks.any?
+
+          when :quote, :verse
+            next unless @config.include?(:quotes)
+
+            attribution = block.attr('attribution')
+            next unless block.title || attribution
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file]        = f if f
+            entry[:title]       = block.title
+            entry[:attribution] = attribution if attribution
+            entry.merge!(starts_at: loc&.lineno, section_id: section_id)
+            @quotes << entry
+
+          when :image
+            next unless @config.include?(:images)
+
+            loc   = block.source_location
+            f     = file_for(loc)
+            entry = { id: block.id }
+            entry[:file]  = f if f
+            entry[:title] = block.title if block.title
+            entry.merge!(
+              target:     block.attr('target'),
+              alt:        block.attr('alt'),
+              starts_at:  loc&.lineno,
+              section_id: section_id)
+            entry[:width]  = block.attr('width') if block.attr('width')
+            entry[:height] = block.attr('height') if block.attr('height')
+            @images << entry
+
+          else
+            process_blocks(block.blocks, level, section_id) if block.respond_to?(:blocks) && block.blocks.any?
+          end
+        end
+        # rubocop:enable Metrics/BlockLength
+      end
+    end
+  end
+end

data/lib/sourcerer/source_skim.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'asciidoctor'
+require 'logger'
+require_relative 'attributes_filter'
+require_relative 'source_skim/config'
+require_relative 'source_skim/skimmer'
+
+module Sourcerer
+  # SourceSkim produces machine-oriented skims of AsciiDoc source documents.
+  #
+  # A skim is a structured, JSON-ready representation of selected source elements
+  # intended to help automated tooling inspect documentation source and identify
+  # likely areas of interest when related product code changes.
+  #
+  # @example Skim a file with default tree output
+  #   skim = Sourcerer::SourceSkim.skim_file('docs/install.adoc')
+  #
+  # @example Skim with both tree and flat section shapes
+  #   skim = Sourcerer::SourceSkim.skim_file('docs/install.adoc', forms: [:tree, :flat])
+  #
+  # @example Skim a content string
+  #   skim = Sourcerer::SourceSkim.skim_string(adoc_content, forms: [:flat])
+  #
+  # @example Skim with caller-supplied attribute overrides
+  #   skim = Sourcerer::SourceSkim.skim_file('docs/ref.adoc', attributes: { 'env' => 'prod' })
+  module SourceSkim
+    NULL_LOGGER = Logger.new(IO::NULL)
+    LOAD_OPTS   = { safe: :safe, sourcemap: true, logger: NULL_LOGGER }.freeze
+
+    # Skim the AsciiDoc file at +file_path+.
+    #
+    # @param file_path [String] path to the .adoc source file
+    # @param forms [Array<Symbol>] section shape(s) to emit: +:tree+, +:flat+, or both
+    # @param categories [Array<Symbol>, nil] element categories to include;
+    #   nil uses {DEFAULT_CATEGORIES} (everything except +attributes_builtin+)
+    # @param attributes [Hash{String => String}] arbitrary Asciidoctor attribute
+    #   overrides applied at parse time, e.g. <tt>'env' => 'test'</tt>.
+    #   Useful for toggling conditionals or injecting values that affect which
+    #   blocks are visible to the parser.
+    # @return [Hash] JSON-ready skim
+    def self.skim_file file_path, forms: [:tree], categories: nil, attributes: {}
+      opts = LOAD_OPTS.merge(attributes: attributes)
+      doc  = Asciidoctor.load_file(file_path, **opts)
+      skim_doc(doc, forms: forms, categories: categories)
+    end
+
+    # Skim AsciiDoc source from a +content+ string.
+    #
+    # @param content [String] raw AsciiDoc markup
+    # @param forms [Array<Symbol>] section shape(s) to emit
+    # @param categories [Array<Symbol>, nil] element categories to include
+    # @param attributes [Hash{String => String}] arbitrary Asciidoctor attribute
+    #   overrides applied at parse time
+    # @return [Hash] JSON-ready skim
+    def self.skim_string content, forms: [:tree], categories: nil, attributes: {}
+      opts = LOAD_OPTS.merge(attributes: attributes)
+      doc  = Asciidoctor.load(content, **opts)
+      skim_doc(doc, forms: forms, categories: categories)
+    end
+
+    # Skim an already-parsed Asciidoctor +document+.
+    #
+    # This entry point is useful when the document has been loaded through
+    # other means, such as from an Asciidoctor extension callback.
+    #
+    # @param doc [Asciidoctor::Document] parsed document object
+    # @param forms [Array<Symbol>] section shape(s) to emit
+    # @param categories [Array<Symbol>, nil] element categories to include
+    # @return [Hash] JSON-ready skim
+    def self.skim_doc doc, forms: [:tree], categories: nil
+      config = Config.new(forms: forms, categories: categories)
+      Skimmer.new.process(doc, config: config)
+    end
+  end
+end