polytypo 1.6.3 → 1.7.0

data/lib/polytypo/data/schema/fixtures.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://polytypo.org/schema/fixtures.schema.json",
   "title": "polytypo conformance fixtures",
-  "description": "Executable form of the rule semantics. Every case is also an idempotency case: a runner must additionally assert transform(out) == out, passing the case's own options (mode, dialect, keys, rules, narrowNbsp) on that second call exactly as on the first — a case is a fixed point under its own options, not under the defaults.",
+  "description": "Executable form of the rule semantics. Every case is also an idempotency case: a runner must additionally assert transform(out) == out, passing the case's own options (mode, dialect, keys, frontmatterKeys, rules, narrowNbsp) on that second call exactly as on the first — a case is a fixed point under its own options, not under the defaults.",
   "type": "object",
   "additionalProperties": false,
   "required": ["spec", "locale", "cases"],
@@ -29,6 +29,11 @@
             "if": { "properties": { "mode": { "const": "yaml" } }, "required": ["mode"] },
             "then": { "required": ["keys"] },
             "else": { "not": { "required": ["keys"] } }
+          },
+          {
+            "if": { "properties": { "mode": { "const": "markdown" } }, "required": ["mode"] },
+            "then": true,
+            "else": { "not": { "required": ["frontmatterKeys"] } }
           }
         ],
         "oneOf": [{ "required": ["out"] }, { "required": ["throws"] }],
@@ -62,6 +67,12 @@
             "items": { "type": "string" },
             "description": "Required when mode is \"yaml\" — spec/rules/modes.md §3.8.2 gives the option no default, so a fixture must name the processable keys exactly as a caller would. An empty array is legal and processes nothing. Meaningless for the other modes."
           },
+          "frontmatterKeys": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Optional, spec 1.7.0, and only meaningful when mode is \"markdown\" — spec/rules/modes.md §3.7.4's opt-out from the frontmatter skip. Absent means the block is skipped whole, exactly as before 1.7.0; an empty array is legal and yields no spans. A runner passes it through to transform() as it passes dialect.",
+            "$comment": "The block is its own text unit (§3.7.4), so this option cannot change a byte of the body — every case released before 1.7.0 keeps its recorded output whatever it is set to."
+          },
           "in": { "type": "string" },
           "out": { "type": "string" },
           "throws": {

data/lib/polytypo/engine/pipeline.rb

@@ -59,6 +59,35 @@ module Polytypo
       value.to_set
     end
 
+    # modes.md 3.7.4: "markdown" mode's `frontmatter_keys` option (spec 1.7.0).
+    #
+    # Optional, unlike `keys` -- nil means the frontmatter block is skipped whole, which is every
+    # pre-1.7.0 document's behaviour -- and an empty Array is legal, exactly as it is for `keys`.
+    # Checked after dialect and before the parse, which is what decides that a document failing to
+    # parse in its dialect still reports the option error rather than CODE_MALFORMED_INPUT.
+    def self.resolve_frontmatter_keys(value)
+      return nil if value.nil?
+
+      unless value.is_a?(Array)
+        raise Polytypo::Error.new(
+          Polytypo::CODE_INVALID_OPTION,
+          '"frontmatter_keys" must be an Array of Strings when given (modes.md 3.7.4). ' \
+          "Received #{value.inspect}.",
+        )
+      end
+
+      value.each do |key|
+        next if key.is_a?(String)
+
+        raise Polytypo::Error.new(
+          Polytypo::CODE_INVALID_OPTION,
+          "\"frontmatter_keys\" must contain only Strings; received #{key.inspect}.",
+        )
+      end
+
+      value.to_set
+    end
+
     # Resolves the locale, builds the rule plan, and runs each enabled rule in
     # spec/rules/order.json order over a code-point array, applying its edits before the next
     # rule sees it. No module-level mutable state beyond the immutable, load-once-on-first-use

data/lib/polytypo/modes/markdown.rb

@@ -2,6 +2,7 @@
 
 require_relative "spans"
 require_relative "html"
+require_relative "yaml"
 require_relative "parse_error"
 require_relative "../errors"
 
@@ -66,6 +67,56 @@ module Polytypo
       end
       private_class_method :detect_frontmatter_delimiter
 
+      # modes.md 3.7.4, spec 1.7.0. The frontmatter block's own spans, which form a SECOND TEXT
+      # UNIT: the pipeline runs over them separately from the body's, so an unbalanced mark in a
+      # metadata field can never pair with one in the first paragraph, and the option cannot
+      # change a byte outside the block.
+      #
+      # Spans come from the scan of modes.md 3.8 -- frontmatter IS YAML, and implementing that
+      # grammar twice is how two implementations of one spec drift -- with +keys+ as step 8's key
+      # predicate. The block is the construct markdown_spans skips (3.7.3, the :frontmatter node),
+      # so the option only ever adds spans where the skip removed them: no source position belongs
+      # to both units. A TOML block yields nothing, with the option or without it -- its quoting is
+      # a second grammar this scan does not claim (modes.md 7.13).
+      #
+      # The content range comes from the node rather than from a second scan of the text: comrak
+      # reports the block from its opening delimiter line through its closing one, so the content
+      # is every line between them, and both delimiters and every line terminator stay outside
+      # every span.
+      def self.frontmatter_spans(source, keys)
+        require "commonmarker"
+        return [] if keys.empty?
+        return [] unless detect_frontmatter_delimiter(source) == "---"
+
+        cp = Polytypo::Engine::Codepoints.to_codepoints(source)
+        chars = cp.map { |c| [c].pack("U") }
+        line_starts = build_line_starts(chars)
+
+        spans = []
+        ParseError.wrap do
+          doc = Commonmarker.parse(
+            source,
+            options: { parse: { sourcepos_chars: true },
+                       extension: { front_matter_delimiter: "---" } },
+          )
+          doc.each do |node|
+            next unless node.type == :frontmatter
+
+            pos = node.source_position
+            content_start = line_starts[pos[:start_line]]
+            content_end = line_starts[pos[:end_line] - 1]
+            break if content_start.nil? || content_end.nil? || content_end <= content_start
+
+            content = chars[content_start...content_end].join
+            Yaml.yaml_spans(content, keys).each do |span|
+              spans << Spans::Span.new(span.start + content_start, span.end + content_start)
+            end
+            break
+          end
+        end
+        spans
+      end
+
       # Locates the processable spans of a Markdown document. dialect must already be validated
       # via resolve_dialect (== "commonmark"); this function does not re-check it.
       def self.markdown_spans(source)

data/lib/polytypo/modes/runner.rb

@@ -34,17 +34,46 @@ module Polytypo
       # source_cp is the whole input already converted to code points; spans address it by
       # code-point index.
       def self.run_over_spans(source_cp, spans, plan, locale_data, ctx)
+        emit(source_cp, replacements_of_unit(source_cp, spans, plan, locale_data, ctx))
+      end
+
+      # modes.md 3.1 and 3.5 step 3 (spec 1.7.0). A document has one text unit, except in
+      # "markdown" with frontmatter_keys, where the frontmatter block's spans form a unit of their
+      # own. The pipeline runs once per unit and the two edit sets are disjoint, because no span of
+      # one unit lies inside the other -- which is what the body's walk skipping the block
+      # guarantees. Only step 5 is shared: the source is emitted once, in document order.
+      def self.run_over_units(source_cp, units, plan, locale_data, ctx)
+        replacements = units.flat_map do |spans|
+          replacements_of_unit(source_cp, spans, plan, locale_data, ctx)
+        end
+        emit(source_cp, replacements.sort_by { |span, _piece| span.start })
+      end
+
+      # analyze_over_spans per text unit (modes.md 3.1), reported in document order.
+      def self.analyze_over_units(source_cp, units, plan, locale_data, ctx)
+        units.flat_map { |spans| analyze_over_spans(source_cp, spans, plan, locale_data, ctx) }
+             .sort_by(&:start)
+      end
+
+      # One text unit: the marker-separated concatenation, the pipeline, and the pieces it
+      # produced, paired with the spans they replace.
+      def self.replacements_of_unit(source_cp, spans, plan, locale_data, ctx)
         normalized = Spans.normalize_spans(spans)
-        return source_cp.pack("U*") if normalized.empty?
+        return [] if normalized.empty?
 
         concatenated = Spans.concatenate_spans(source_cp, normalized)
         transformed = run_rules_over_spans(concatenated, plan, locale_data, ctx)
         pieces = Spans.split_on_marker(transformed, normalized.length)
+        normalized.each_with_index.map { |span, i| [span, pieces[i]] }
+      end
+      private_class_method :replacements_of_unit
 
+      # modes.md 4: the source with disjoint replacements applied at recorded offsets, and nothing
+      # else changed.
+      def self.emit(source_cp, replacements)
         out = []
         cursor = 0
-        normalized.each_with_index do |span, i|
-          piece = pieces[i]
+        replacements.each do |span, piece|
           original = source_cp[span.start...span.end]
           out.concat(source_cp[cursor...span.start])
           out.concat(piece == original ? original : piece)
@@ -53,6 +82,7 @@ module Polytypo
         out.concat(source_cp[cursor..])
         out.pack("U*")
       end
+      private_class_method :emit
 
       # run_over_spans, reporting instead of applying (analyze.md section 1). The span table
       # supplies the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md

data/lib/polytypo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polytypo
-  VERSION = "1.6.3"
+  VERSION = "1.7.0"
 end

data/lib/polytypo.rb

@@ -32,7 +32,8 @@ module Polytypo
   #
   # Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily
   # requires "commonmarker" from within Modes::Markdown, never at load time of this file.
-  def self.transform(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil)
+  def self.transform(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
+                     frontmatter_keys: nil)
     resolved_mode = resolve_mode(mode)
     narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
 
@@ -53,7 +54,7 @@ module Polytypo
       end
       transform_yaml(input, locale, keys, rules, narrow_target)
     else # "markdown"
-      transform_markdown(input, locale, dialect, rules, narrow_target)
+      transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
     end
   end
 
@@ -69,7 +70,8 @@ module Polytypo
   # for the text (analyze.md sections 4 and 5).
   #
   # Pure and thread-safe on the same terms as .transform.
-  def self.analyze(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil)
+  def self.analyze(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
+                   frontmatter_keys: nil)
     resolved_mode = resolve_mode(mode)
     narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
 
@@ -90,7 +92,7 @@ module Polytypo
       end
       analyze_yaml(input, locale, keys, rules, narrow_target)
     else # "markdown"
-      analyze_markdown(input, locale, dialect, rules, narrow_target)
+      analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
     end
   end
 
@@ -143,18 +145,30 @@ module Polytypo
   end
   private_class_method :transform_yaml
 
-  def self.transform_markdown(input, locale, dialect, rules, narrow_target)
+  def self.transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys = nil)
     # Validation order is public, tested behaviour, identical across every runtime: rules (an
-    # unknown rule id), then locale (an unknown locale), then dialect/parsing.
+    # unknown rule id), then locale (an unknown locale), then dialect, then frontmatter_keys, then
+    # parsing (modes.md 3.7.4).
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     require_relative "polytypo/modes/markdown"
     Modes::Markdown.resolve_dialect(dialect)
-    spans = Modes::Markdown.markdown_spans(input)
+    resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
     ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
                                   narrow_target: narrow_target)
     cp = Engine::Codepoints.to_codepoints(input)
-    Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx)
+    Modes::Runner.run_over_units(cp, markdown_units(input, resolved_keys), plan, locale_data, ctx)
   end
+
+  # modes.md 3.7.4: the body, and -- only when the caller named frontmatter keys -- the
+  # frontmatter block as a second text unit. With frontmatter_keys nil this is exactly the single
+  # unit every document had before spec 1.7.0, which is why no released output can move.
+  def self.markdown_units(input, resolved_keys)
+    body = Modes::Markdown.markdown_spans(input)
+    return [body] if resolved_keys.nil?
+
+    [Modes::Markdown.frontmatter_spans(input, resolved_keys), body]
+  end
+  private_class_method :markdown_units
   private_class_method :transform_markdown
 
   def self.analyze_text(input, locale, rules, narrow_target)
@@ -187,16 +201,17 @@ module Polytypo
   end
   private_class_method :analyze_yaml
 
-  def self.analyze_markdown(input, locale, dialect, rules, narrow_target)
+  def self.analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys = nil)
     # Validation order is public, tested behaviour and is shared with .transform: rules, then
-    # locale, then dialect/parsing (analyze.md section 4, A1).
+    # locale, then dialect, then frontmatter_keys, then parsing (analyze.md section 4, A1).
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     require_relative "polytypo/modes/markdown"
     Modes::Markdown.resolve_dialect(dialect)
-    spans = Modes::Markdown.markdown_spans(input)
+    resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
     ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
                                   narrow_target: narrow_target)
-    Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx)
+    Modes::Runner.analyze_over_units(Engine::Codepoints.to_codepoints(input),
+                                     markdown_units(input, resolved_keys), plan, locale_data, ctx)
   end
   private_class_method :analyze_markdown
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polytypo
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.7.0
 platform: ruby
 authors:
 - Iurii Rogulia
@@ -36,6 +36,7 @@ files:
 - README.md
 - lib/polytypo.rb
 - lib/polytypo/data/.not-canonical
+- lib/polytypo/data/.not-vendored
 - lib/polytypo/data/README.md
 - lib/polytypo/data/UNICODE
 - lib/polytypo/data/VERSION