polytypo 1.12.0 → 1.13.0

data/lib/polytypo/engine/origin.rb

@@ -54,6 +54,18 @@ module Polytypo
         out
       end
 
+      # Where a change ends in the input (analyze.md section 2). A non-empty edit ends one past
+      # the input offset of the last code point it replaced that has an origin. Scanning forward
+      # from edit.end instead, as origin_at does for a start, would carry the end across a skipped
+      # element or a protected run that follows -- the boundary marker has no origin -- and the
+      # reported range would contain text the rules never saw.
+      def self.end_of(origin, edit, input_length)
+        (edit.end - 1).downto(edit.start) do |i|
+          return origin[i] + 1 unless origin[i] == NO_ORIGIN
+        end
+        origin_at(origin, edit.end, input_length)
+      end
+
       # One rule's edits, in the coordinates that rule saw, rendered as Changes in input
       # coordinates. +before+ is the text this rule replaced and +after+ what it replaced it with
       # (analyze.md section 2), so on a text two rules have both touched, +before+ is what the
@@ -64,7 +76,7 @@ module Polytypo
           Polytypo::Change.new(
             rule_id: rule_id,
             start: origin_at(origin, edit.start, input_length),
-            end: origin_at(origin, edit.end, input_length),
+            end: end_of(origin, edit, input_length),
             before: Codepoints.from_codepoints(cp[edit.start...edit.end]),
             after: Codepoints.from_codepoints(edit.replacement)
           )

data/lib/polytypo/engine/protect.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "codepoints"
+require_relative "origin"
+require_relative "pipeline"
+require_relative "sentinels"
+require_relative "../errors"
+require_relative "../modes/spans"
+require_relative "../modes/runner"
+
+module Polytypo
+  module Engine
+    # modes.md 3.9: the caller's own skipped regions, "text" mode only (spec 1.13.0). Each maximal
+    # protected run becomes ONE boundary marker in the code-point array -- at the very start or
+    # end too, which is the one difference from the html/markdown/yaml span concatenation -- the
+    # rules run once over that array, and the protected code points are put back at their markers.
+    module Protect
+      LINE_TERMINATORS = [0x0A, 0x0D, 0x0B, 0x0C, 0x85, 0x2028, 0x2029].freeze
+
+      def self.invalid(message)
+        raise Polytypo::Error.new(Polytypo::CODE_INVALID_OPTION, message)
+      end
+      private_class_method :invalid
+
+      # modes.md 3.9.2: validates +value+ against the input's length in code points and returns
+      # the protected runs as [start, end] pairs with touching ranges merged. nil and [] are the
+      # same call.
+      def self.resolve(value, length)
+        return [] if value.nil?
+
+        invalid("\"protect\" must be an Array of [start, end] pairs. Received #{value.inspect}.") unless value.is_a?(Array)
+
+        holes = []
+        previous_end = 0
+        value.each do |range|
+          unless range.is_a?(Array) && range.length == 2 && range.all?(Integer)
+            invalid("\"protect\" entries must be [start, end] pairs of Integers; received #{range.inspect}.")
+          end
+          s, e = range
+          invalid("protect range [#{s}, #{e}) is empty or negative.") if s.negative? || e <= s
+          invalid("protect range [#{s}, #{e}) overlaps or precedes the previous one.") if s < previous_end
+          invalid("protect range [#{s}, #{e}) runs past the end of the input (#{length}).") if e > length
+
+          if !holes.empty? && holes.last[1] == s
+            holes.last[1] = e
+          else
+            holes << [s, e]
+          end
+          previous_end = e
+        end
+        holes
+      end
+
+      # modes.md 3.9.2: outside "text" mode a non-empty +protect+ is refused rather than ignored.
+      def self.refuse_outside_text(value, mode)
+        return if value.nil? || (value.is_a?(Array) && value.empty?)
+
+        invalid("\"protect\" is only available when mode is \"text\", not #{mode.inspect}.")
+      end
+
+      # [S0] m [S1] m ... [Sk], a marker for every run, edges included, and the origin map.
+      def self.layout(cp, holes)
+        array = []
+        origin = []
+        cursor = 0
+        holes.each do |s, e|
+          (cursor...s).each do |i|
+            array << cp[i]
+            origin << i
+          end
+          line = (s...e).any? { |i| LINE_TERMINATORS.include?(cp[i]) }
+          array << (line ? Engine::LINE_MARKER : Engine::MARKER)
+          origin << Origin::NO_ORIGIN
+          cursor = e
+        end
+        (cursor...cp.length).each do |i|
+          array << cp[i]
+          origin << i
+        end
+        [array, origin]
+      end
+      private_class_method :layout
+
+      # Returns [text, moved] where +moved+ is the protected runs located in the output, the
+      # ranges a second call has to be given. Public API returns the text alone.
+      def self.run_mapped(cp, holes, plan, locale_data, ctx)
+        array, = layout(cp, holes)
+        out = Modes::Runner.run_rules_over_spans(array, plan, locale_data, ctx)
+        result = []
+        moved = []
+        index = 0
+        out.each do |value|
+          unless Engine.marker?(value)
+            result << value
+            next
+          end
+
+          s, e = holes[index]
+          moved << [result.length, result.length + (e - s)]
+          result.concat(cp[s...e])
+          index += 1
+        end
+        if index != holes.length
+          raise Polytypo::Error.new(Polytypo::CODE_RULE_CONTRACT, "boundary markers did not survive the pipeline")
+        end
+
+        [Codepoints.from_codepoints(result), moved]
+      end
+
+      def self.run(cp, holes, plan, locale_data, ctx)
+        run_mapped(cp, holes, plan, locale_data, ctx).first
+      end
+
+      def self.analyze(cp, holes, plan, locale_data, ctx)
+        array, origin = layout(cp, holes)
+        Pipeline.run_rules_recording(array, plan, locale_data, ctx, origin, cp.length) do |current, edits|
+          Modes::Spans.filter_boundary_edits(current, edits, Modes::Spans.span_ranges_of(current))
+        end
+      end
+    end
+  end
+end

data/lib/polytypo/modes/runner.rb

@@ -25,7 +25,6 @@ module Polytypo
         end
         current
       end
-      private_class_method :run_rules_over_spans
 
       # The output is the input with a set of disjoint substring replacements applied and nothing
       # else (modes.md 4). A span whose content the rules did not change contributes no

data/lib/polytypo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polytypo
-  VERSION = "1.12.0"
+  VERSION = "1.13.0"
 end

data/lib/polytypo.rb

@@ -5,6 +5,7 @@ require_relative "polytypo/errors"
 require_relative "polytypo/engine/codepoints"
 require_relative "polytypo/engine/origin"
 require_relative "polytypo/engine/pipeline"
+require_relative "polytypo/engine/protect"
 require_relative "polytypo/engine/rules" # side effect: registers all 9 rules
 require_relative "polytypo/modes/spans"
 require_relative "polytypo/modes/runner"
@@ -24,7 +25,9 @@ module Polytypo
   # "mdx" is not implemented by this runtime). +keys+ is required iff mode is "yaml" and names
   # the mapping keys whose scalar values are prose (modes.md 3.8.2) -- it has no default, because
   # nothing in YAML's syntax separates "description:" from "run:". +rules+ is an opt-out Hash
-  # keyed by rule id.
+  # keyed by rule id. +protect+ is "text" mode only: an Array of [start, end] pairs of
+  # code-point offsets (half-open, ascending, non-overlapping) into +input+ that are never edited
+  # and that the rules read as opaque content (modes.md 3.9). nil and [] are the same call.
   #
   # Pure: no I/O, no environment, no clock, no globals, no class-level mutable state --
   # thread-safe and reentrant, callable concurrently from any number of Threads with no external
@@ -33,7 +36,7 @@ module Polytypo
   # Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily
   # requires "commonmarker" from within Modes::Markdown, never at load time of this file.
   def self.transform(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
-                     frontmatter_keys: nil)
+                     frontmatter_keys: nil, protect: nil)
     resolved_mode = resolve_mode(mode)
     narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
 
@@ -42,19 +45,19 @@ module Polytypo
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      transform_text(input, locale, rules, narrow_target)
+      transform_text(input, locale, rules, narrow_target, protect)
     when "html"
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      transform_html(input, locale, rules, narrow_target)
+      transform_html(input, locale, rules, narrow_target, protect)
     when "yaml"
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      transform_yaml(input, locale, keys, rules, narrow_target)
+      transform_yaml(input, locale, keys, rules, narrow_target, protect)
     else # "markdown"
-      transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
+      transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: frontmatter_keys, protect: protect)
     end
   end
 
@@ -71,7 +74,7 @@ module Polytypo
   #
   # Pure and thread-safe on the same terms as .transform.
   def self.analyze(input, locale:, mode: "text", dialect: nil, keys: nil, rules: nil, narrow_nbsp: nil,
-                   frontmatter_keys: nil)
+                   frontmatter_keys: nil, protect: nil)
     resolved_mode = resolve_mode(mode)
     narrow_target = Engine.resolve_narrow_target(narrow_nbsp)
 
@@ -80,19 +83,19 @@ module Polytypo
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      analyze_text(input, locale, rules, narrow_target)
+      analyze_text(input, locale, rules, narrow_target, protect)
     when "html"
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      analyze_html(input, locale, rules, narrow_target)
+      analyze_html(input, locale, rules, narrow_target, protect)
     when "yaml"
       if dialect
         raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
       end
-      analyze_yaml(input, locale, keys, rules, narrow_target)
+      analyze_yaml(input, locale, keys, rules, narrow_target, protect)
     else # "markdown"
-      analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys)
+      analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: frontmatter_keys, protect: protect)
     end
   end
 
@@ -109,18 +112,22 @@ module Polytypo
   end
   private_class_method :resolve_mode
 
-  def self.transform_text(input, locale, rules, narrow_target)
+  def self.transform_text(input, locale, rules, narrow_target, protect)
     _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     cp = Engine::Codepoints.to_codepoints(input)
+    holes = Engine::Protect.resolve(protect, cp.length)
     ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
                                   narrow_target: narrow_target)
+    return Engine::Protect.run(cp, holes, plan, locale_data, ctx) unless holes.empty?
+
     result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx)
     Engine::Codepoints.from_codepoints(result)
   end
   private_class_method :transform_text
 
-  def self.transform_html(input, locale, rules, narrow_target)
+  def self.transform_html(input, locale, rules, narrow_target, protect)
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
+    Engine::Protect.refuse_outside_text(protect, "html")
     spans = Modes::Html.html_spans(input)
     ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
                                   narrow_target: narrow_target)
@@ -134,9 +141,10 @@ module Polytypo
   # counterpart: with no declared grammar to violate, a file that is not YAML yields few spans or
   # none and comes back byte for byte (modes.md 3.8.3). The only raise this mode adds is +keys+,
   # which is about the call and not the input.
-  def self.transform_yaml(input, locale, keys, rules, narrow_target)
+  def self.transform_yaml(input, locale, keys, rules, narrow_target, protect)
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     resolved_keys = Engine.resolve_yaml_keys(keys)
+    Engine::Protect.refuse_outside_text(protect, "yaml")
     spans = Modes::Yaml.yaml_spans(input, resolved_keys)
     ctx = Engine::RuleContext.new(mode: "yaml", dialect: nil, locale: resolved_locale,
                                   narrow_target: narrow_target)
@@ -145,7 +153,7 @@ module Polytypo
   end
   private_class_method :transform_yaml
 
-  def self.transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys = nil)
+  def self.transform_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: nil, protect: nil)
     # Validation order is public, tested behaviour, identical across every runtime: rules (an
     # unknown rule id), then locale (an unknown locale), then dialect, then frontmatter_keys, then
     # parsing (modes.md 3.7.4).
@@ -153,6 +161,7 @@ module Polytypo
     require_relative "polytypo/modes/markdown"
     Modes::Markdown.resolve_dialect(dialect)
     resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
+    Engine::Protect.refuse_outside_text(protect, "markdown")
     ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
                                   narrow_target: narrow_target)
     cp = Engine::Codepoints.to_codepoints(input)
@@ -171,17 +180,21 @@ module Polytypo
   private_class_method :markdown_units
   private_class_method :transform_markdown
 
-  def self.analyze_text(input, locale, rules, narrow_target)
+  def self.analyze_text(input, locale, rules, narrow_target, protect)
     _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     cp = Engine::Codepoints.to_codepoints(input)
+    holes = Engine::Protect.resolve(protect, cp.length)
     ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
                                   narrow_target: narrow_target)
+    return Engine::Protect.analyze(cp, holes, plan, locale_data, ctx) unless holes.empty?
+
     Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length)
   end
   private_class_method :analyze_text
 
-  def self.analyze_html(input, locale, rules, narrow_target)
+  def self.analyze_html(input, locale, rules, narrow_target, protect)
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
+    Engine::Protect.refuse_outside_text(protect, "html")
     spans = Modes::Html.html_spans(input)
     ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
                                   narrow_target: narrow_target)
@@ -191,9 +204,10 @@ module Polytypo
 
   # analyze.md section 1, "yaml" mode: offsets are into the document, not into a span
   # (analyze.md section 6).
-  def self.analyze_yaml(input, locale, keys, rules, narrow_target)
+  def self.analyze_yaml(input, locale, keys, rules, narrow_target, protect)
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     resolved_keys = Engine.resolve_yaml_keys(keys)
+    Engine::Protect.refuse_outside_text(protect, "yaml")
     spans = Modes::Yaml.yaml_spans(input, resolved_keys)
     ctx = Engine::RuleContext.new(mode: "yaml", dialect: nil, locale: resolved_locale,
                                   narrow_target: narrow_target)
@@ -201,13 +215,14 @@ module Polytypo
   end
   private_class_method :analyze_yaml
 
-  def self.analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys = nil)
+  def self.analyze_markdown(input, locale, dialect, rules, narrow_target, frontmatter_keys: nil, protect: nil)
     # Validation order is public, tested behaviour and is shared with .transform: rules, then
     # locale, then dialect, then frontmatter_keys, then parsing (analyze.md section 4, A1).
     resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
     require_relative "polytypo/modes/markdown"
     Modes::Markdown.resolve_dialect(dialect)
     resolved_keys = Engine.resolve_frontmatter_keys(frontmatter_keys)
+    Engine::Protect.refuse_outside_text(protect, "markdown")
     ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
                                   narrow_target: narrow_target)
     Modes::Runner.analyze_over_units(Engine::Codepoints.to_codepoints(input),

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polytypo
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.13.0
 platform: ruby
 authors:
 - Iurii Rogulia
@@ -113,6 +113,7 @@ files:
 - lib/polytypo/engine/locale.rb
 - lib/polytypo/engine/origin.rb
 - lib/polytypo/engine/pipeline.rb
+- lib/polytypo/engine/protect.rb
 - lib/polytypo/engine/registry.rb
 - lib/polytypo/engine/rules.rb
 - lib/polytypo/engine/rules/apostrophe.rb