textprov 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 12591d9466d6f8f852a3aca23ff985f8183cf6013d95bb2a5f56282641d97a76
+  data.tar.gz: 11e17c28beb87e5a9e7ea32ffc6ba21dd80af8ad93a2c334545a2d9e080d824e
+SHA512:
+  metadata.gz: 07f475db040b23d302de0bcea2b0ecbd02aa8ba7271bf1a9f0613fb4ebaebdcf58c2328cb96dc16b0d16c8ab8be7eb1ed53b1b28d018202640af4a546d2d634a
+  data.tar.gz: b51d9352b3f35ace72bb733afbe51c4321a79febde03dd7fc1c74f6257df1253aed97b3359bfbde5ea774cc47fa94f47338c0c0f988d969c10df1f1fc2ee229b

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 TextProv contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# textprov (Ruby)
+
+The TextProv reference implementation in Ruby: it puts provenance marks in text
+and reads them back out. Standard library only.
+
+```ruby
+require "textprov"
+
+Textprov.mark("foo", state: "ai")             # "f\u{E0101}o\u{E0101}o\u{E0101}"
+Textprov.mark_added("abc", "abXc")            # mark only what an edit added
+Textprov.convert(marked, "vs", "pua")         # same states, other encoding
+Textprov.strip_marks(marked)                  # back to the original text
+
+Textprov.runs("f\u{E0101}oo")
+# => [["ai", "f\u{E0101}"], [nil, "oo"]]
+
+Textprov.to_html("f\u{E0101}oo")
+# => '<span class="prov prov-ai" data-prov="ai">f\u{E0101}</span>oo'
+
+Textprov.inspect_text(marked)                 # a 'key: value' report of the states
+```
+
+`runs` and `to_html` take `strip:`, `merge_whitespace:` (also `mergeWhitespace:`
+as an alias per SPEC's cross-language rule), and (for `to_html`)
+`class_prefix:`. `mark` and `mark_added` take `state:` and `mode:` (`"vs"` or
+`"pua"`). Every function takes `mapping: Textprov::Mapping.load(path)` to use a
+registry other than the copy vendored in this package.
+
+The Ruby method is `Textprov.inspect_text`, not `.inspect`, so it does not
+shadow `Kernel#inspect`.
+
+## Install
+
+```sh
+gem install textprov
+```
+
+Ruby 3.2+ is required. No runtime dependencies.
+
+## CLI
+
+```sh
+textprov -o marked.txt mark --ai draft.txt
+textprov mark-added old.txt new.txt
+textprov convert --from vs --to pua marked.txt
+textprov strip marked.txt
+textprov render marked.txt
+textprov inspect marked.txt
+```
+
+A file argument may be `-` for stdin; output goes to stdout unless `-o` is
+given. Text is read and written as UTF-8 verbatim, so CRLF survives.
+
+## Contract and mapping versions
+
+- Contract version 1 (see [SPEC.md](../SPEC.md)).
+- Mapping version 1 (see [mapping.json](../mapping.json)). `lib/textprov/mapping.json`
+  is a vendored copy and the test suite fails if it drifts from the canonical
+  file.
+
+## Tests
+
+```sh
+bundle install
+bundle exec rake test
+```
+
+The suite runs every decoder, producer, and convert case in
+[fixtures.json](../fixtures.json), tests the producer properties from
+[SPEC.md](../SPEC.md) directly rather than only through recorded outputs, and
+unit-tests `cluster_end`, `to_html`, `strip_marks`, and the CLI.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

data/exe/textprov

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+# ruby/exe/textprov
+#
+# frozen_string_literal: true
+
+require "textprov/cli"
+
+exit Textprov::CLI.main(ARGV)

data/lib/textprov/cli.rb

@@ -0,0 +1,179 @@
+# ruby/lib/textprov/cli.rb
+#
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "../textprov"
+
+module Textprov
+  module CLI
+    PROG = "textprov"
+
+    module_function
+
+    def read_input(path)
+      if path == "-"
+        $stdin.binmode
+        $stdin.read.force_encoding("UTF-8")
+      else
+        File.binread(path).force_encoding("UTF-8")
+      end
+    end
+
+    def write_output(text, path)
+      bytes = text.dup.force_encoding("UTF-8").b
+      if path
+        File.binwrite(path, bytes)
+      else
+        $stdout.binmode
+        $stdout.write(bytes)
+        $stdout.flush
+      end
+    end
+
+    def version_string
+      "#{PROG} #{Textprov::VERSION} (contract #{Textprov::CONTRACT_VERSION}, mapping #{Textprov.default_mapping.version})"
+    end
+
+    def usage
+      <<~USAGE
+        Usage: #{PROG} [-o FILE] COMMAND [OPTIONS] ARGS...
+
+        Commands:
+          mark [--human|--ai|--mixed] [--mode vs|pua] FILE
+          mark-added [--human|--ai|--mixed] [--mode vs|pua] OLD NEW
+          convert --from vs|pua --to vs|pua FILE
+          strip FILE
+          render [--strip] [--no-merge-whitespace] [--class-prefix P] FILE
+          inspect FILE
+
+        FILE may be '-' for stdin. Text is read and written as UTF-8 verbatim.
+        --version prints the version and exits.
+      USAGE
+    end
+
+    def parse_state(argv, defaults: { state: "ai", mode: "vs" })
+      opts = defaults.dup
+      rest = []
+      i = 0
+      while i < argv.length
+        case argv[i]
+        when "--human", "--ai", "--mixed"
+          opts[:state] = argv[i].sub(/^--/, "")
+        when "--mode"
+          opts[:mode] = argv[i + 1]
+          raise ArgumentError, "invalid --mode #{opts[:mode].inspect}" unless %w[vs
+                                                                                 pua].include?(opts[:mode])
+
+          i += 1
+        else
+          rest << argv[i]
+        end
+        i += 1
+      end
+      [opts, rest]
+    end
+
+    def main(argv)
+      argv = argv.dup
+      # Top-level --version / --help handled before subcommand dispatch.
+      if argv.include?("--version")
+        $stdout.puts version_string
+        return 0
+      end
+      if argv.include?("--help") || argv.include?("-h") || argv.empty?
+        $stdout.puts usage
+        return argv.empty? ? 1 : 0
+      end
+
+      # -o/--output may appear before or after the command.
+      output = nil
+      filtered = []
+      i = 0
+      while i < argv.length
+        case argv[i]
+        when "-o", "--output"
+          output = argv[i + 1]
+          i += 1
+        else
+          filtered << argv[i]
+        end
+        i += 1
+      end
+      argv = filtered
+
+      command = argv.shift
+      out =
+        case command
+        when "mark"
+          opts, rest = parse_state(argv)
+          raise ArgumentError, "mark: FILE required" if rest.empty?
+
+          Textprov.mark(read_input(rest[0]), state: opts[:state], mode: opts[:mode])
+        when "mark-added"
+          opts, rest = parse_state(argv)
+          raise ArgumentError, "mark-added: OLD and NEW required" if rest.length < 2
+
+          Textprov.mark_added(read_input(rest[0]), read_input(rest[1]),
+                              state: opts[:state], mode: opts[:mode])
+        when "convert"
+          from_mode = nil
+          to_mode = nil
+          rest = []
+          i = 0
+          while i < argv.length
+            case argv[i]
+            when "--from" then from_mode = argv[i + 1]
+                               i += 1
+            when "--to" then to_mode = argv[i + 1]
+                             i += 1
+            else rest << argv[i]
+            end
+            i += 1
+          end
+          raise ArgumentError, "convert: --from and --to required" unless from_mode && to_mode
+          raise ArgumentError, "convert: FILE required" if rest.empty?
+
+          Textprov.convert(read_input(rest[0]), from_mode, to_mode)
+        when "strip"
+          raise ArgumentError, "strip: FILE required" if argv.empty?
+
+          Textprov.strip_marks(read_input(argv[0]))
+        when "render"
+          strip = false
+          merge_ws = true
+          class_prefix = "prov"
+          rest = []
+          i = 0
+          while i < argv.length
+            case argv[i]
+            when "--strip" then strip = true
+            when "--no-merge-whitespace" then merge_ws = false
+            when "--class-prefix" then class_prefix = argv[i + 1]
+                                       i += 1
+            else rest << argv[i]
+            end
+            i += 1
+          end
+          raise ArgumentError, "render: FILE required" if rest.empty?
+
+          Textprov.to_html(read_input(rest[0]),
+                           strip: strip, merge_whitespace: merge_ws, class_prefix: class_prefix)
+        when "inspect"
+          raise ArgumentError, "inspect: FILE required" if argv.empty?
+
+          Textprov.inspect_text(read_input(argv[0]))
+        else
+          warn "unknown command: #{command}"
+          warn usage
+          return 2
+        end
+
+      write_output(out, output)
+      0
+    rescue ArgumentError => e
+      warn "#{PROG}: #{e.message}"
+      2
+    end
+  end
+end

data/lib/textprov/core.rb

@@ -0,0 +1,212 @@
+# ruby/lib/textprov/core.rb
+#
+# frozen_string_literal: true
+
+require "cgi"
+require "json"
+
+module Textprov
+  CONTRACT_VERSION = 1
+  GENERATED_STATES = %w[human ai mixed].freeze
+  PROPOSED_STATES  = %w[edited unknown].freeze
+
+  MAPPING_PATH = File.expand_path("mapping.json", __dir__)
+
+  ZWJ  = 0x200D
+  VS15 = 0xFE0E
+  VS16 = 0xFE0F
+  SKIN_TONES = (0x1F3FB..0x1F3FF)
+  REGIONAL   = (0x1F1E6..0x1F1FF)
+
+  # Categories Mn/Mc/Me: extracted from Unicode data. Ruby's regexp \p{Mn} etc.
+  # covers this; using a regex avoids shipping our own category table.
+  COMBINING_RE = /[\p{Mn}\p{Mc}\p{Me}]/
+
+  module_function
+
+  def is_combining(char)
+    !!(char =~ COMBINING_RE)
+  end
+
+  # A cluster is a base character plus everything that visually belongs to it:
+  # combining marks, VS15/VS16, skin-tone modifiers, ZWJ joins, and the second
+  # half of a regional-indicator pair. A provenance selector always ends the
+  # cluster.
+  def cluster_end(chars, start, sel_cps)
+    index = start + 1
+    if REGIONAL.include?(chars[start].ord) &&
+       index < chars.length &&
+       REGIONAL.include?(chars[index].ord)
+      index += 1
+    end
+    while index < chars.length
+      cp = chars[index].ord
+      break if sel_cps.include?(cp)
+
+      if cp == ZWJ
+        index += (index + 1 < chars.length ? 2 : 1)
+        next
+      end
+      if cp == VS15 || cp == VS16 || SKIN_TONES.include?(cp) || is_combining(chars[index])
+        index += 1
+        next
+      end
+      break
+    end
+    index
+  end
+
+  class Mapping
+    attr_reader :raw, :version, :selectors, :pua2base, :base2pua
+
+    def initialize(raw:, version:, selectors:, pua2base:, base2pua:)
+      @raw = raw
+      @version = version
+      @selectors = selectors
+      @pua2base = pua2base
+      @base2pua = base2pua
+    end
+
+    def self.load(path = MAPPING_PATH)
+      raw = JSON.parse(File.read(path, encoding: "UTF-8"))
+      selectors = {}
+      raw["variation_selectors"].each do |name, value|
+        selectors[name] = Integer(value.sub(/^U\+/, ""), 16)
+      end
+      pua2base = {}
+      base2pua = {}
+      raw["pua"].each do |pua_string, entry|
+        pua  = Integer(pua_string.sub(/^U\+/, ""), 16)
+        base = Integer(entry["base"].sub(/^U\+/, ""), 16)
+        state = entry["provenance"] || "ai"
+        pua2base[pua] = [base, state]
+        base2pua[base] = pua if state == "ai"
+      end
+      new(raw: raw, version: raw["version"], selectors: selectors,
+          pua2base: pua2base, base2pua: base2pua)
+    end
+  end
+
+  @default_mapping = nil
+
+  def default_mapping
+    @default_mapping ||= Mapping.load
+  end
+
+  class << self
+    attr_writer :default_mapping
+  end
+
+  # `text` is a String; we work over its per-codepoint characters. Ruby's
+  # String#chars yields one Unicode scalar per element, which is what the
+  # Python port iterates.
+  def _strip(text, selectors, pua2base)
+    sel_cps = selectors.values.to_set
+    out = String.new(encoding: "UTF-8")
+    text.each_char do |char|
+      cp = char.ord
+      next if sel_cps.include?(cp)
+
+      entry = pua2base[cp]
+      out << (entry ? entry[0].chr(Encoding::UTF_8) : char)
+    end
+    out
+  end
+
+  def _runs(text, selectors, pua2base, strip: false, merge_whitespace: true)
+    sel_cps = selectors.values.to_set
+    sel2name = selectors.each_with_object({}) { |(name, cp), h| h[cp] = name }
+    chars = text.chars
+    items = []
+    index = 0
+    while index < chars.length
+      char = chars[index]
+      cp = char.ord
+      if pua2base.key?(cp)
+        base_cp, state = pua2base[cp]
+        out = strip ? base_cp.chr(Encoding::UTF_8) : base_cp.chr(Encoding::UTF_8) + selectors[state].chr(Encoding::UTF_8)
+        index += 1
+      elsif char.match?(/\s/)
+        state = "ws"
+        out = char
+        index += 1
+      else
+        endi = cluster_end(chars, index, sel_cps)
+        cluster = chars[index...endi].join
+        state = nil
+        out = cluster
+        index = endi
+        if index < chars.length && sel_cps.include?(chars[index].ord)
+          state = sel2name[chars[index].ord]
+          out += chars[index] unless strip
+          index += 1
+        end
+      end
+      if !items.empty? && items.last[0] == state
+        items.last[1] << out
+      else
+        items << [state, out.dup]
+      end
+    end
+
+    merged = []
+    items.each_with_index do |(state, out), position|
+      following = position + 1 < items.length ? items[position + 1][0] : nil
+      if merge_whitespace && state == "ws" && !merged.empty? && merged.last[0] && merged.last[0] == following
+        merged.last[1] << out
+        next
+      end
+      state = nil if state == "ws"
+      if !merged.empty? && merged.last[0] == state
+        merged.last[1] << out
+      else
+        merged << [state, out.dup]
+      end
+    end
+    merged.map { |state, out| [state, out] }
+  end
+
+  def _to_html(text, selectors, pua2base, strip: false, merge_whitespace: true,
+               class_prefix: "prov")
+    parts = []
+    _runs(text, selectors, pua2base, strip: strip,
+                                     merge_whitespace: merge_whitespace).each do |state, out|
+      escaped = CGI.escapeHTML(out)
+      if state
+        parts << %(<span class="#{class_prefix} #{class_prefix}-#{state}" data-prov="#{state}">#{escaped}</span>)
+      else
+        parts << escaped
+      end
+    end
+    parts.join
+  end
+
+  # Public API accepts either merge_whitespace (contract spelling) or
+  # mergeWhitespace (camelCase alias per SPEC's cross-language rule).
+  def runs(text, mapping: nil, strip: false, merge_whitespace: nil, mergeWhitespace: nil)
+    mw = if merge_whitespace.nil?
+           mergeWhitespace.nil? || mergeWhitespace
+         else
+           merge_whitespace
+         end
+    m = mapping || default_mapping
+    _runs(text, m.selectors, m.pua2base, strip: strip, merge_whitespace: mw)
+  end
+
+  def to_html(text, mapping: nil, strip: false, merge_whitespace: nil, mergeWhitespace: nil,
+              class_prefix: "prov")
+    mw = if merge_whitespace.nil?
+           mergeWhitespace.nil? || mergeWhitespace
+         else
+           merge_whitespace
+         end
+    m = mapping || default_mapping
+    _to_html(text, m.selectors, m.pua2base, strip: strip, merge_whitespace: mw,
+                                            class_prefix: class_prefix)
+  end
+
+  def strip_marks(text, mapping: nil)
+    m = mapping || default_mapping
+    _strip(text, m.selectors, m.pua2base)
+  end
+end

data/lib/textprov/encode.rb

@@ -0,0 +1,171 @@
+# ruby/lib/textprov/encode.rb
+#
+# frozen_string_literal: true
+
+require_relative "core"
+
+module Textprov
+  module_function
+
+  def _mark(text, state, mode, selectors, pua2base, base2pua)
+    sel_cps = selectors.values.to_set
+    selector = selectors[state].chr(Encoding::UTF_8)
+    chars = text.chars
+    out = String.new(encoding: "UTF-8")
+    index = 0
+    while index < chars.length
+      char = chars[index]
+      cp = char.ord
+      if sel_cps.include?(cp) || pua2base.key?(cp) || char.match?(/\s/)
+        out << char
+        index += 1
+        next
+      end
+      start = index
+      index = cluster_end(chars, index, sel_cps)
+      cluster = chars[start...index]
+      if index < chars.length && sel_cps.include?(chars[index].ord)
+        cluster.each { |c| out << c }
+        next
+      end
+      if mode == "pua" && state == "ai" && cluster.length == 1 && base2pua.key?(cp)
+        out << base2pua[cp].chr(Encoding::UTF_8)
+      else
+        cluster.each { |c| out << c }
+        out << selector
+      end
+    end
+    out
+  end
+
+  # `old_text` and `new_text` are code-point-wise compared via chars, matching
+  # the Python impl (Python indexes by code point).
+  def _mark_added(old_text, new_text, state, mode, selectors, pua2base, base2pua)
+    return _mark(new_text, state, mode, selectors, pua2base, base2pua) if old_text.empty?
+
+    old_chars = old_text.chars
+    new_chars = new_text.chars
+    pre = 0
+    while pre < [old_chars.length, new_chars.length].min && old_chars[pre] == new_chars[pre]
+      pre += 1
+    end
+    suf = 0
+    while suf < [old_chars.length,
+                 new_chars.length].min - pre && old_chars[-1 - suf] == new_chars[-1 - suf]
+      suf += 1
+    end
+    endi = new_chars.length - suf
+    middle = new_chars[pre...endi].join
+    middle_marked = _mark(middle, state, mode, selectors, pua2base, base2pua)
+    new_chars[0...pre].join + middle_marked + new_chars[endi...new_chars.length].join
+  end
+
+  def _convert(text, from_mode, to_mode, selectors, pua2base, base2pua)
+    return text if from_mode == to_mode
+
+    vs_ai = selectors["ai"]
+    chars = text.chars
+    out = String.new(encoding: "UTF-8")
+    if from_mode == "vs"
+      index = 0
+      while index < chars.length
+        char = chars[index]
+        index += 1
+        if index < chars.length && chars[index].ord == vs_ai && base2pua.key?(char.ord)
+          out << base2pua[char.ord].chr(Encoding::UTF_8)
+          index += 1
+        else
+          out << char
+        end
+      end
+      return out
+    end
+    chars.each do |char|
+      entry = pua2base[char.ord]
+      if entry && entry[1] == "ai"
+        out << entry[0].chr(Encoding::UTF_8)
+        out << vs_ai.chr(Encoding::UTF_8)
+      else
+        out << char
+      end
+    end
+    out
+  end
+
+  def _inspect(text, selectors, pua2base)
+    sel2name = selectors.each_with_object({}) { |(name, cp), h| h[cp] = name }
+    counts = { "unmarked" => 0, "human" => 0, "ai_vs" => 0, "ai_pua" => 0,
+               "unknown" => 0, "edited" => 0, "mixed" => 0, "whitespace" => 0 }
+    unrecognised_selectors = []
+    unrecognised_pua = []
+    sel_cp_set = selectors.values.to_set
+
+    chars = text.chars
+    index = 0
+    while index < chars.length
+      char = chars[index]
+      cp = char.ord
+      index += 1
+      if sel2name.key?(cp)
+        unrecognised_selectors << cp
+        next
+      end
+      if cp.between?(0xE0100, 0xE01EF)
+        unrecognised_selectors << cp
+        next
+      end
+      if pua2base.key?(cp)
+        counts["ai_pua"] += 1
+        next
+      end
+      if cp.between?(0x100000, 0x10FFFD)
+        unrecognised_pua << cp
+        next
+      end
+      if char.match?(/\s/)
+        counts["whitespace"] += 1
+        next
+      end
+      index = cluster_end(chars, index - 1, sel_cp_set)
+      if index < chars.length && sel_cp_set.include?(chars[index].ord)
+        name = sel2name[chars[index].ord]
+        index += 1
+        key = name == "ai" ? "ai_vs" : name
+        counts[key] += 1
+      else
+        counts["unmarked"] += 1
+      end
+    end
+
+    lines = ["characters: #{chars.length}"]
+    %w[unmarked human ai_vs ai_pua unknown edited mixed whitespace].each do |key|
+      lines << "#{key}: #{counts[key]}"
+    end
+    sels = unrecognised_selectors.uniq.sort.map { |cp| format("U+%04X", cp) }.join(" ")
+    puas = unrecognised_pua.uniq.sort.map { |cp| format("U+%04X", cp) }.join(" ")
+    lines << "unrecognised_selectors: #{sels.empty? ? "-" : sels}"
+    lines << "unrecognised_pua: #{puas.empty? ? "-" : puas}"
+    "#{lines.join("\n")}\n"
+  end
+
+  def mark(text, state: "ai", mode: "vs", mapping: nil)
+    m = mapping || default_mapping
+    _mark(text, state, mode, m.selectors, m.pua2base, m.base2pua)
+  end
+
+  def mark_added(old_text, new_text, state: "ai", mode: "vs", mapping: nil)
+    m = mapping || default_mapping
+    _mark_added(old_text, new_text, state, mode, m.selectors, m.pua2base, m.base2pua)
+  end
+
+  def convert(text, from_mode, to_mode, mapping: nil)
+    m = mapping || default_mapping
+    _convert(text, from_mode, to_mode, m.selectors, m.pua2base, m.base2pua)
+  end
+
+  # Named inspect_text to avoid shadowing Kernel#inspect / Object#inspect.
+  def inspect_text(text, mapping: nil)
+    m = mapping || default_mapping
+    _inspect(text, m.selectors, m.pua2base)
+  end
+end