isodoc-i18n 1.4.0 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 134575d665c75368d3640ef87c9c63a3f6fdc2bf668e8d8df233a3cb139fce6b
-  data.tar.gz: ed1fc0e49c62f27b3199ba8357b2f1454309485efffbcb93ed30d51f01c36fe7
+  metadata.gz: 02e65f54740c83c1d9698623c18711219680d6541f88e2cd0aaa62d3595c103d
+  data.tar.gz: f57b0e7bf29a2d02576dda04b3d7b34ded97a42ec382ebdbc9caefb12a1aa17b
 SHA512:
-  metadata.gz: f6e5ff44068372afc9ad75e1d6bff6483114a3424071c136a86f17745513ff5f31794c4555c9484eff556c50f638cb13c3ba712c4ae3e3b4d762685d1e888c2d
-  data.tar.gz: 9c5131cb3dbb8800304530629dfe5b47a3d7b15695ff0762011998c5aa3d3273b7730dde057dd2f2f15e3f997c9466704be80d7ef9cd47a5055e93afc0a5ba32
+  metadata.gz: b0a869f0df1ef445e5f18d336d0d3f18968e7055444e81535d7da4219a7c24f4e370ea45e0033d1ed2766d8fc5ea2c577f74ea07c232fc6217ff236aacc65e8e
+  data.tar.gz: 801f9ccb9c81c372c77834ad3667b88491f0aa1429c588e1a8c1854fc5ca8f1a817607de4bd0c44ed1d02f2a3984d5c336714692cff31c860d6c14a658502e9b

data/isodoc-i18n.gemspec

@@ -22,12 +22,13 @@ Gem::Specification.new do |spec|
   end
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
+  spec.add_dependency "base64"
   spec.add_dependency "htmlentities", "~> 4.3.4"
   spec.add_dependency "liquid", "~> 5"
   spec.add_dependency "metanorma-utils", ">= 1.7.0"
   spec.add_dependency "twitter_cldr"
-  spec.add_dependency "base64"
 
+  spec.add_development_dependency "canon", "= 0.1.3"
   spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "guard", "~> 2.14"
@@ -35,10 +36,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
   spec.add_development_dependency "rubocop", "~> 1"
-spec.add_development_dependency "rubocop-performance"
+  spec.add_development_dependency "rubocop-performance"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "canon"
   # spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/isodoc/i18n/version.rb

@@ -1,5 +1,5 @@
 module IsoDoc
   class I18n
-    VERSION = "1.4.0".freeze
+    VERSION = "1.4.2".freeze
   end
 end

data/lib/isodoc/i18n-yaml.rb

@@ -7,11 +7,80 @@ module IsoDoc
 
     def load_yaml(lang, script, i18nyaml = nil, i18nhash = nil)
       ret = load_yaml1(lang, script)
-      i18nyaml and
-        return normalise_hash(ret.deep_merge(YAML.load_file(i18nyaml)))
-      i18nhash and return normalise_hash(ret.deep_merge(i18nhash))
+      if i18nyaml
+        Array(i18nyaml).compact.each do |y|
+          ret = ret.deep_merge(YAML.load_file(y))
+        end
+        return postprocess(ret)
+      end
+      i18nhash and return postprocess(ret.deep_merge(i18nhash))
+      postprocess(ret)
+    end
+
+    def postprocess(labels)
+      self_reference_resolve(normalise_hash(labels))
+    end
+
+    def self_reference_resolve(labels)
+      resolve_references(labels, labels)
+    end
 
-      normalise_hash(ret)
+    def resolve_references(obj, labels)
+      case obj
+      when Hash
+        obj.transform_values { |v| resolve_references(v, labels) }
+      when Array
+        obj.map { |item| resolve_references(item, labels) }
+      when String
+        resolve_string_references(obj, labels)
+      else
+        obj
+      end
+    end
+
+    def resolve_string_references(str, labels)
+      # Match patterns like #{self["key"]["subkey"]} or #{self.key.subkey}
+      # Allow spaces around the self expression
+      str.gsub(/\#\{\s*self([^}]+?)\s*\}/) do |match|
+        path_expr = Regexp.last_match(1)
+        resolve_path(path_expr, labels, match)
+      end
+    end
+
+    def resolve_path(path_expr, labels, original_expr)
+      segments = parse_path(path_expr)
+      current = labels
+
+      segments.each do |segment|
+        case current
+        when Hash
+          current.key?(segment) or
+            raise "Self-reference error: Path '#{original_expr}' not found - key '#{segment}' does not exist"
+          current = current[segment]
+        when Array
+          index = segment.to_i
+          segment =~ /^\d+$/ && index >= 0 && index < current.length or
+            raise "Self-reference error: Path '#{original_expr}' not found - invalid array index '#{segment}'"
+          current = current[index]
+        else
+          raise "Self-reference error: Path '#{original_expr}' not found - cannot navigate through non-collection type"
+        end
+      end
+
+      current.to_s
+    end
+
+    def parse_path(path_expr)
+      # Split by dots and brackets while preserving the content
+      parts = path_expr.sub(/^\./, "").scan(/\.?([\w-]+)|\[([^\]]+)\]/)
+      parts.each_with_object([]) do |(dot_part, bracket_part), segments|
+        if dot_part
+          segments << dot_part
+        elsif bracket_part
+          segment = bracket_part.strip.gsub(/^["']|["']$/, "")
+          segments << segment
+        end
+      end
     end
 
     def normalise_hash(ret)
@@ -30,8 +99,8 @@ module IsoDoc
     def load_yaml1(lang, script)
       case lang
       when "zh"
-        if script == "Hans" then load_yaml2("zh-Hans")
-        else load_yaml2("en")
+        if script then load_yaml2("zh-#{script}")
+        else load_yaml2("zh-Hans")
         end
       else
         load_yaml2(lang)
@@ -54,5 +123,9 @@ module IsoDoc
     def set(key, val)
       @labels[key] = val
     end
+
+    def merge(new_labels)
+      @labels = @labels.deep_merge(new_labels)
+    end
   end
 end

data/lib/isodoc/i18n.rb

@@ -27,7 +27,6 @@ module IsoDoc
     CJK_SCRIPTS = %w(Hans Hant Jpan Kore).freeze
 
     def liquid_init
-      ::IsoDoc::I18n::Liquid.set(self)
       ::Liquid::Environment.default.register_filter(::IsoDoc::I18n::Liquid)
     end
 
@@ -50,6 +49,7 @@ module IsoDoc
 
     # populate with variables, Liquid, inflections, ordinals/spellout
     def populate(keys, vars = {})
+      ::IsoDoc::I18n::Liquid.set(self)
       ::Liquid::Template.parse(@labels.dig(*Array(keys)))
         .render(vars.merge("labels" => @labels))
     end
@@ -68,7 +68,9 @@ module IsoDoc
     end
 
     def enum_comma
-      CJK_SCRIPTS.include?(@script) and return "<enum-comma>、</enum-comma>"
+      c = @labels.dig("punct", "enum-comma")
+      c && CJK_SCRIPTS.include?(@script) and
+        return "<enum-comma>#{c}</enum-comma>"
       "<enum-comma>,</enum-comma> "
     end
 

data/lib/isodoc/l10n.rb

@@ -1,55 +1,23 @@
 require "metanorma-utils"
+require_relative "l10n_cjk"
 
 module IsoDoc
   class I18n
-    # Use comprehensive CJK definition from metanorma-utils
-    # This includes Han, Katakana, Hiragana, Hangul, Bopomofo and all CJK extensions
-    ZH_CHAR = "(#{Metanorma::Utils::CJK})".freeze
-    LATIN_PUNCT = /[:,.()\[\];?!-]/.freeze
-    
-    # Condition for converting punctuation to double width:
-    # 1. (Strict condition) CJK before, CJK after, modulo ignorable characters:
-    # 1a. CJK character, or start of string. Latin spaces optional.
-    ZH1_PUNCT = /(#{ZH_CHAR}|^)(\s*)$/xo.freeze 
-    # 1b. Latin spaces optional, Latin punct which will also convert to CJK, 
-    # CJK character, or end of string.
-    ZH2_PUNCT = /^\s*#{LATIN_PUNCT}*(#{ZH_CHAR}|$)/xo.freeze
-    # 2. CJK before, space after:
-    # 2a.  CJK char, followed by optional Latin punct which will also convert to CJK
-    ZH1_NO_SPACE = /#{ZH_CHAR}#{LATIN_PUNCT}*$/xo.freeze
-    # 2b. optional Latin punct which wil also convert to CJK, then space
-    OPT_PUNCT_SPACE = /^($|#{LATIN_PUNCT}*\s)/xo.freeze
-
-    # Contexts for converting en-dashes to full-width
-    # Before: CJK or start of string, optional digits
-    ZH1_DASH = /(#{ZH_CHAR}|^)(\d*)$/xo.freeze
-    # After: optional digits, CJK or end of string
-    ZH2_DASH = /^\d*(#{ZH_CHAR}|$)/xo.freeze
-    
-    # Pre-defined punctuation mappings for efficiency
-    ZH_PUNCT_MAP = [
-      [":：", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      [",，", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      [".。", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      [")）", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      ["]］", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      [";；", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      ["?？", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      ["!！", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      ["(（", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
-      ["[［", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]]
-    ].freeze
-
     def self.l10n(text, lang = @lang, script = @script, options = {})
       l10n(text, lang, script, options)
     end
 
     # function localising spaces and punctuation
     # options[:prev] and options[:foll] are optional context strings
+    # options[:proportional_mixed_cjk] allows contextual full-width vs
+    # half-width punctuation
     def l10n(text, lang = @lang, script = @script, options = {})
       locale = options[:locale] || @locale
-      %w(zh ja ko).include?(lang) and text = l10n_zh(text, script, options[:prev], options[:foll])
-      lang == "fr" && text = l10n_fr(text, locale || "FR", options[:prev], options[:foll])
+      %w(zh ja ko).include?(lang) and
+        text = l10n_zh(text, script, options)
+      lang == "fr" and
+        text = l10n_fr(text, locale || "FR", options)
+      text&.gsub!(/<esc>|<\/esc>/, "") # Strip esc tags
       bidiwrap(text, lang, script)
     end
 
@@ -71,30 +39,36 @@ module IsoDoc
          .default_script(@lang))]
     end
 
-    # CJK
-    def l10n_zh(text, script, prev, foll)
-      script ||= "Hans"
-      t, text_cache, xml = l10n_prep(text, prev, foll)
-      t.each_with_index do |n, i|
-        # Adjust index if prev context prepended
-        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
-        text = cleanup_entities(n.text, is_xml: false)
-        n.replace(l10_zh1(text, prev_ctx, foll_ctx, script))
-      end
-      to_xml(xml).gsub(/<b>|<\/b>|<\?[^>]+>/, "")
+    def l10n_prep(text, options)
+      xml = Nokogiri::XML::DocumentFragment.parse(text)
+      t = xml.xpath(".//text()").reject { |node| node.text.empty? }
+      text_cache = build_text_cache(t, options[:prev], options[:foll])
+
+      # Find all text nodes within <esc> tags in one XPath query
+      # This is O(n) instead of O(n*m) where m is tree depth
+      esc_indices = build_esc_indices(xml, t)
+
+      [t, text_cache, xml, options[:prev], options[:foll], esc_indices]
     end
 
-    def l10n_prep(text, prev, foll)
-            xml = Nokogiri::XML::DocumentFragment.parse(text)
-      t = xml.xpath(".//text()")
-      text_cache = build_text_cache(t, prev, foll)
-      [t, text_cache, xml]
+    # Build set of indices for text nodes within <esc> tags
+    # Handles both namespaced and non-namespaced <esc> elements
+    def build_esc_indices(xml, text_nodes)
+      # Try both non-namespaced and namespace-agnostic queries
+      esc_text_nodes = Set.new(xml.xpath(".//esc//text()") +
+                               xml.xpath(".//*[local-name()='esc']//text()"))
+      Set.new.tap do |indices|
+        text_nodes.each_with_index do |node, i|
+          indices.add(i) if esc_text_nodes.include?(node)
+        end
+      end
     end
 
     # Cache text content once per method call to avoid repeated .text calls
     # Build text cache with optional prepended/appended context
+    # Also, reduce multiple spaces to single, to avoid miscrecognition of space
     def build_text_cache(text_nodes, prev_context = nil, foll_context = nil)
-      text_cache = text_nodes.map(&:text)
+      text_cache = text_nodes.map(&:text).map { |x| x.gsub(/\s+/, " ") }
       text_cache.unshift(prev_context) if prev_context
       text_cache.push(foll_context) if foll_context
       text_cache
@@ -117,9 +91,11 @@ module IsoDoc
       [prev, foll]
     end
 
-    def l10n_fr(text, locale, prev, foll)
-      t, text_cache, xml = l10n_prep(text, prev, foll)
+    def l10n_fr(text, locale, options)
+      t, text_cache, xml, prev, _foll, esc_indices = l10n_prep(text, options)
       t.each_with_index do |n, i|
+        next if esc_indices.include?(i) # Skip escaped nodes
+
         prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
         text = cleanup_entities(n.text, is_xml: false)
         n.replace(l10n_fr1(text, prev_ctx, foll_ctx, locale))
@@ -127,35 +103,15 @@ module IsoDoc
       to_xml(xml)
     end
 
-    # note: we can't differentiate comma from enumeration comma 、
-    # def l10_zh1(text, _script)
-    def l10_zh1(text, prev, foll, _script)
-      r = l10n_zh_punct(text, prev, foll)
-      r = l10n_zh_remove_space(r, prev, foll)
-      l10n_zh_dash(r, prev, foll)
-    end
-
-    def l10n_zh_punct(text, prev, foll)
-      # Use pre-defined mapping for better performance
-      ZH_PUNCT_MAP.each do |mapping|
-        punct_pair, regexes = mapping
-        text = l10n_gsub(text, prev, foll, [punct_pair[0], punct_pair[1]], regexes)
-      end
-      text
-    end
-
-    def l10n_zh_dash(text, prev, foll)
-      l10n_gsub(text, prev, foll, %w(– ～), [[ZH1_DASH, ZH2_DASH]])
-    end
-
     # text: string we are scanning for instances of delim[0] to replace
     # prev: string preceding text, as additional token of context
     # foll: string following text, as additional token of context
     # delim: delim[0] is the symbol we want to replace, delim[1] its replacement
     # regexes: a list of regex pairs: the context before the found token,
     # and the context after the found token, under which replacing it
-    # with delim[1] is permitted
+    # with delim[1] is permitted. If regex is nil, always allow the replacement
     def l10n_gsub(text, prev, foll, delim, regexes)
+      delim[1] or return text
       context = l10n_gsub_context(text, prev, foll, delim) or return text
       (1...(context.size - 1)).each do |i|
         l10_context_valid?(context, i, delim, regexes) and
@@ -170,11 +126,12 @@ module IsoDoc
       d = delim[0].is_a?(Regexp) ? delim[0] : Regexp.quote(delim[0])
       context = text.split(/(#{d})/) # delim to replace
       context.size == 1 and return
-      [prev, context, foll].flatten
+      [prev, context.reject(&:empty?), foll].flatten
     end
 
     def l10_context_valid?(context, idx, delim, regex)
       l10n_context_found_delimiter?(context[idx], delim) or return false
+      regex.nil? and return true
       regex.detect do |r|
         r[0].match?(context[0...idx].join) && # preceding context
           r[1].match?(context[(idx + 1)..-1].join) # foll context
@@ -189,13 +146,6 @@ module IsoDoc
       end
     end
 
-    def l10n_zh_remove_space(text, prev, foll)
-      text = l10n_gsub(text, prev, foll, [" ", ""],
-                       [[/(#{ZH_CHAR}|\d)$/o, /^#{ZH_CHAR}/o]])
-      l10n_gsub(text, prev, foll, [" ", ""],
-                [[/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o]])
-    end
-
     def l10n_fr1(text, prev, foll, locale)
       text = l10n_gsub(text, prev, foll, [/[»›;?!]/, "\u202f\\0"],
                        [[/\p{Alnum}$/, /^(\s|$)/]])
@@ -206,30 +156,6 @@ module IsoDoc
                 [[/\p{Alnum}$/, /^(\s|$)/]])
     end
 
-    def self.cjk_extend(text)
-      cjk_extend(text)
-    end
-
-    def cjk_extend(title)
-      @c.decode(title).chars.map.with_index do |n, i|
-        if i.zero? || !interleave_space_cjk?(title[i - 1] + title[i])
-          n
-        else "\u3000#{n}"
-        end
-      end.join
-    end
-
-    def interleave_space_cjk?(text)
-      text.size == 2 or return
-      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026",
-       "\u22ef\u22ef"].include?(text) ||
-        /\d\d|\p{Latin}\p{Latin}|[[:space:]]/.match?(text) ||
-        /^[\u2018\u201c(\u3014\[{\u3008\u300a\u300c\u300e\u3010\u2985\u3018\u3016\u00ab\u301d]/.match?(text) ||
-        /[\u2019\u201d)\u3015\]}\u3009\u300b\u300d\u300f\u3011\u2986\u3019\u3017\u00bb\u301f]$/.match?(text) ||
-        /[\u3002.\u3001,\u30fb:;\u2010\u301c\u30a0\u2013!?\u203c\u2047\u2048\u2049]/.match?(text) and return false
-      true
-    end
-
     def to_xml(node)
       node&.to_xml(encoding: "UTF-8", indent: 0,
                    save_with: Nokogiri::XML::Node::SaveOptions::AS_XML)

data/lib/isodoc/l10n_cjk.rb

@@ -0,0 +1,165 @@
+module IsoDoc
+  class I18n
+    # Use comprehensive CJK definition from metanorma-utils
+    # This includes Han, Katakana, Hiragana, Hangul, Bopomofo
+    # and all CJK extensions
+    ZH_CHAR = "(#{Metanorma::Utils::CJK})".freeze
+    LATIN_PUNCT = /[:,.()\[\];?!-]/.freeze
+    # CJK character which is not punctuation
+    ZH_NON_PUNCT = "(#{
+      [
+        Metanorma::Utils.singleton_class::HAN,
+        Metanorma::Utils.singleton_class::HAN_IDC,
+        Metanorma::Utils.singleton_class::KANBUN,
+        Metanorma::Utils.singleton_class::CJK_COMPAT_IDEOGRAPHS,
+        Metanorma::Utils.singleton_class::HAN_COMPAT_IDEOGRAPHS,
+        Metanorma::Utils.singleton_class::HANGUL,
+        Metanorma::Utils.singleton_class::HIRAGANA,
+        Metanorma::Utils.singleton_class::KATAKANA,
+        Metanorma::Utils.singleton_class::BOPOMOFO,
+      ].join("|")})".freeze
+
+    # Condition for converting punctuation to double width,
+    # in case of options[:proportional_mixed_cjk]
+    # 1. (Strict condition) CJK before, CJK after, modulo ignorable characters:
+    # 1a. CJK character, or start of string. Latin spaces optional.
+    ZH1_PUNCT = /(#{ZH_CHAR}|^)(\s*)$/xo.freeze
+    # 1b. Latin spaces optional, Latin punct which will also convert to CJK,
+    # CJK character, or end of string.
+    ZH2_PUNCT = /^\s*#{LATIN_PUNCT}*(#{ZH_CHAR}|$)/xo.freeze
+    # 2. CJK before, space after:
+    # 2a. CJK char, followed by optional Latin punct which will also convert to CJK
+    ZH1_NO_SPACE = /#{ZH_CHAR}#{LATIN_PUNCT}*$/xo.freeze
+    # 2b. optional Latin punct which wil also convert to CJK, then space
+    OPT_PUNCT_SPACE = /^($|#{LATIN_PUNCT}*\s)/xo.freeze
+
+    # Chinese numerals (common + formal/financial forms)
+    # Explicit characters needed because Chinese numeral ideographs
+    # are not tagged with Unicode Number property
+    # Using alternation instead of character class to properly include \p{N}
+    ZH_NUMERALS = "(?:[零一二三四五六七八九十百千万亿壹贰叁肆伍陆柒捌玖拾佰仟萬億兆]|\\p{N})".freeze
+
+    # Contexts for converting en-dashes to full-width
+    # Before: CJK or start of string, no digits
+    ZH1_DASH = /(#{ZH_CHAR}|^)(?<!=#{ZH_NUMERALS})$/xo.freeze
+    # After: no optional digits, CJK or end of string
+    ZH2_DASH = /^(?!#{ZH_NUMERALS})(#{ZH_CHAR}|$)/xo.freeze
+    # Before: CJK or start of string, optional digits
+    ZH1_NUM_DASH = /#{ZH_NUMERALS}$/xo.freeze
+    # After: optional digits, CJK or end of string
+    ZH2_NUM_DASH = /^#{ZH_NUMERALS}/xo.freeze
+
+    ZH_PUNCT_CONTEXTS =
+      [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE],
+       [/(\s|^)$/, /^#{ZH_CHAR}/o]].freeze
+
+    # map of YAML punct keys to auto-text Latin equivalents
+    ZH_PUNCT_AUTOTEXT = {
+      colon: ":",
+      comma: ",",
+      # "enum-comma": ",", # enum-comma is ambiguous with comma
+      semicolon: ";",
+      period: ".",
+      "close-paren": ")",
+      "open-paren": "(",
+      "close-bracket": "]",
+      "open-bracket": "[",
+      "question-mark": "?",
+      "exclamation-mark": "!",
+      "em-dash": "—",
+      "open-quote": "“",
+      "close-quote": "”",
+      "open-nested-quote": "’",
+      "close-nested-quote": "’",
+      ellipse: "…",
+    }.freeze
+
+    # Pre-defined punctuation mappings for efficiency
+    def init_zh_punct_map
+      ZH_PUNCT_AUTOTEXT.each_with_object([]) do |(k, v), m|
+        @labels.dig("punct", k.to_s) or next
+        m << [v, @labels["punct"][k.to_s], ZH_PUNCT_CONTEXTS]
+      end
+    end
+
+    def l10n_zh(text, script, options)
+      script ||= "Hans"
+      t, text_cache, xml, prev, _foll, esc_indices = l10n_prep(text, options)
+      t.each_with_index do |n, i|
+        next if esc_indices.include?(i) # Skip escaped nodes
+
+        # Adjust index if prev context prepended
+        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
+        text = cleanup_entities(n.text, is_xml: false)
+        n.replace(l10_zh1(text, prev_ctx, foll_ctx, script, options))
+      end
+      to_xml(xml) #.gsub(/<\/?em>|<\/?strong>|<\/?i>|<\/?b>/, "")
+    end
+
+    # note: we can't differentiate comma from enumeration comma 、
+    # def l10_zh1(text, _script)
+    def l10_zh1(text, prev, foll, _script, options)
+      r = l10n_zh_punct(text, prev, foll, options)
+      r = l10n_zh_remove_space(r, prev, foll)
+      l10n_zh_dash(r, prev, foll)
+    end
+
+    def l10n_zh_punct(text, prev, foll, options)
+      # Use pre-defined mapping for better performance
+      @zh_punct_map ||= init_zh_punct_map
+      @zh_punct_map.each do |mapping|
+        punct_from, punct_to, regexes = mapping
+        options[:proportional_mixed_cjk] or regexes = nil
+        text = l10n_gsub(text, prev, foll, [punct_from, punct_to],
+                         regexes)
+      end
+      text
+    end
+
+    def l10n_zh_dash(text, prev, foll)
+      text = l10n_gsub(text, prev, foll, ["–", @labels.dig("punct", "en-dash")],
+                       [[ZH1_DASH, ZH2_DASH]])
+      l10n_gsub(text, prev, foll, ["–", @labels.dig("punct", "number-en-dash")],
+                [[ZH1_NUM_DASH, ZH2_NUM_DASH]])
+    end
+
+    def l10n_zh_remove_space(text, prev, foll)
+      text = l10n_gsub(text, prev, foll, [/\s+/, ""],
+                       [[/(#{ZH_CHAR})$/o, /^#{ZH_CHAR}/o]])
+      if sep = @labels.dig("punct", "cjk-latin-separator")
+        # Skip over punctuation to find Latin letters/numbers
+        text = l10n_gsub(text, prev, foll, [/\s+/, sep],
+                         [[/#{ZH_CHAR}$/o, /^\p{P}*[\p{Latin}\p{N}]/o]])
+        l10n_gsub(text, prev, foll, [/\s+/, sep],
+                  [[/[\p{Latin}\p{N}]\p{P}*$/o, /^#{ZH_NON_PUNCT}/o]])
+      else
+        l10n_gsub(text, prev, foll, [/\s+/, ""],
+                  [[/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o]])
+      end
+    end
+
+    def self.cjk_extend(text)
+      cjk_extend(text)
+    end
+
+    def cjk_extend(title)
+      @c.decode(title).chars.map.with_index do |n, i|
+        if i.zero? || !interleave_space_cjk?(title[i - 1] + title[i])
+          n
+        else "\u3000#{n}"
+        end
+      end.join
+    end
+
+    def interleave_space_cjk?(text)
+      text.size == 2 or return
+      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026",
+       "\u22ef\u22ef"].include?(text) ||
+        /\d\d|\p{Latin}\p{Latin}|[[:space:]]/.match?(text) ||
+        /^[\u2018\u201c(\u3014\[{\u3008\u300a\u300c\u300e\u3010\u2985\u3018\u3016\u00ab\u301d]/.match?(text) ||
+        /[\u2019\u201d)\u3015\]}\u3009\u300b\u300d\u300f\u3011\u2986\u3019\u3017\u00bb\u301f]$/.match?(text) ||
+        /[\u3002.\u3001,\u30fb:;\u2010\u301c\u30a0\u2013!?\u203c\u2047\u2048\u2049]/.match?(text) and return false
+      true
+    end
+  end
+end

data/lib/isodoc-yaml/i18n-en.yaml

@@ -2,3 +2,24 @@ text: text
 at: at
 binary_and: "%1 <conn>and</conn> %2"
 multiple_and: "%1<conn>, and</conn> %2"
+punct:
+  colon: ":"
+  comma: ","
+  enum_comma: ","
+  semicolon: ";"
+  period: "."
+  close_paren: ")"
+  open_paren: "("
+  close_bracket: "]"
+  open_bracket: "["
+  question_mark: "?"
+  exclamation_mark: "!"
+  em_dash: "—"
+  en_dash: "–"
+  number_en_dash: "–"
+  open_quote: "“"
+  close_quote: "”"
+  open_nested_quote: "’"
+  close_nested_quote: "’"
+  ellipse: …
+

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: isodoc-i18n
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-09-29 00:00:00.000000000 Z
+date: 2025-11-03 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -67,19 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: base64
+  name: canon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: 0.1.3
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.3
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -234,20 +248,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: canon
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: 'Internationalisation for Metanorma rendering
 
   '
@@ -271,6 +271,7 @@ files:
 - lib/isodoc/i18n.rb
 - lib/isodoc/i18n/version.rb
 - lib/isodoc/l10n.rb
+- lib/isodoc/l10n_cjk.rb
 - lib/isodoc/liquid/liquid.rb
 homepage: https://github.com/metanorma/isodoc-i18n
 licenses: