isodoc-i18n 1.3.2 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a242d0bc7609246ea502ffec35e515457e739c478b25478a01c7ff641dc4523
-  data.tar.gz: 0f89c271766d244f037a0b6c6f40431888c2b6140814ac2b6e4d2906e142b1d5
+  metadata.gz: e08e5e4a6c9b89a5f628ee426ce22f8bcdbae97e7cccd4411f4962844d29b231
+  data.tar.gz: 9d8cbc5526c1d8aabe9db00c25deeb581a3604358b50b18e6f5a9bb7faab6909
 SHA512:
-  metadata.gz: cb3f3f3a28b1b8fddd35ff74290ccf62f905ef54d51d62efd4fd8000d2d38a389eb4c78ec02aefd83ab0d1d98aaec462a0138c51257f8548c07f4a4726854c38
-  data.tar.gz: f6dcb2d4c05c630527d02888bf614a0e09396166e2f27437dedc0cc0a619149c6606fe2a8a78ce3c98948d4279ef3bfa224d9c1b18fbb1d38e74d92d280b74db
+  metadata.gz: 3880d68a1f094ab500840e74767f970af57fca01a12ae751da68e9cdc72ba00df9139516df3c74dda5059114fe723b800c057c665efdd485e44aa9a247cdde22
+  data.tar.gz: 80643511d523a3535293a2b7b82e60fe81a8910dbe9fc336cf8a4ab95a9763b1adf1375f802f5b36177a6e4570c51688b803576f7d2d39b8fc7c29e894d2f235

data/README.adoc

@@ -2,7 +2,7 @@
 
 image:https://img.shields.io/gem/v/isodoc-i18n.svg["Gem Version", link="https://rubygems.org/gems/isodoc-i18n"]
 image:https://github.com/metanorma/isodoc-i18n/workflows/rake/badge.svg["Build Status", link="https://github.com/metanorma/isodoc-i18n/actions?query=workflow%3Arake"]
-image:https://codeclimate.com/github/metanorma/isodoc-i18n/badges/gpa.svg["Code Climate", link="https://codeclimate.com/github/metanorma/isodoc-i18n"]
+// image:https://codeclimate.com/github/metanorma/isodoc-i18n/badges/gpa.svg["Code Climate", link="https://codeclimate.com/github/metanorma/isodoc-i18n"]
 image:https://img.shields.io/github/issues-pr-raw/metanorma/isodoc-i18n.svg["Pull Requests", link="https://github.com/metanorma/isodoc-i18n/pulls"]
 image:https://img.shields.io/github/commits-since/metanorma/isodoc-i18n/latest.svg["Commits since latest",link="https://github.com/metanorma/isodoc-i18n/releases"]
 

data/isodoc-i18n.gemspec

@@ -22,22 +22,23 @@ Gem::Specification.new do |spec|
   end
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
+  spec.add_dependency "base64"
   spec.add_dependency "htmlentities", "~> 4.3.4"
   spec.add_dependency "liquid", "~> 5"
   spec.add_dependency "metanorma-utils", ">= 1.7.0"
   spec.add_dependency "twitter_cldr"
-  spec.add_dependency "base64"
 
+  spec.add_development_dependency "canon"
   spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-rspec", "~> 4.7"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
-  spec.add_development_dependency "rubocop", "~> 1.5.2"
+  spec.add_development_dependency "rubocop", "~> 1"
+  spec.add_development_dependency "rubocop-performance"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "xml-c14n"
   # spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/isodoc/i18n/version.rb

@@ -1,5 +1,5 @@
 module IsoDoc
   class I18n
-    VERSION = "1.3.2".freeze
+    VERSION = "1.4.1".freeze
   end
 end

data/lib/isodoc/i18n-yaml.rb

@@ -8,10 +8,79 @@ module IsoDoc
     def load_yaml(lang, script, i18nyaml = nil, i18nhash = nil)
       ret = load_yaml1(lang, script)
       i18nyaml and
-        return normalise_hash(ret.deep_merge(YAML.load_file(i18nyaml)))
-      i18nhash and return normalise_hash(ret.deep_merge(i18nhash))
+        return postprocess(ret.deep_merge(YAML.load_file(i18nyaml)))
+      i18nhash and return postprocess(ret.deep_merge(i18nhash))
 
-      normalise_hash(ret)
+      postprocess(ret)
+    end
+
+    def postprocess(labels)
+      self_reference_resolve(normalise_hash(labels))
+    end
+
+    def self_reference_resolve(labels)
+      resolve_references(labels, labels)
+    end
+
+    def resolve_references(obj, labels)
+      case obj
+      when Hash
+        obj.transform_values { |v| resolve_references(v, labels) }
+      when Array
+        obj.map { |item| resolve_references(item, labels) }
+      when String
+        resolve_string_references(obj, labels)
+      else
+        obj
+      end
+    end
+
+    def resolve_string_references(str, labels)
+      # Match patterns like #{self["key"]["subkey"]} or #{self.key.subkey}
+      # Allow spaces around the self expression
+      str.gsub(/\#\{\s*self([^\}]+?)\s*\}/) do |match|
+        path_expr = Regexp.last_match(1)
+        resolve_path(path_expr, labels, match)
+      end
+    end
+
+    def resolve_path(path_expr, labels, original_expr)
+      segments = parse_path(path_expr)
+      current = labels
+
+      segments.each do |segment|
+        case current
+        when Hash
+          current.key?(segment) or
+            raise "Self-reference error: Path '#{original_expr}' not found - key '#{segment}' does not exist"
+          current = current[segment]
+        when Array
+          index = segment.to_i
+          segment =~ /^\d+$/ && index >= 0 && index < current.length or
+            raise "Self-reference error: Path '#{original_expr}' not found - invalid array index '#{segment}'"
+          current = current[index]
+        else
+          raise "Self-reference error: Path '#{original_expr}' not found - cannot navigate through non-collection type"
+        end
+      end
+
+      current.to_s
+    end
+
+    def parse_path(path_expr)
+      segments = []
+      path_expr = path_expr.sub(/^\./, "")
+      # Split by dots and brackets while preserving the content
+      parts = path_expr.scan(/\.?([\w-]+)|\[([^\]]+)\]/)
+      parts.each do |dot_part, bracket_part|
+        if dot_part
+          segments << dot_part
+        elsif bracket_part
+          segment = bracket_part.strip.gsub(/^["']|["']$/, "")
+          segments << segment
+        end
+      end
+      segments
     end
 
     def normalise_hash(ret)
@@ -30,8 +99,8 @@ module IsoDoc
     def load_yaml1(lang, script)
       case lang
       when "zh"
-        if script == "Hans" then load_yaml2("zh-Hans")
-        else load_yaml2("en")
+        if script then load_yaml2("zh-#{script}")
+        else load_yaml2("zh-Hans")
         end
       else
         load_yaml2(lang)

data/lib/isodoc/i18n.rb

@@ -24,6 +24,8 @@ module IsoDoc
       self
     end
 
+    CJK_SCRIPTS = %w(Hans Hant Jpan Kore).freeze
+
     def liquid_init
       ::IsoDoc::I18n::Liquid.set(self)
       ::Liquid::Environment.default.register_filter(::IsoDoc::I18n::Liquid)
@@ -66,7 +68,9 @@ module IsoDoc
     end
 
     def enum_comma
-      %w(Hans Hant).include?(@script) and return "<enum-comma>、</enum-comma>"
+      c = @labels.dig("punct", "enum-comma")
+      c && CJK_SCRIPTS.include?(@script) and
+        return "<enum-comma>#{c}</enum-comma>"
       "<enum-comma>,</enum-comma> "
     end
 

data/lib/isodoc/l10n.rb

@@ -1,13 +1,23 @@
+require "metanorma-utils"
+require_relative "l10n_cjk"
+
 module IsoDoc
   class I18n
-    def self.l10n(text, lang = @lang, script = @script, locale = @locale)
-      l10n(text, lang, script, locale)
-    end
-
-    # function localising spaces and punctuation.
-    def l10n(text, lang = @lang, script = @script, locale = @locale)
-      %w(zh ja ko).include?(lang) and text = l10n_zh(text, script)
-      lang == "fr" && text = l10n_fr(text, locale || "FR")
+    def self.l10n(text, lang = @lang, script = @script, options = {})
+      l10n(text, lang, script, options)
+    end
+
+    # function localising spaces and punctuation
+    # options[:prev] and options[:foll] are optional context strings
+    # options[:proportional_mixed_cjk] allows contextual full-width vs
+    # half-width punctuation
+    def l10n(text, lang = @lang, script = @script, options = {})
+      locale = options[:locale] || @locale
+      %w(zh ja ko).include?(lang) and
+        text = l10n_zh(text, script, options)
+      lang == "fr" and
+        text = l10n_fr(text, locale || "FR", options)
+      text&.gsub!(/<esc>|<\/esc>/, "") # Strip esc tags
       bidiwrap(text, lang, script)
     end
 
@@ -29,145 +39,110 @@ module IsoDoc
          .default_script(@lang))]
     end
 
-    # CJK
-    def l10n_zh(text, script = "Hans")
+    def l10n_prep(text, options)
       xml = Nokogiri::XML::DocumentFragment.parse(text)
-      t = xml.xpath(".//text()")
-      t.each_with_index do |n, i|
-        prev, foll = l10n_context(t, i)
-        text = cleanup_entities(n.text, is_xml: false)
-        n.replace(l10_zh1(text, prev, foll, script))
+      t = xml.xpath(".//text()").reject { |node| node.text.empty? }
+      text_cache = build_text_cache(t, options[:prev], options[:foll])
+
+      # Identify which text nodes are within <esc> tags
+      esc_indices = Set.new
+      t.each_with_index do |node, i|
+        esc_indices.add(i) if node.ancestors("esc").any?
       end
-      to_xml(xml).gsub(/<b>/, "").gsub("</b>", "")
-        .gsub(/<\?[^>]+>/, "")
+
+      [t, text_cache, xml, options[:prev], options[:foll], esc_indices]
+    end
+
+    # Cache text content once per method call to avoid repeated .text calls
+    # Build text cache with optional prepended/appended context
+    # Also, reduce multiple spaces to single, to avoid miscrecognition of space
+    def build_text_cache(text_nodes, prev_context = nil, foll_context = nil)
+      text_cache = text_nodes.map(&:text).map { |x| x.gsub(/\s+/, " ") }
+      text_cache.unshift(prev_context) if prev_context
+      text_cache.push(foll_context) if foll_context
+      text_cache
     end
 
     # previous, following context of current text node:
     # do not use just the immediately adjoining text tokens for context
     # deal with spaces and empty text by just concatenating entire context
+    # Optimized to avoid O(n²) complexity by using pre-cached text content
+    def l10n_context_cached(text_cache, idx)
+      prev = text_cache[0...idx].join
+      foll = text_cache[(idx + 1)...text_cache.size].join
+      [prev, foll]
+    end
+
+    # Fallback method for backward compatibility
     def l10n_context(nodes, idx)
       prev = nodes[0...idx].map(&:text).join
       foll = nodes[(idx + 1)...(nodes.size)].map(&:text).join
       [prev, foll]
     end
 
-    def l10n_fr(text, locale)
-      xml = Nokogiri::XML::DocumentFragment.parse(text)
-      t = xml.xpath(".//text()")
+    def l10n_fr(text, locale, options)
+      t, text_cache, xml, prev, _foll, esc_indices = l10n_prep(text, options)
       t.each_with_index do |n, i|
-        prev, foll = l10n_context(t, i)
+        next if esc_indices.include?(i) # Skip escaped nodes
+
+        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
         text = cleanup_entities(n.text, is_xml: false)
-        n.replace(l10n_fr1(text, prev, foll, locale))
+        n.replace(l10n_fr1(text, prev_ctx, foll_ctx, locale))
       end
       to_xml(xml)
     end
 
-    ZH_CHAR = "(\\p{Han}|\\p{In CJK Symbols And Punctuation}|" \
-              "\\p{In Halfwidth And Fullwidth Forms})".freeze
-
-    # note: we can't differentiate comma from enumeration comma 、
-    # def l10_zh1(text, _script)
-    def l10_zh1(text, prev, foll, _script)
-      # l10n_zh_dash(l10n_zh_remove_space(l10n_zh_punct(text)))
-      r = l10n_zh_punct(text, prev, foll)
-      r = l10n_zh_remove_space(r, prev, foll)
-      l10n_zh_dash(r, prev, foll)
-    end
-
-    ZH1_PUNCT = /(#{ZH_CHAR}|^)   # CJK character, or start of string
-         (\s*)$                   # Latin spaces optional
-    /xo.freeze
-    ZH2_PUNCT = /^\s*             # followed by ignorable Latin spaces
-                [:,.()\[\];?!-]*  # Latin punct which will also convert to CJK
-                (#{ZH_CHAR}|$)    # CJK character, or end of string
-      /xo.freeze
-
-    # CJK punct if (^|CJK).($|CJK)
-    def l10n_zh_punct(text, prev, foll)
-      [":：", ",，", ".．", ")）", "]］", ";；", "?？", "!！", "(（", "[［"].each do |m|
-        text = l10n_gsub(text, prev, foll, [m[0], m[1]],
-                         [ZH1_PUNCT, ZH2_PUNCT])
-      end
-      text
-    end
-
-    ZH1_DASH = /(#{ZH_CHAR}|^)    # CJK character, or start of string
-                (\d*)             # optional digits
-    $/xo.freeze
-
-    ZH2_DASH = /^\d*              # followed by optional digits
-                (#{ZH_CHAR}|$)    # CJK character, or end of string
-      /xo.freeze
-
-    def l10n_zh_dash(text, prev, foll)
-      l10n_gsub(text, prev, foll, %w(– ～), [ZH1_DASH, ZH2_DASH])
-    end
-
-    def l10n_gsub(text, prev, foll, delim, regex)
+    # text: string we are scanning for instances of delim[0] to replace
+    # prev: string preceding text, as additional token of context
+    # foll: string following text, as additional token of context
+    # delim: delim[0] is the symbol we want to replace, delim[1] its replacement
+    # regexes: a list of regex pairs: the context before the found token,
+    # and the context after the found token, under which replacing it
+    # with delim[1] is permitted. If regex is nil, always allow the replacement
+    def l10n_gsub(text, prev, foll, delim, regexes)
+      delim[1] or return text
       context = l10n_gsub_context(text, prev, foll, delim) or return text
       (1...(context.size - 1)).each do |i|
-        l10_context_valid?(context, i, delim, regex) and
+        l10_context_valid?(context, i, delim, regexes) and
           context[i] = delim[1].gsub("\\0", context[i]) # Full-width equivalent
       end
       context[1...(context.size - 1)].join
     end
 
+    # split string being scanned, and its contextual tokens before and after,
+    # into array of tokens determining whether to replace instances of delim[0]
     def l10n_gsub_context(text, prev, foll, delim)
       d = delim[0].is_a?(Regexp) ? delim[0] : Regexp.quote(delim[0])
       context = text.split(/(#{d})/) # delim to replace
       context.size == 1 and return
-      [prev, context, foll].flatten
+      [prev, context.reject(&:empty?), foll].flatten
     end
 
     def l10_context_valid?(context, idx, delim, regex)
-      found_delim = if delim[0].is_a?(Regexp) # punct to convert
-                      delim[0].match?(context[idx])
-                    else
-                      context[idx] == delim[0]
-                    end
-      found_delim &&
-        regex[0].match?(context[0...idx].join) && # preceding context
-        regex[1].match?(context[(idx + 1)..-1].join) # foll context
+      l10n_context_found_delimiter?(context[idx], delim) or return false
+      regex.nil? and return true
+      regex.detect do |r|
+        r[0].match?(context[0...idx].join) && # preceding context
+          r[1].match?(context[(idx + 1)..-1].join) # foll context
+      end
     end
 
-    def l10n_zh_remove_space(text, prev, foll)
-      text = l10n_gsub(text, prev, foll, [" ", ""],
-                       [/(#{ZH_CHAR}|\d)$/o, /^#{ZH_CHAR}/o])
-      l10n_gsub(text, prev, foll, [" ", ""],
-                [/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o])
+    def l10n_context_found_delimiter?(token, delim)
+      if delim[0].is_a?(Regexp) # punct to convert
+        delim[0].match?(token)
+      else
+        token == delim[0]
+      end
     end
 
     def l10n_fr1(text, prev, foll, locale)
       text = l10n_gsub(text, prev, foll, [/[»›;?!]/, "\u202f\\0"],
-                       [/\p{Alnum}$/, /^(\s|$)/])
-      text = l10n_gsub(text, prev, foll, [/[«‹]/, "\\0\u202f"], [/$/, /^(?!\p{Zs})./])
+                       [[/\p{Alnum}$/, /^(\s|$)/]])
+      text = l10n_gsub(text, prev, foll, [/[«‹]/, "\\0\u202f"],
+                       [[/$/, /^(?!\p{Zs})./]])
       colonsp = locale == "CH" ? "\u202f" : "\u00a0"
       l10n_gsub(text, prev, foll, [":", "#{colonsp}\\0"],
-                [/\p{Alnum}$/, /^(\s|$)/])
-    end
-
-    def self.cjk_extend(text)
-      cjk_extend(text)
-    end
-
-    def cjk_extend(title)
-      @c.decode(title).chars.map.with_index do |n, i|
-        if i.zero? || !interleave_space_cjk?(title[i - 1] + title[i])
-          n
-        else "\u3000#{n}"
-        end
-      end.join
-    end
-
-    def interleave_space_cjk?(text)
-      text.size == 2 or return
-      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026",
-       "\u22ef\u22ef"].include?(text) ||
-        /\d\d|\p{Latin}\p{Latin}|[[:space:]]/.match?(text) ||
-        /^[\u2018\u201c(\u3014\[{\u3008\u300a\u300c\u300e\u3010\u2985\u3018\u3016\u00ab\u301d]/.match?(text) ||
-        /[\u2019\u201d)\u3015\]}\u3009\u300b\u300d\u300f\u3011\u2986\u3019\u3017\u00bb\u301f]$/.match?(text) ||
-        /[\u3002.\u3001,\u30fb:;\u2010\u301c\u30a0\u2013!?\u203c\u2047\u2048\u2049]/.match?(text) and return false
-      true
+                [[/\p{Alnum}$/, /^(\s|$)/]])
     end
 
     def to_xml(node)

data/lib/isodoc/l10n_cjk.rb

@@ -0,0 +1,165 @@
+module IsoDoc
+  class I18n
+    # Use comprehensive CJK definition from metanorma-utils
+    # This includes Han, Katakana, Hiragana, Hangul, Bopomofo
+    # and all CJK extensions
+    ZH_CHAR = "(#{Metanorma::Utils::CJK})".freeze
+    LATIN_PUNCT = /[:,.()\[\];?!-]/.freeze
+    # CJK character which is not punctuation
+    ZH_NON_PUNCT = "(#{
+      [
+        Metanorma::Utils.singleton_class::HAN,
+        Metanorma::Utils.singleton_class::HAN_IDC,
+        Metanorma::Utils.singleton_class::KANBUN,
+        Metanorma::Utils.singleton_class::CJK_COMPAT_IDEOGRAPHS,
+        Metanorma::Utils.singleton_class::HAN_COMPAT_IDEOGRAPHS,
+        Metanorma::Utils.singleton_class::HANGUL,
+        Metanorma::Utils.singleton_class::HIRAGANA,
+        Metanorma::Utils.singleton_class::KATAKANA,
+        Metanorma::Utils.singleton_class::BOPOMOFO,
+      ].join("|")})".freeze
+
+    # Condition for converting punctuation to double width,
+    # in case of options[:proportional_mixed_cjk]
+    # 1. (Strict condition) CJK before, CJK after, modulo ignorable characters:
+    # 1a. CJK character, or start of string. Latin spaces optional.
+    ZH1_PUNCT = /(#{ZH_CHAR}|^)(\s*)$/xo.freeze
+    # 1b. Latin spaces optional, Latin punct which will also convert to CJK,
+    # CJK character, or end of string.
+    ZH2_PUNCT = /^\s*#{LATIN_PUNCT}*(#{ZH_CHAR}|$)/xo.freeze
+    # 2. CJK before, space after:
+    # 2a. CJK char, followed by optional Latin punct which will also convert to CJK
+    ZH1_NO_SPACE = /#{ZH_CHAR}#{LATIN_PUNCT}*$/xo.freeze
+    # 2b. optional Latin punct which wil also convert to CJK, then space
+    OPT_PUNCT_SPACE = /^($|#{LATIN_PUNCT}*\s)/xo.freeze
+
+    # Chinese numerals (common + formal/financial forms)
+    # Explicit characters needed because Chinese numeral ideographs
+    # are not tagged with Unicode Number property
+    # Using alternation instead of character class to properly include \p{N}
+    ZH_NUMERALS = "(?:[零一二三四五六七八九十百千万亿壹贰叁肆伍陆柒捌玖拾佰仟萬億兆]|\\p{N})".freeze
+
+    # Contexts for converting en-dashes to full-width
+    # Before: CJK or start of string, no digits
+    ZH1_DASH = /(#{ZH_CHAR}|^)(?<!=#{ZH_NUMERALS})$/xo.freeze
+    # After: no optional digits, CJK or end of string
+    ZH2_DASH = /^(?!#{ZH_NUMERALS})(#{ZH_CHAR}|$)/xo.freeze
+    # Before: CJK or start of string, optional digits
+    ZH1_NUM_DASH = /#{ZH_NUMERALS}$/xo.freeze
+    # After: optional digits, CJK or end of string
+    ZH2_NUM_DASH = /^#{ZH_NUMERALS}/xo.freeze
+
+    ZH_PUNCT_CONTEXTS =
+      [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE],
+       [/(\s|^)$/, /^#{ZH_CHAR}/o]].freeze
+
+    # map of YAML punct keys to auto-text Latin equivalents
+    ZH_PUNCT_AUTOTEXT = {
+      colon: ":",
+      comma: ",",
+      "enum-comma": ",",
+      semicolon: ";",
+      period: ".",
+      "close-paren": ")",
+      "open-paren": "(",
+      "close-bracket": "]",
+      "open-bracket": "[",
+      "question-mark": "?",
+      "exclamation-mark": "!",
+      "em-dash": "—",
+      "open-quote": "“",
+      "close-quote": "”",
+      "open-nested-quote": "’",
+      "close-nested-quote": "’",
+      ellipse: "…",
+    }.freeze
+
+    # Pre-defined punctuation mappings for efficiency
+    def init_zh_punct_map
+      ZH_PUNCT_AUTOTEXT.each_with_object([]) do |(k, v), m|
+        @labels.dig("punct", k.to_s) or next
+        m << [v, @labels["punct"][k.to_s], ZH_PUNCT_CONTEXTS]
+      end
+    end
+
+    def l10n_zh(text, script, options)
+      script ||= "Hans"
+      t, text_cache, xml, prev, _foll, esc_indices = l10n_prep(text, options)
+      t.each_with_index do |n, i|
+        next if esc_indices.include?(i) # Skip escaped nodes
+
+        # Adjust index if prev context prepended
+        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
+        text = cleanup_entities(n.text, is_xml: false)
+        n.replace(l10_zh1(text, prev_ctx, foll_ctx, script, options))
+      end
+      to_xml(xml) #.gsub(/<\/?em>|<\/?strong>|<\/?i>|<\/?b>/, "")
+    end
+
+    # note: we can't differentiate comma from enumeration comma 、
+    # def l10_zh1(text, _script)
+    def l10_zh1(text, prev, foll, _script, options)
+      r = l10n_zh_punct(text, prev, foll, options)
+      r = l10n_zh_remove_space(r, prev, foll)
+      l10n_zh_dash(r, prev, foll)
+    end
+
+    def l10n_zh_punct(text, prev, foll, options)
+      # Use pre-defined mapping for better performance
+      @zh_punct_map ||= init_zh_punct_map
+      @zh_punct_map.each do |mapping|
+        punct_from, punct_to, regexes = mapping
+        options[:proportional_mixed_cjk] or regexes = nil
+        text = l10n_gsub(text, prev, foll, [punct_from, punct_to],
+                         regexes)
+      end
+      text
+    end
+
+    def l10n_zh_dash(text, prev, foll)
+      text = l10n_gsub(text, prev, foll, ["–", @labels.dig("punct", "en-dash")],
+                       [[ZH1_DASH, ZH2_DASH]])
+      l10n_gsub(text, prev, foll, ["–", @labels.dig("punct", "number-en-dash")],
+                [[ZH1_NUM_DASH, ZH2_NUM_DASH]])
+    end
+
+    def l10n_zh_remove_space(text, prev, foll)
+      text = l10n_gsub(text, prev, foll, [/\s+/, ""],
+                       [[/(#{ZH_CHAR})$/o, /^#{ZH_CHAR}/o]])
+      if sep = @labels.dig("punct", "cjk-latin-separator")
+        # Skip over punctuation to find Latin letters/numbers
+        text = l10n_gsub(text, prev, foll, [/\s+/, sep],
+                         [[/#{ZH_CHAR}$/o, /^\p{P}*[\p{Latin}\p{N}]/o]])
+        l10n_gsub(text, prev, foll, [/\s+/, sep],
+                  [[/[\p{Latin}\p{N}]\p{P}*$/o, /^#{ZH_NON_PUNCT}/o]])
+      else
+        l10n_gsub(text, prev, foll, [/\s+/, ""],
+                  [[/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o]])
+      end
+    end
+
+    def self.cjk_extend(text)
+      cjk_extend(text)
+    end
+
+    def cjk_extend(title)
+      @c.decode(title).chars.map.with_index do |n, i|
+        if i.zero? || !interleave_space_cjk?(title[i - 1] + title[i])
+          n
+        else "\u3000#{n}"
+        end
+      end.join
+    end
+
+    def interleave_space_cjk?(text)
+      text.size == 2 or return
+      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026",
+       "\u22ef\u22ef"].include?(text) ||
+        /\d\d|\p{Latin}\p{Latin}|[[:space:]]/.match?(text) ||
+        /^[\u2018\u201c(\u3014\[{\u3008\u300a\u300c\u300e\u3010\u2985\u3018\u3016\u00ab\u301d]/.match?(text) ||
+        /[\u2019\u201d)\u3015\]}\u3009\u300b\u300d\u300f\u3011\u2986\u3019\u3017\u00bb\u301f]$/.match?(text) ||
+        /[\u3002.\u3001,\u30fb:;\u2010\u301c\u30a0\u2013!?\u203c\u2047\u2048\u2049]/.match?(text) and return false
+      true
+    end
+  end
+end

data/lib/isodoc-yaml/i18n-en.yaml

@@ -2,3 +2,24 @@ text: text
 at: at
 binary_and: "%1 <conn>and</conn> %2"
 multiple_and: "%1<conn>, and</conn> %2"
+punct:
+  colon: ":"
+  comma: ","
+  enum_comma: ","
+  semicolon: ";"
+  period: "."
+  close_paren: ")"
+  open_paren: "("
+  close_bracket: "]"
+  open_bracket: "["
+  question_mark: "?"
+  exclamation_mark: "!"
+  em_dash: "—"
+  en_dash: "–"
+  number_en_dash: "–"
+  open_quote: "“"
+  close_quote: "”"
+  open_nested_quote: "’"
+  close_nested_quote: "’"
+  ellipse: …
+

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: isodoc-i18n
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-13 00:00:00.000000000 Z
+date: 2025-10-13 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
@@ -67,13 +81,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: base64
+  name: canon
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -170,58 +184,58 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: timecop
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
-  name: xml-c14n
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -257,6 +271,7 @@ files:
 - lib/isodoc/i18n.rb
 - lib/isodoc/i18n/version.rb
 - lib/isodoc/l10n.rb
+- lib/isodoc/l10n_cjk.rb
 - lib/isodoc/liquid/liquid.rb
 homepage: https://github.com/metanorma/isodoc-i18n
 licenses:
@@ -277,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: isodoc-i18n