isodoc-i18n 1.3.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a242d0bc7609246ea502ffec35e515457e739c478b25478a01c7ff641dc4523
-  data.tar.gz: 0f89c271766d244f037a0b6c6f40431888c2b6140814ac2b6e4d2906e142b1d5
+  metadata.gz: 134575d665c75368d3640ef87c9c63a3f6fdc2bf668e8d8df233a3cb139fce6b
+  data.tar.gz: ed1fc0e49c62f27b3199ba8357b2f1454309485efffbcb93ed30d51f01c36fe7
 SHA512:
-  metadata.gz: cb3f3f3a28b1b8fddd35ff74290ccf62f905ef54d51d62efd4fd8000d2d38a389eb4c78ec02aefd83ab0d1d98aaec462a0138c51257f8548c07f4a4726854c38
-  data.tar.gz: f6dcb2d4c05c630527d02888bf614a0e09396166e2f27437dedc0cc0a619149c6606fe2a8a78ce3c98948d4279ef3bfa224d9c1b18fbb1d38e74d92d280b74db
+  metadata.gz: f6e5ff44068372afc9ad75e1d6bff6483114a3424071c136a86f17745513ff5f31794c4555c9484eff556c50f638cb13c3ba712c4ae3e3b4d762685d1e888c2d
+  data.tar.gz: 9c5131cb3dbb8800304530629dfe5b47a3d7b15695ff0762011998c5aa3d3273b7730dde057dd2f2f15e3f997c9466704be80d7ef9cd47a5055e93afc0a5ba32

data/README.adoc

@@ -2,7 +2,7 @@
 
 image:https://img.shields.io/gem/v/isodoc-i18n.svg["Gem Version", link="https://rubygems.org/gems/isodoc-i18n"]
 image:https://github.com/metanorma/isodoc-i18n/workflows/rake/badge.svg["Build Status", link="https://github.com/metanorma/isodoc-i18n/actions?query=workflow%3Arake"]
-image:https://codeclimate.com/github/metanorma/isodoc-i18n/badges/gpa.svg["Code Climate", link="https://codeclimate.com/github/metanorma/isodoc-i18n"]
+// image:https://codeclimate.com/github/metanorma/isodoc-i18n/badges/gpa.svg["Code Climate", link="https://codeclimate.com/github/metanorma/isodoc-i18n"]
 image:https://img.shields.io/github/issues-pr-raw/metanorma/isodoc-i18n.svg["Pull Requests", link="https://github.com/metanorma/isodoc-i18n/pulls"]
 image:https://img.shields.io/github/commits-since/metanorma/isodoc-i18n/latest.svg["Commits since latest",link="https://github.com/metanorma/isodoc-i18n/releases"]
 

data/isodoc-i18n.gemspec

@@ -34,10 +34,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard-rspec", "~> 4.7"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
-  spec.add_development_dependency "rubocop", "~> 1.5.2"
+  spec.add_development_dependency "rubocop", "~> 1"
+spec.add_development_dependency "rubocop-performance"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "xml-c14n"
+  spec.add_development_dependency "canon"
   # spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/isodoc/i18n/version.rb

@@ -1,5 +1,5 @@
 module IsoDoc
   class I18n
-    VERSION = "1.3.2".freeze
+    VERSION = "1.4.0".freeze
   end
 end

data/lib/isodoc/i18n.rb

@@ -24,6 +24,8 @@ module IsoDoc
       self
     end
 
+    CJK_SCRIPTS = %w(Hans Hant Jpan Kore).freeze
+
     def liquid_init
       ::IsoDoc::I18n::Liquid.set(self)
       ::Liquid::Environment.default.register_filter(::IsoDoc::I18n::Liquid)
@@ -66,7 +68,7 @@ module IsoDoc
     end
 
     def enum_comma
-      %w(Hans Hant).include?(@script) and return "<enum-comma>、</enum-comma>"
+      CJK_SCRIPTS.include?(@script) and return "<enum-comma>、</enum-comma>"
       "<enum-comma>,</enum-comma> "
     end
 

data/lib/isodoc/l10n.rb

@@ -1,13 +1,55 @@
+require "metanorma-utils"
+
 module IsoDoc
   class I18n
-    def self.l10n(text, lang = @lang, script = @script, locale = @locale)
-      l10n(text, lang, script, locale)
-    end
-
-    # function localising spaces and punctuation.
-    def l10n(text, lang = @lang, script = @script, locale = @locale)
-      %w(zh ja ko).include?(lang) and text = l10n_zh(text, script)
-      lang == "fr" && text = l10n_fr(text, locale || "FR")
+    # Use comprehensive CJK definition from metanorma-utils
+    # This includes Han, Katakana, Hiragana, Hangul, Bopomofo and all CJK extensions
+    ZH_CHAR = "(#{Metanorma::Utils::CJK})".freeze
+    LATIN_PUNCT = /[:,.()\[\];?!-]/.freeze
+    
+    # Condition for converting punctuation to double width:
+    # 1. (Strict condition) CJK before, CJK after, modulo ignorable characters:
+    # 1a. CJK character, or start of string. Latin spaces optional.
+    ZH1_PUNCT = /(#{ZH_CHAR}|^)(\s*)$/xo.freeze 
+    # 1b. Latin spaces optional, Latin punct which will also convert to CJK, 
+    # CJK character, or end of string.
+    ZH2_PUNCT = /^\s*#{LATIN_PUNCT}*(#{ZH_CHAR}|$)/xo.freeze
+    # 2. CJK before, space after:
+    # 2a.  CJK char, followed by optional Latin punct which will also convert to CJK
+    ZH1_NO_SPACE = /#{ZH_CHAR}#{LATIN_PUNCT}*$/xo.freeze
+    # 2b. optional Latin punct which wil also convert to CJK, then space
+    OPT_PUNCT_SPACE = /^($|#{LATIN_PUNCT}*\s)/xo.freeze
+
+    # Contexts for converting en-dashes to full-width
+    # Before: CJK or start of string, optional digits
+    ZH1_DASH = /(#{ZH_CHAR}|^)(\d*)$/xo.freeze
+    # After: optional digits, CJK or end of string
+    ZH2_DASH = /^\d*(#{ZH_CHAR}|$)/xo.freeze
+    
+    # Pre-defined punctuation mappings for efficiency
+    ZH_PUNCT_MAP = [
+      [":：", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      [",，", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      [".。", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      [")）", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      ["]］", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      [";；", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      ["?？", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      ["!！", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      ["(（", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]],
+      ["[［", [[ZH1_PUNCT, ZH2_PUNCT], [ZH1_NO_SPACE, OPT_PUNCT_SPACE], [/(\s|^)$/, /^#{ZH_CHAR}/o]]]
+    ].freeze
+
+    def self.l10n(text, lang = @lang, script = @script, options = {})
+      l10n(text, lang, script, options)
+    end
+
+    # function localising spaces and punctuation
+    # options[:prev] and options[:foll] are optional context strings
+    def l10n(text, lang = @lang, script = @script, options = {})
+      locale = options[:locale] || @locale
+      %w(zh ja ko).include?(lang) and text = l10n_zh(text, script, options[:prev], options[:foll])
+      lang == "fr" && text = l10n_fr(text, locale || "FR", options[:prev], options[:foll])
       bidiwrap(text, lang, script)
     end
 
@@ -30,88 +72,100 @@ module IsoDoc
     end
 
     # CJK
-    def l10n_zh(text, script = "Hans")
-      xml = Nokogiri::XML::DocumentFragment.parse(text)
-      t = xml.xpath(".//text()")
+    def l10n_zh(text, script, prev, foll)
+      script ||= "Hans"
+      t, text_cache, xml = l10n_prep(text, prev, foll)
       t.each_with_index do |n, i|
-        prev, foll = l10n_context(t, i)
+        # Adjust index if prev context prepended
+        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
         text = cleanup_entities(n.text, is_xml: false)
-        n.replace(l10_zh1(text, prev, foll, script))
+        n.replace(l10_zh1(text, prev_ctx, foll_ctx, script))
       end
-      to_xml(xml).gsub(/<b>/, "").gsub("</b>", "")
-        .gsub(/<\?[^>]+>/, "")
+      to_xml(xml).gsub(/<b>|<\/b>|<\?[^>]+>/, "")
+    end
+
+    def l10n_prep(text, prev, foll)
+            xml = Nokogiri::XML::DocumentFragment.parse(text)
+      t = xml.xpath(".//text()")
+      text_cache = build_text_cache(t, prev, foll)
+      [t, text_cache, xml]
+    end
+
+    # Cache text content once per method call to avoid repeated .text calls
+    # Build text cache with optional prepended/appended context
+    def build_text_cache(text_nodes, prev_context = nil, foll_context = nil)
+      text_cache = text_nodes.map(&:text)
+      text_cache.unshift(prev_context) if prev_context
+      text_cache.push(foll_context) if foll_context
+      text_cache
     end
 
     # previous, following context of current text node:
     # do not use just the immediately adjoining text tokens for context
     # deal with spaces and empty text by just concatenating entire context
+    # Optimized to avoid O(n²) complexity by using pre-cached text content
+    def l10n_context_cached(text_cache, idx)
+      prev = text_cache[0...idx].join
+      foll = text_cache[(idx + 1)...text_cache.size].join
+      [prev, foll]
+    end
+
+    # Fallback method for backward compatibility
     def l10n_context(nodes, idx)
       prev = nodes[0...idx].map(&:text).join
       foll = nodes[(idx + 1)...(nodes.size)].map(&:text).join
       [prev, foll]
     end
 
-    def l10n_fr(text, locale)
-      xml = Nokogiri::XML::DocumentFragment.parse(text)
-      t = xml.xpath(".//text()")
+    def l10n_fr(text, locale, prev, foll)
+      t, text_cache, xml = l10n_prep(text, prev, foll)
       t.each_with_index do |n, i|
-        prev, foll = l10n_context(t, i)
+        prev_ctx, foll_ctx = l10n_context_cached(text_cache, prev ? i + 1 : i)
         text = cleanup_entities(n.text, is_xml: false)
-        n.replace(l10n_fr1(text, prev, foll, locale))
+        n.replace(l10n_fr1(text, prev_ctx, foll_ctx, locale))
       end
       to_xml(xml)
     end
 
-    ZH_CHAR = "(\\p{Han}|\\p{In CJK Symbols And Punctuation}|" \
-              "\\p{In Halfwidth And Fullwidth Forms})".freeze
-
     # note: we can't differentiate comma from enumeration comma 、
     # def l10_zh1(text, _script)
     def l10_zh1(text, prev, foll, _script)
-      # l10n_zh_dash(l10n_zh_remove_space(l10n_zh_punct(text)))
       r = l10n_zh_punct(text, prev, foll)
       r = l10n_zh_remove_space(r, prev, foll)
       l10n_zh_dash(r, prev, foll)
     end
 
-    ZH1_PUNCT = /(#{ZH_CHAR}|^)   # CJK character, or start of string
-         (\s*)$                   # Latin spaces optional
-    /xo.freeze
-    ZH2_PUNCT = /^\s*             # followed by ignorable Latin spaces
-                [:,.()\[\];?!-]*  # Latin punct which will also convert to CJK
-                (#{ZH_CHAR}|$)    # CJK character, or end of string
-      /xo.freeze
-
-    # CJK punct if (^|CJK).($|CJK)
     def l10n_zh_punct(text, prev, foll)
-      [":：", ",，", ".．", ")）", "]］", ";；", "?？", "!！", "(（", "[［"].each do |m|
-        text = l10n_gsub(text, prev, foll, [m[0], m[1]],
-                         [ZH1_PUNCT, ZH2_PUNCT])
+      # Use pre-defined mapping for better performance
+      ZH_PUNCT_MAP.each do |mapping|
+        punct_pair, regexes = mapping
+        text = l10n_gsub(text, prev, foll, [punct_pair[0], punct_pair[1]], regexes)
       end
       text
     end
 
-    ZH1_DASH = /(#{ZH_CHAR}|^)    # CJK character, or start of string
-                (\d*)             # optional digits
-    $/xo.freeze
-
-    ZH2_DASH = /^\d*              # followed by optional digits
-                (#{ZH_CHAR}|$)    # CJK character, or end of string
-      /xo.freeze
-
     def l10n_zh_dash(text, prev, foll)
-      l10n_gsub(text, prev, foll, %w(– ～), [ZH1_DASH, ZH2_DASH])
+      l10n_gsub(text, prev, foll, %w(– ～), [[ZH1_DASH, ZH2_DASH]])
     end
 
-    def l10n_gsub(text, prev, foll, delim, regex)
+    # text: string we are scanning for instances of delim[0] to replace
+    # prev: string preceding text, as additional token of context
+    # foll: string following text, as additional token of context
+    # delim: delim[0] is the symbol we want to replace, delim[1] its replacement
+    # regexes: a list of regex pairs: the context before the found token,
+    # and the context after the found token, under which replacing it
+    # with delim[1] is permitted
+    def l10n_gsub(text, prev, foll, delim, regexes)
       context = l10n_gsub_context(text, prev, foll, delim) or return text
       (1...(context.size - 1)).each do |i|
-        l10_context_valid?(context, i, delim, regex) and
+        l10_context_valid?(context, i, delim, regexes) and
           context[i] = delim[1].gsub("\\0", context[i]) # Full-width equivalent
       end
       context[1...(context.size - 1)].join
     end
 
+    # split string being scanned, and its contextual tokens before and after,
+    # into array of tokens determining whether to replace instances of delim[0]
     def l10n_gsub_context(text, prev, foll, delim)
       d = delim[0].is_a?(Regexp) ? delim[0] : Regexp.quote(delim[0])
       context = text.split(/(#{d})/) # delim to replace
@@ -120,30 +174,36 @@ module IsoDoc
     end
 
     def l10_context_valid?(context, idx, delim, regex)
-      found_delim = if delim[0].is_a?(Regexp) # punct to convert
-                      delim[0].match?(context[idx])
-                    else
-                      context[idx] == delim[0]
-                    end
-      found_delim &&
-        regex[0].match?(context[0...idx].join) && # preceding context
-        regex[1].match?(context[(idx + 1)..-1].join) # foll context
+      l10n_context_found_delimiter?(context[idx], delim) or return false
+      regex.detect do |r|
+        r[0].match?(context[0...idx].join) && # preceding context
+          r[1].match?(context[(idx + 1)..-1].join) # foll context
+      end
+    end
+
+    def l10n_context_found_delimiter?(token, delim)
+      if delim[0].is_a?(Regexp) # punct to convert
+        delim[0].match?(token)
+      else
+        token == delim[0]
+      end
     end
 
     def l10n_zh_remove_space(text, prev, foll)
       text = l10n_gsub(text, prev, foll, [" ", ""],
-                       [/(#{ZH_CHAR}|\d)$/o, /^#{ZH_CHAR}/o])
+                       [[/(#{ZH_CHAR}|\d)$/o, /^#{ZH_CHAR}/o]])
       l10n_gsub(text, prev, foll, [" ", ""],
-                [/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o])
+                [[/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o]])
     end
 
     def l10n_fr1(text, prev, foll, locale)
       text = l10n_gsub(text, prev, foll, [/[»›;?!]/, "\u202f\\0"],
-                       [/\p{Alnum}$/, /^(\s|$)/])
-      text = l10n_gsub(text, prev, foll, [/[«‹]/, "\\0\u202f"], [/$/, /^(?!\p{Zs})./])
+                       [[/\p{Alnum}$/, /^(\s|$)/]])
+      text = l10n_gsub(text, prev, foll, [/[«‹]/, "\\0\u202f"],
+                       [[/$/, /^(?!\p{Zs})./]])
       colonsp = locale == "CH" ? "\u202f" : "\u00a0"
       l10n_gsub(text, prev, foll, [":", "#{colonsp}\\0"],
-                [/\p{Alnum}$/, /^(\s|$)/])
+                [[/\p{Alnum}$/, /^(\s|$)/]])
     end
 
     def self.cjk_extend(text)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isodoc-i18n
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-13 00:00:00.000000000 Z
+date: 2025-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: htmlentities
@@ -170,14 +170,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -221,7 +235,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: xml-c14n
+  name: canon
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -277,7 +291,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: isodoc-i18n