isodoc-i18n 1.2.1 → 1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7552e0a62364e47c4f8ea7b2810e1ba5d11e39d2a3711792c826412b1aef661f
-  data.tar.gz: 29d4eef4f2a07bcf6ad1de7ddc229cf7ff9b0ffa8e55c81652d2174165cf2214
+  metadata.gz: d1cb9e9bc5f9e053a31ca971936606b5248577c9b001d3c858d8ecaed201ced8
+  data.tar.gz: 0ea1ee1c8b6913c3a708d63eee6f6f0af5f6ba171a65b04a7792063bd117a339
 SHA512:
-  metadata.gz: 220fddc821f5f00d4f1eecca1f7583b0236254bb0a5141c50d7583f4c725be2b4aca949ffc4f89ed390c01850f2912d86b9562959773efabf1a3df4838e8ec60
-  data.tar.gz: 59373ef3a3688bf4f382da1e2a5df55b3b7ae1615385bc04504faf9421e9c5e66333ed1859243f0e8a86d4f42724692ab1cd9b8f9f0bd62d4c4e70cceb7e93cd
+  metadata.gz: a40fba88473c09f93b3eaf3da323c22c6974e9a50a2a287d4235ba5a757852a38a553765e9fdf6289d0dd97721e4c03660e390d06ece34a6f082b154377c041b
+  data.tar.gz: c8e0127577dda72be74e84f9e211169b682817f12ba38507d98c81ac5da649b5ae92d3a4d29f1d63b38ec244c4dbc44a23cf95eb0043773d4e6ae3b59309bd1c

data/isodoc-i18n.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "liquid", "~> 5"
   spec.add_dependency "metanorma-utils", ">= 1.7.0"
   spec.add_dependency "twitter_cldr"
+  spec.add_dependency "base64"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
@@ -37,5 +38,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "webmock"
+  spec.add_development_dependency "xml-c14n"
   # spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/isodoc/i18n/version.rb

@@ -1,5 +1,5 @@
 module IsoDoc
   class I18n
-    VERSION = "1.2.1".freeze
+    VERSION = "1.2.3".freeze
   end
 end

data/lib/isodoc/i18n.rb

@@ -6,6 +6,7 @@ require_relative "l10n"
 require_relative "liquid/liquid"
 require "liquid"
 require_relative "i18n/version"
+require "base64"
 
 module IsoDoc
   class I18n

data/lib/isodoc/l10n.rb

@@ -5,9 +5,8 @@ module IsoDoc
     end
 
     # function localising spaces and punctuation.
-    # Not clear if period needs to be localised for zh
     def l10n(text, lang = @lang, script = @script, locale = @locale)
-      lang == "zh" and text = l10n_zh(text, script)
+      %w(zh ja ko).include?(lang) and text = l10n_zh(text, script)
       lang == "fr" && text = l10n_fr(text, locale || "FR")
       bidiwrap(text, lang, script)
     end
@@ -30,23 +29,35 @@ module IsoDoc
          .default_script(@lang))]
     end
 
+    # CJK
     def l10n_zh(text, script = "Hans")
       xml = Nokogiri::XML::DocumentFragment.parse(text)
-      xml.traverse do |n|
-        next unless n.text?
-
-        n.replace(l10_zh1(cleanup_entities(n.text, is_xml: false), script))
+      t = xml.xpath(".//text()")
+      t.each_with_index do |n, i|
+        prev, foll = l10n_context(t, i)
+        text = cleanup_entities(n.text, is_xml: false)
+        n.replace(l10_zh1(text, prev, foll, script))
       end
       xml.to_xml(encoding: "UTF-8").gsub(/<b>/, "").gsub("</b>", "")
         .gsub(/<\?[^>]+>/, "")
     end
 
+    # previous, following context of current text node:
+    # do not use just the immediately adjoining text tokens for context
+    # deal with spaces and empty text by just concatenating entire context
+    def l10n_context(nodes, idx)
+      prev = nodes[0...idx].map(&:text).join
+      foll = nodes[(idx + 1)...(nodes.size)].map(&:text).join
+      [prev, foll]
+    end
+
     def l10n_fr(text, locale)
       xml = Nokogiri::XML::DocumentFragment.parse(text)
-      xml.traverse do |n|
-        next unless n.text?
-
-        n.replace(l10n_fr1(cleanup_entities(n.text, is_xml: false), locale))
+      t = xml.xpath(".//text()")
+      t.each_with_index do |n, i|
+        prev, foll = l10n_context(t, i)
+        text = cleanup_entities(n.text, is_xml: false)
+        n.replace(l10n_fr1(text, prev, foll, locale))
       end
       xml.to_xml(encoding: "UTF-8")
     end
@@ -55,36 +66,84 @@ module IsoDoc
               "\\p{In Halfwidth And Fullwidth Forms}".freeze
 
     # note: we can't differentiate comma from enumeration comma 、
-    def l10_zh1(text, _script)
-      l10n_zh_remove_space(l10n_zh_punct(text))
+    # def l10_zh1(text, _script)
+    def l10_zh1(text, prev, foll, _script)
+      # l10n_zh_dash(l10n_zh_remove_space(l10n_zh_punct(text)))
+      r = l10n_zh_punct(text, prev, foll)
+      r = l10n_zh_remove_space(r, prev, foll)
+      l10n_zh_dash(r, prev, foll)
     end
 
-    def l10n_zh_punct(text)
-      [":：", ",，", ".．", ")）", "]］", ":：", ";；", "?？", "!！", "–～"].each do |m|
-        text = text.gsub(/#{Regexp.quote m[0]}/, m[1])
-      end
-      ["(（", "[［"].each do |m|
-        text = text.gsub(/#{Regexp.quote m[0]}/, m[1])
+    ZH1_PUNCT = /(#{ZH_CHAR}|^)   # CJK character, or start of string
+         (\s*)$                   # Latin spaces optional
+    /xo.freeze
+    ZH2_PUNCT = /^\s*             # followed by ignorable Latin spaces
+                [:,.()\[\];?!-]*  # Latin punct which will also convert to CJK
+                (#{ZH_CHAR}|$)    # CJK character, or end of string
+      /xo.freeze
+
+    # CJK punct if (^|CJK).($|CJK)
+    def l10n_zh_punct(text, prev, foll)
+      [":：", ",，", ".．", ")）", "]］", ";；", "?？", "!！", "(（", "[［"].each do |m|
+        text = l10n_gsub(text, prev, foll, [m[0], m[1]],
+                         [ZH1_PUNCT, ZH2_PUNCT])
       end
       text
     end
 
-    def l10n_zh_remove_space(text)
-      text.gsub(/(?<=#{ZH_CHAR}) (?=#{ZH_CHAR})/o, "")
-        .gsub(/(?<=\d) (?=#{ZH_CHAR})/o, "")
-        .gsub(/(?<=#{ZH_CHAR}) (?=\d)/o, "")
-        .gsub(/(?<=#{ZH_CHAR}) (?=[A-Za-z](#{ZH_CHAR}|$))/o, "")
+    ZH1_DASH = /(#{ZH_CHAR}|^)    # CJK character, or start of string
+                (\d*)             # optional digits
+    $/xo.freeze
+
+    ZH2_DASH = /^\d*              # followed by optional digits
+                (#{ZH_CHAR}|$)    # CJK character, or end of string
+      /xo.freeze
+
+    def l10n_zh_dash(text, prev, foll)
+      l10n_gsub(text, prev, foll, %w(– ～), [ZH1_DASH, ZH2_DASH])
+    end
+
+    def l10n_gsub(text, prev, foll, delim, regex)
+      context = l10n_gsub_context(text, prev, foll, delim) or return text
+      (1...(context.size - 1)).each do |i|
+        l10_context_valid?(context, i, delim, regex) and
+          context[i] = delim[1].gsub("\\0", context[i]) # Full-width equivalent
+      end
+      context[1...(context.size - 1)].join
+    end
+
+    def l10n_gsub_context(text, prev, foll, delim)
+      d = delim[0].is_a?(Regexp) ? delim[0] : Regexp.quote(delim[0])
+      context = text.split(/(#{d})/) # delim to replace
+      context.size == 1 and return
+      [prev, context, foll].flatten
+    end
+
+    def l10_context_valid?(context, idx, delim, regex)
+      found_delim = if delim[0].is_a?(Regexp) # punct to convert
+                      delim[0].match?(context[idx])
+                    else
+                      context[idx] == delim[0]
+                    end
+      found_delim &&
+        regex[0].match?(context[0...idx].join) && # preceding context
+        regex[1].match?(context[(idx + 1)..-1].join) # foll context
+    end
+
+    def l10n_zh_remove_space(text, prev, foll)
+      text = l10n_gsub(text, prev, foll, [" ", ""],
+                       [/(#{ZH_CHAR}|\d)$/o, /^#{ZH_CHAR}/o])
+      l10n_gsub(text, prev, foll, [" ", ""],
+                [/#{ZH_CHAR}$/o, /^(\d|[A-Za-z](#{ZH_CHAR}|$))/o])
     end
 
-    def l10n_fr1(text, locale)
-      text = text.gsub(/(?<=\p{Alnum})([»›;?!])(?=\s)/, "\u202f\\1")
-      text = text.gsub(/(?<=\p{Alnum})([»›;?!])$/, "\u202f\\1")
-      text = text.gsub(/^([»›;?!])/, "\u202f\\1")
-      text = text.gsub(/([«‹])/, "\\1\u202f")
+    def l10n_fr1(text, prev, foll, locale)
+      text = l10n_gsub(text, prev, foll, [/[»›;?!]/, "\u202f\\0"],
+                       [/\p{Alnum}$/, /^(\s|$)/])
+      text = l10n_gsub(text, prev, foll, [/[«‹]/, "\\0\u202f"], [/$/, /^./])
       colonsp = locale == "CH" ? "\u202f" : "\u00a0"
-      text = text.gsub(/(?<=\p{Alnum})(:)(?=\s)/, "#{colonsp}\\1")
-      text = text.gsub(/(?<=\p{Alnum})(:)$/, "#{colonsp}\\1")
-      text.gsub(/^(:\s)/, "#{colonsp}\\1")
+      l10n_gsub(text, prev, foll, [":", "#{colonsp}\\0"],
+                [/\p{Alnum}$/, /^(\s|$)/])
     end
 
     def self.cjk_extend(text)
@@ -102,7 +161,8 @@ module IsoDoc
 
     def interleave_space_cjk?(text)
       text.size == 2 or return
-      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026", "\u22ef\u22ef"].include?(text) ||
+      ["\u2014\u2014", "\u2025\u2025", "\u2026\u2026",
+       "\u22ef\u22ef"].include?(text) ||
         /\d\d|\p{Latin}\p{Latin}|[[:space:]]/.match?(text) ||
         /^[\u2018\u201c(\u3014\[{\u3008\u300a\u300c\u300e\u3010\u2985\u3018\u3016\u00ab\u301d]/.match?(text) ||
         /[\u2019\u201d)\u3015\]}\u3009\u300b\u300d\u300f\u3011\u2986\u3019\u3017\u00bb\u301f]$/.match?(text) ||

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: isodoc-i18n
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-19 00:00:00.000000000 Z
+date: 2024-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: htmlentities
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +220,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: xml-c14n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'Internationalisation for Metanorma rendering
 
   '
@@ -249,7 +277,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key: 
 specification_version: 4
 summary: isodoc-i18n