metanorma-utils 1.10.3 → 1.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1529d7af0a046a40b3b079adbc36786882be1abce4b515b2ec644c9ade8fb6eb
-  data.tar.gz: 756653fafb5a833206d2f60e7cbcc64c8c0f7dc2d94149655efbd06395694be7
+  metadata.gz: bb5a317fefc85b76b0442eadf4805403c76e68384b9ea22685941442db330a3f
+  data.tar.gz: 01cf01d1d045ab721fdba705d79757af0e18dd025f67f070731a7029f1736c1e
 SHA512:
-  metadata.gz: f62f4879f665181f06d1102785dac3e8481cd4d15c9e982c481af56bd079a4440205a7d18549dd1b53ed0c8bc127119742904b088e145ee4f0710f5745b129da
-  data.tar.gz: 8234bac72f1dd5f432af91442c3802a2c03f3ba7f928ef89c2feba5c5a6838176fd2d271f2beb63a70a0c613b24d94a5b6816e594e40c7851070f1125b336716
+  metadata.gz: be7834de49cb96791995188b265fb1cf97fa29366ff367931007ea4101233aa16317f5a31e4b4cb837b9ea6418f92c68df0848575d3ba6b3fce8e50c072e9571
+  data.tar.gz: 2f2b23b525347b8cb07d2cba54376685357c1ef1b2d78120504f91e91fa10eef1c0298cf5c4ef03ef1e97ee43d9ac9340b4c2657d2196980dfd548ee1b4f7a94

data/lib/utils/cjk.rb

@@ -0,0 +1,110 @@
+module Metanorma
+  module Utils
+    class << self
+      # Basic CJK scripts
+      HAN = "\\p{Han}".freeze
+      BOPOMOFO = "\\p{Bopomofo}".freeze
+      HANGUL = "\\p{Hangul}".freeze
+      HIRAGANA = "\\p{Hiragana}".freeze
+      KATAKANA = "\\p{Katakana}".freeze
+      
+      # Script extensions - characters shared between scripts
+      
+      # CJK Symbols and Punctuation (U+3000–U+303F)
+      # Used across all CJK scripts
+      CJK_SYMBOLS = "[\\u3000-\\u303F]".freeze
+      
+      # CJK Punctuation (subset of CJK Symbols commonly used)
+      CJK_PUNCTUATION = "[\\u3001-\\u3003\\u3008-\\u3011\\u3014-\\u301F]".freeze
+      
+      # Halfwidth and Fullwidth Forms (U+FF00–U+FFEF)
+      # Used in all CJK contexts
+      CJK_HALFWIDTH_FULLWIDTH = "[\\uFF00-\\uFFEF]".freeze
+      
+      # CJK Compatibility Forms (U+FE30–U+FE4F)
+      # Primarily used with Han but relevant for all CJK
+      CJK_COMPAT = "[\\uFE30-\\uFE4F]".freeze
+      
+      # Vertical Forms (U+FE10–U+FE1F)
+      # Used in vertical text layout for all CJK
+      CJK_VERTICAL = "[\\uFE10-\\uFE1F]".freeze
+      
+      # Small Form Variants (U+FE50–U+FE6F)
+      # Used in all CJK contexts
+      CJK_SMALL_FORMS = "[\\uFE50-\\uFE6F]".freeze
+      
+      # Ideographic Description Characters (U+2FF0–U+2FFF)
+      # Used with Han script
+      HAN_IDC = "[\\u2FF0-\\u2FFF]".freeze
+      
+      # Kanbun (U+3190–U+319F)
+      # Used with Han script for Japanese
+      KANBUN = "[\\u3190-\\u319F]".freeze
+      
+      # CJK Compatibility (U+3300–U+33FF)
+      # Used with Han script
+      CJK_COMPAT_IDEOGRAPHS = "[\\u3300-\\u33FF]".freeze
+      
+      # CJK Compatibility Ideographs (U+F900–U+FAFF)
+      HAN_COMPAT_IDEOGRAPHS = "[\\uF900-\\uFAFF]".freeze
+      
+      # Script extensions by primary script
+      HAN_EXTENSIONS = [
+        HAN,
+        CJK_SYMBOLS,
+        CJK_PUNCTUATION,
+        CJK_HALFWIDTH_FULLWIDTH,
+        CJK_COMPAT,
+        CJK_VERTICAL,
+        CJK_SMALL_FORMS,
+        HAN_IDC,
+        KANBUN,
+        CJK_COMPAT_IDEOGRAPHS,
+        HAN_COMPAT_IDEOGRAPHS
+      ].join("|").freeze
+      
+      HANGUL_EXTENSIONS = [
+        HANGUL,
+        CJK_SYMBOLS,
+        CJK_PUNCTUATION,
+        CJK_HALFWIDTH_FULLWIDTH,
+        CJK_VERTICAL,
+        CJK_SMALL_FORMS
+      ].join("|").freeze
+      
+      HIRAGANA_EXTENSIONS = [
+        HIRAGANA,
+        CJK_SYMBOLS,
+        CJK_PUNCTUATION,
+        CJK_HALFWIDTH_FULLWIDTH,
+        CJK_VERTICAL,
+        CJK_SMALL_FORMS
+      ].join("|").freeze
+      
+      KATAKANA_EXTENSIONS = [
+        KATAKANA,
+        CJK_SYMBOLS,
+        CJK_PUNCTUATION,
+        CJK_HALFWIDTH_FULLWIDTH,
+        CJK_VERTICAL,
+        CJK_SMALL_FORMS
+      ].join("|").freeze
+      
+      BOPOMOFO_EXTENSIONS = [
+        BOPOMOFO,
+        CJK_SYMBOLS,
+        CJK_PUNCTUATION,
+        CJK_HALFWIDTH_FULLWIDTH
+      ].join("|").freeze
+      
+      # Combined CJK pattern including all script extensions
+      CJK = [
+        HAN_EXTENSIONS,
+        HANGUL_EXTENSIONS,
+        HIRAGANA_EXTENSIONS,
+        KATAKANA_EXTENSIONS,
+        BOPOMOFO_EXTENSIONS
+      ].join("|").freeze
+    end
+  end
+end

data/lib/utils/log.rb

@@ -86,7 +86,7 @@ module Metanorma
           while !node.nil? && node["id"].nil? && node.respond_to?(:parent)
             node = node.parent
           end
-          node.respond_to?(:parent) ? "ID #{node['id']}" : ""
+          node.respond_to?(:parent) ? "ID #{node['anchor'] || node['id']}" : ""
         elsif node.is_a? String then node
         elsif node.respond_to?(:lineno) && !node.lineno.nil? &&
             !node.lineno.empty?
@@ -224,7 +224,7 @@ module Metanorma
         /^ID /.match?(loc) or return [loc, nil]
         loc.sub!(/^ID /, "")
         loc = @mapid[loc] while @mapid[loc]
-        url = "#{@htmlfilename}##{loc}"
+        url = "#{@htmlfilename}##{to_ncname loc}"
         [loc, url]
       end
 

data/lib/utils/main.rb

@@ -5,6 +5,7 @@ require "htmlentities"
 require "nokogiri"
 require "csv"
 require_relative "../sterile/sterile"
+require_relative "cjk"
 
 module Metanorma
   module Utils
@@ -35,9 +36,6 @@ module Metanorma
         docfile.nil? ? "./" : "#{Pathname.new(docfile).parent}/"
       end
 
-      CJK = "\\p{Han}|\\p{Bopomofo}|\\p{Hangul}|\\p{Hiragana}|\\p{Katakana}"
-        .freeze
-
       # TODO needs internationalisation of quote
       def smartformat(text)
         ret = HTMLEntities.new.decode(

data/lib/utils/version.rb

@@ -1,5 +1,5 @@
 module Metanorma
   module Utils
-    VERSION = "1.10.3".freeze
+    VERSION = "1.11.3".freeze
   end
 end

data/lib/utils/xml.rb

@@ -40,6 +40,41 @@ module Metanorma
         (ret1 || "") + ret2.gsub(%r([#{NAMECHAR}#])o, "_")
       end
 
+      # Following XML requirements: https://www.w3.org/TR/REC-xml/#NT-Name
+      TAG_NAME_START_CODEPOINTS = "@:A-Z_a-z\u{C0}-\u{D6}\u{D8}-\u{F6}\u{F8}-\u{2FF}\u{370}-\u{37D}\u{37F}-\u{1FFF}" \
+                                  "\u{200C}-\u{200D}\u{2070}-\u{218F}\u{2C00}-\u{2FEF}\u{3001}-\u{D7FF}\u{F900}-\u{FDCF}" \
+                                  "\u{FDF0}-\u{FFFD}\u{10000}-\u{EFFFF}"
+      INVALID_TAG_NAME_START_REGEXP = /[^#{TAG_NAME_START_CODEPOINTS}]/
+      TAG_NAME_FOLLOWING_CODEPOINTS = "#{TAG_NAME_START_CODEPOINTS}\\-.0-9\u{B7}\u{0300}-\u{036F}\u{203F}-\u{2040}"
+      INVALID_TAG_NAME_FOLLOWING_REGEXP = /[^#{TAG_NAME_FOLLOWING_CODEPOINTS}]/
+      SAFE_XML_TAG_NAME_REGEXP = /\A[#{TAG_NAME_START_CODEPOINTS}][#{TAG_NAME_FOLLOWING_CODEPOINTS}]*\z/
+      TAG_NAME_REPLACEMENT_CHAR = "_"
+
+      # from: https://github.com/rails/rails/blob/3235827585d87661942c91bc81f64f56d710f0b2/activesupport/lib/active_support/core_ext/erb/util.rb
+      # A utility method for escaping XML names of tags and names of attributes.
+      #
+      #   xml_name_escape('1 < 2 & 3')
+      #   # => "1___2___3"
+      #
+      # It follows the requirements of the specification: https://www.w3.org/TR/REC-xml/#NT-Name
+      def to_ncname(name, asciionly: true)
+        name = name.to_s
+        return "" if name.nil? || name.empty?
+        return name if name.match?(SAFE_XML_TAG_NAME_REGEXP)
+
+        starting_char = name[0]
+        starting_char.gsub!(INVALID_TAG_NAME_START_REGEXP,
+                            TAG_NAME_REPLACEMENT_CHAR)
+
+        return starting_char if name.size == 1
+
+        following_chars = name[1..-1]
+        following_chars.gsub!(INVALID_TAG_NAME_FOLLOWING_REGEXP,
+                              TAG_NAME_REPLACEMENT_CHAR)
+
+        starting_char << following_chars
+      end
+
       def anchor_or_uuid(node = nil)
         uuid = UUIDTools::UUID.random_create
         node.nil? || node.id.nil? || node.id.empty? ? "_#{uuid}" : node.id
@@ -69,7 +104,7 @@ module Metanorma
           cjk2 = /#{CJK}/o.match?(nextfirst)
           text1 = /[^\p{Z}\p{C}]/.match?(last)
           text2 = /[^\p{Z}\p{C}]/.match?(nextfirst)
-          (cjk1 && (cjk2 || !text2)) and next
+          cjk1 && (cjk2 || !text2) and next
           !text1 && cjk2 and next
           ret[i] += " "
         end
@@ -125,10 +160,18 @@ module Metanorma
       end
 
       # all element/attribute pairs that are ID anchors in Metanorma
-      def anchor_attributes
-        [%w[* id], %w[* bibitemid], %w[review from],
-         %w[review to], %w[index to], %w[xref target],
-         %w[callout target], %w[location target]]
+      def anchor_attributes(presxml: false)
+        ret = [%w(review from), %w(review to), %w(callout target), %w(xref to),
+               %w(eref bibitemid), %w(citation bibitemid), %w(xref target),
+               %w(label for), %w(location target), %w(index to),
+               %w(termsource bibitemid), %w(admonition target)]
+        ret1 = [%w(fn target), %w(semx source), %w(fmt-title source),
+                %w(fmt-xref to), %w(fmt-xref target), %w(fmt-eref bibitemid),
+                %w(fmt-xref-label container), %w(fmt-fn-body target),
+                %w(fmt-review-start source), %w(fmt-review-start end),
+                %w(fmt-review-start target), %w(fmt-review-end source),
+                %w(fmt-review-end start), %w(fmt-review-end target)]
+        presxml ? ret + ret1 : ret
       end
 
       # convert definition list term/value pair into Nokogiri XML attribute

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-utils
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.11.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-14 00:00:00.000000000 Z
+date: 2025-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -321,6 +321,7 @@ files:
 - README.adoc
 - lib/metanorma-utils.rb
 - lib/sterile/sterile.rb
+- lib/utils/cjk.rb
 - lib/utils/hash_transform_keys.rb
 - lib/utils/image.rb
 - lib/utils/linestatus.rb