metanorma-utils 1.0.6 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a46112cc35941403c557671059d0f31be446e84cf5a91d6fc9cc3e16e6b5b00
-  data.tar.gz: 7704505b81224537aabb3679a753a5ee4a5b02832034f7265a467232dbfb26e7
+  metadata.gz: a355e89f8c66d323f0ca9954226b2255c1c4706ab9346db5004aa7984ceade6c
+  data.tar.gz: cc240e1eec9ea056d87b38bae0fea45ac16f1e6f2cef1a6cb2d04c2d20de50e9
 SHA512:
-  metadata.gz: 7be04b8ee6662d7b82f0c6a5d29026a371607ccc1e5259781a66d6bd10980be3a37eab4f5a5d44aaa6c228f6dcb35b1801842c17362d46b9ab501d3e94888363
-  data.tar.gz: 912b02b4e6d6b5a2f9b3338b4868f03cfc36c3a96f9d9cb25b7f07a4a02acb5b2d73f857fa19b238a364da8ae1c30d0e37ca609281c69bd14cb9f502b70aa1c5
+  metadata.gz: 20d8d717e2113249d060bf15f15faf1150cf5e06d29368f9faf00b0a496aa7c3e711a80c83dbb6fd328cf39ac2e65359825a6632199fb6cdf13cb882934ac11f
+  data.tar.gz: be4fe517ccc3731678ebaa5fedf2b7ec0923e510c83193a5f73d26db3b9a290cbe3be3aaeda4dbfa90fd76fd8c1bf14099f283b6bd158b00c5138d488c2fdf14

data/.github/workflows/rake.yml

@@ -30,7 +30,9 @@ jobs:
             os: 'macos-latest'
             experimental: true
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - uses: ruby/setup-ruby@v1
         with:
@@ -45,7 +47,7 @@ jobs:
     steps:
       - uses: peter-evans/repository-dispatch@v1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.METANORMA_CI_PAT_TOKEN || secrets.GITHUB_TOKEN }}
           repository: ${{ github.repository }}
           event-type: notify
-          client-payload: '{"ref": "${{ github.ref }}"}'
+          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

data/.gitignore

@@ -0,0 +1 @@
+.rubocop-https--*

data/.rubocop.yml

@@ -1,6 +1,14 @@
 # This project follows the Ribose OSS style guide.
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
-
 inherit_from:
-  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+
+# local repo-specific modifications
+
+AllCops:
+  DisplayCopNames: false
+  StyleGuideCopsOnly: false
+  TargetRubyVersion: 2.4
+Rails:
+  Enabled: true

data/lib/metanorma-utils.rb

@@ -1,3 +1,4 @@
 require_relative "utils/version"
 require_relative "utils/main"
+require_relative "utils/image"
 require_relative "utils/log"

data/lib/utils/image.rb

@@ -0,0 +1,143 @@
+require "asciidoctor"
+require "tempfile"
+require "marcel"
+require "mime/types"
+require "base64"
+
+module Metanorma
+  module Utils
+    class << self
+      class Namespace
+        def initialize(xmldoc)
+          @namespace = xmldoc.root.namespace
+        end
+
+        def ns(path)
+          return path if @namespace.nil?
+
+          path.gsub(%r{/([a-zA-z])}, "/xmlns:\\1")
+            .gsub(%r{::([a-zA-z])}, "::xmlns:\\1")
+            .gsub(%r{\[([a-zA-z][a-z0-9A-Z@/]* ?=)}, "[xmlns:\\1")
+            .gsub(%r{\[([a-zA-z][a-z0-9A-Z@/]*\])}, "[xmlns:\\1")
+        end
+      end
+
+      def save_dataimage(uri)
+        %r{^data:(image|application)/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ uri
+        imgtype.sub!(/\+[a-z0-9]+$/, "") # svg+xml
+        imgtype = "png" unless /^[a-z0-9]+$/.match? imgtype
+        Tempfile.open(["image", ".#{imgtype}"]) do |f|
+          f.binmode
+          f.write(Base64.strict_decode64(imgdata))
+          f.path
+        end
+      end
+
+      SVG_NS = "http://www.w3.org/2000/svg".freeze
+
+      def svgmap_rewrite(xmldoc, localdirectory = "")
+        n = Namespace.new(xmldoc)
+        xmldoc.xpath(n.ns("//svgmap")).each do |s|
+          next unless svgmap_rewrite0(s, n, localdirectory)
+          next if s.at(n.ns("./target/eref"))
+
+          s.replace(s.at(n.ns("./figure")))
+        end
+      end
+
+      def svgmap_rewrite0(svgmap, namespace, localdirectory)
+        if (i = svgmap.at(namespace.ns(".//image"))) && (src = i["src"])
+          path = svgmap_rewrite0_path(src, localdirectory)
+          File.file?(path) or return false
+          svg = Nokogiri::XML(File.read(path, encoding: "utf-8"))
+          i.replace(svgmap_rewrite1(svgmap, svg, namespace))
+          /^data:/.match(src) and i["src"] = datauri(path)
+        elsif i = svgmap.at(".//m:svg", "m" => SVG_NS)
+          i.replace(svgmap_rewrite1(svgmap, i, namespace))
+        else
+          return false
+        end
+        true
+      end
+
+      def svgmap_rewrite0_path(src, localdirectory)
+        if /^data:/.match?(src)
+          save_dataimage(src)
+        else
+          File.file?(src) ? src : localdirectory + src
+        end
+      end
+
+      def svgmap_rewrite1(svgmap, svg, namespace)
+        targ = svgmap_rewrite1_targets(svgmap, namespace)
+        svg.xpath(".//m:a", "m" => SVG_NS).each do |a|
+          ["xlink:href", "href"].each do |p|
+            a[p] and x = targ[File.expand_path(a[p])] and a[p] = x
+          end
+        end
+        svg.to_xml
+      end
+
+      def svgmap_rewrite1_targets(svgmap, namespace)
+        svgmap.xpath(namespace.ns("./target"))
+          .each_with_object({}) do |t, m|
+          x = t.at(namespace.ns("./xref")) and
+            m[File.expand_path(t["href"])] = "##{x['target']}"
+          x = t.at(namespace.ns("./link")) and
+            m[File.expand_path(t["href"])] = x["target"]
+          t.remove if t.at(namespace.ns("./xref | ./link"))
+        end
+      end
+
+      # sources/plantuml/plantuml20200524-90467-1iqek5i.png
+      # already includes localdir
+      def datauri(uri, localdirectory = ".")
+        return uri if /^data:/.match?(uri)
+
+        path = datauri_path(uri, localdirectory)
+        return path unless File.exist?(path)
+
+        types = MIME::Types.type_for(path)
+        type = types ? types.first.to_s : 'text/plain; charset="utf-8"'
+        bin = File.open(path, "rb", &:read)
+        data = Base64.strict_encode64(bin)
+        "data:#{type};base64,#{data}"
+      end
+
+      def datauri_path(uri, localdirectory)
+        path = if %r{^([A-Z]:)?/}.match?(uri) then uri
+               else
+                 File.exist?(uri) ? uri : File.join(localdirectory, uri)
+               end
+        unless File.exist?(path)
+          warn "image at #{path} not found"
+          return uri
+        end
+        path
+      end
+
+      def datauri2mime(uri)
+        %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ uri
+        type = nil
+        imgtype = "png" unless /^[a-z0-9]+$/.match? imgtype
+        ::Tempfile.open(["imageuri", ".#{imgtype}"]) do |file|
+          type = datauri2mime1(file, imgdata)
+        end
+        [type]
+      end
+
+      def datauri2mime1(file, imgdata)
+        type = nil
+        begin
+          file.binmode
+          file.write(Base64.strict_decode64(imgdata))
+          file.rewind
+          type = Marcel::MimeType.for file
+        ensure
+          file.close!
+        end
+        type
+      end
+    end
+  end
+end

data/lib/utils/log.rb

@@ -7,48 +7,51 @@ module Metanorma
 
       def add(category, loc, msg)
         return if @novalid
+
         @log[category] = [] unless @log[category]
         @log[category] << { location: current_location(loc), message: msg,
                             context: context(loc) }
         loc = loc.nil? ? "" : "(#{current_location(loc)}): "
-        warn "#{category}: #{loc}#{msg}" 
+        warn "#{category}: #{loc}#{msg}"
       end
 
-      def current_location(n)
-        return "" if n.nil?
-        return n if n.is_a? String
-        return "Asciidoctor Line #{"%06d" % n.lineno}" if n.respond_to?(:lineno) &&
-          !n.lineno.nil? && !n.lineno.empty?
-        return "XML Line #{"%06d" % n.line}" if n.respond_to?(:line) &&
-          !n.line.nil?
-        return "ID #{n.id}" if n.respond_to?(:id) && !n.id.nil?
-        while !n.nil? &&
-            (!n.respond_to?(:level) || n.level.positive?) &&
-            (!n.respond_to?(:context) || n.context != :section)
-          n = n.parent
-          return "Section: #{n.title}" if n&.respond_to?(:context) &&
-            n&.context == :section
+      def current_location(node)
+        if node.nil? then ""
+        elsif node.is_a? String then node
+        elsif node.respond_to?(:lineno) && !node.lineno.nil? &&
+          !node.lineno.empty?
+          "Asciidoctor Line #{'%06d' % node.lineno}"
+        elsif node.respond_to?(:line) && !node.line.nil?
+          "XML Line #{'%06d' % node.line}"
+        elsif node.respond_to?(:id) && !node.id.nil? then "ID #{node.id}"
+        else
+          while !node.nil? &&
+              (!node.respond_to?(:level) || node.level.positive?) &&
+              (!node.respond_to?(:context) || node.context != :section)
+            node = node.parent
+            return "Section: #{node.title}" if node&.respond_to?(:context) &&
+              node&.context == :section
+          end
+          "??"
         end
-        "??"
       end
 
-      def context(n)
-        return nil if n.is_a? String
-        n.respond_to?(:to_xml) and return n.to_xml
-        n.respond_to?(:to_s) and return n.to_s
+      def context(node)
+        return nil if node.is_a? String
+
+        node.respond_to?(:to_xml) and return node.to_xml
+        node.respond_to?(:to_s) and return node.to_s
         nil
       end
 
       def write(file)
         File.open(file, "w:UTF-8") do |f|
           f.puts "#{file} errors"
-          @log.keys.each do |key|
+          @log.each_key do |key|
             f.puts "\n\n== #{key}\n\n"
-            @log[key].sort do |a, b|
-              a[:location] <=> b[:location]
-            end.each do |n|
+            @log[key].sort_by { |a| a[:location] }.each do |n|
               loc = n[:location] ? "(#{n[:location]}): " : ""
-              f.puts "#{loc}#{n[:message]}" 
+              f.puts "#{loc}#{n[:message]}"
               n[:context]&.split(/\n/)&.first(5)&.each { |l| f.puts "\t#{l}" }
             end
           end

data/lib/utils/main.rb

@@ -2,56 +2,61 @@ require "asciidoctor"
 require "tempfile"
 require "sterile"
 require "uuidtools"
-require "mimemagic"
-require "mime/types"
-require "base64"
 
 module Metanorma
   module Utils
-      NAMECHAR = "\u0000-\u0022\u0024\u002c\u002f\u003a-\u0040\\u005b-\u005e"\
-        "\u0060\u007b-\u00b6\u00b8-\u00bf\u00d7\u00f7\u037e\u2000-\u200b"\
-        "\u200e-\u203e\u2041-\u206f\u2190-\u2bff\u2ff0-\u3000".freeze
-      #"\ud800-\uf8ff\ufdd0-\ufdef\ufffe-\uffff".freeze
-      NAMESTARTCHAR = "\\u002d\u002e\u0030-\u0039\u00b7\u0300-\u036f"\
-        "\u203f-\u2040".freeze
+    NAMECHAR = "\u0000-\u0022\u0024\u002c\u002f\u003a-\u0040\\u005b-\u005e"\
+      "\u0060\u007b-\u00b6\u00b8-\u00bf\u00d7\u00f7\u037e\u2000-\u200b"\
+      "\u200e-\u203e\u2041-\u206f\u2190-\u2bff\u2ff0-\u3000".freeze
+    NAMESTARTCHAR = "\\u002d\u002e\u0030-\u0039\u00b7\u0300-\u036f"\
+      "\u203f-\u2040".freeze
 
     class << self
-      def to_ncname(s)
-        start = s[0]
-        ret1 = %r([#{NAMECHAR}#]).match(start) ? "_" :
-          (%r([#{NAMESTARTCHAR}#]).match(start) ? "_#{start}" : start)
-        ret2 = s[1..-1] || ""
-        ret = (ret1 || "") + ret2.gsub(%r([#{NAMECHAR}#]), "_")
-        ret
+      def to_ncname(tag)
+        start = tag[0]
+        ret1 = if %r([#{NAMECHAR}#]).match?(start)
+                 "_"
+               else
+                 (%r([#{NAMESTARTCHAR}#]).match?(start) ? "_#{start}" : start)
+               end
+        ret2 = tag[1..-1] || ""
+        (ret1 || "") + ret2.gsub(%r([#{NAMECHAR}#]), "_")
       end
 
       def anchor_or_uuid(node = nil)
         uuid = UUIDTools::UUID.random_create
-        node.nil? || node.id.nil? || node.id.empty? ? "_" + uuid : node.id
+        node.nil? || node.id.nil? || node.id.empty? ? "_#{uuid}" : node.id
       end
 
-      def asciidoc_sub(x)
-        return nil if x.nil?
-        return "" if x.empty?
-        d = Asciidoctor::Document.new(x.lines.entries, { header_footer: false, backend: :html })
+      def asciidoc_sub(text, flavour = :standoc)
+        return nil if text.nil?
+        return "" if text.empty?
+
+        d = Asciidoctor::Document.new(
+          text.lines.entries,
+          { header_footer: false, backend: flavour }
+        )
         b = d.parse.blocks.first
         b.apply_subs(b.source)
       end
 
       def localdir(node)
         docfile = node.attr("docfile")
-        docfile.nil? ? './' : Pathname.new(docfile).parent.to_s + '/'
+        docfile.nil? ? "./" : "#{Pathname.new(docfile).parent}/"
       end
 
       # TODO needs internationalisation
-      def smartformat(n)
-        n.gsub(/ --? /, "&#8201;&#8212;&#8201;").
-          gsub(/--/, "&#8212;").smart_format.gsub(/</, "&lt;").gsub(/>/, "&gt;")
+      def smartformat(text)
+        text.gsub(/ --? /, "&#8201;&#8212;&#8201;")
+          .gsub(/--/, "&#8212;").smart_format.gsub(/</, "&lt;")
+          .gsub(/>/, "&gt;")
       end
 
       def endash_date(elem)
         elem.traverse do |n|
-          n.text? and n.replace(n.text.gsub(/\s+--?\s+/, "&#8211;").gsub(/--/, "&#8211;"))
+          next unless n.text?
+
+          n.replace(n.text.gsub(/\s+--?\s+/, "&#8211;").gsub(/--/, "&#8211;"))
         end
       end
 
@@ -60,95 +65,34 @@ module Metanorma
       def set_nested_value(hash, keys, new_val)
         key = keys[0]
         if keys.length == 1
-          hash[key] = hash[key].is_a?(Array) ?  (hash[key] << new_val) :
-            hash[key].nil? ?  new_val : [hash[key], new_val]
+          hash[key] = if hash[key].is_a?(Array)
+                        (hash[key] << new_val)
+                      else
+                        hash[key].nil? ? new_val : [hash[key], new_val]
+                      end
+        elsif hash[key].is_a?(Array)
+          hash[key][-1] = {} if !hash[key].empty? && hash[key][-1].nil?
+          hash[key] << {} if hash[key].empty? || !hash[key][-1].is_a?(Hash)
+          set_nested_value(hash[key][-1], keys[1..-1], new_val)
+        elsif hash[key].nil? || hash[key].empty?
+          hash[key] = {}
+          set_nested_value(hash[key], keys[1..-1], new_val)
+        elsif hash[key].is_a?(Hash) && !hash[key][keys[1]]
+          set_nested_value(hash[key], keys[1..-1], new_val)
+        elsif !hash[key][keys[1]]
+          hash[key] = [hash[key], {}]
+          set_nested_value(hash[key][-1], keys[1..-1], new_val)
         else
-          if hash[key].is_a?(Array)
-            hash[key][-1] = {} if !hash[key].empty? && hash[key][-1].nil?
-            hash[key] << {} if hash[key].empty? || !hash[key][-1].is_a?(Hash)
-            set_nested_value(hash[key][-1], keys[1..-1], new_val)
-          elsif hash[key].nil? || hash[key].empty?
-            hash[key] = {}
-            set_nested_value(hash[key], keys[1..-1], new_val)
-          elsif hash[key].is_a?(Hash) && !hash[key][keys[1]]
-            set_nested_value(hash[key], keys[1..-1], new_val)
-          elsif !hash[key][keys[1]]
-            hash[key] = [hash[key], {}]
-            set_nested_value(hash[key][-1], keys[1..-1], new_val)
-          else
-            set_nested_value(hash[key], keys[1..-1], new_val)
-          end
+          set_nested_value(hash[key], keys[1..-1], new_val)
         end
         hash
       end
 
-      class Namespace
-        def initialize(xmldoc)
-          @namespace = xmldoc.root.namespace
-        end
-
-        def ns(path)
-          return path if @namespace.nil?
-          path.gsub(%r{/([a-zA-z])}, "/xmlns:\\1").
-            gsub(%r{::([a-zA-z])}, "::xmlns:\\1").
-            gsub(%r{\[([a-zA-z][a-z0-9A-Z@/]* ?=)}, "[xmlns:\\1").
-            gsub(%r{\[([a-zA-z][a-z0-9A-Z@/]*\])}, "[xmlns:\\1")
-        end
-      end
-
-      def save_dataimage(uri)
-        %r{^data:(image|application)/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ uri
-        imgtype.sub!(/\+[a-z0-9]+$/, '') # svg+xml
-        imgtype = 'png' unless /^[a-z0-9]+$/.match imgtype
-        Tempfile.open(['image', ".#{imgtype}"]) do |f|
-          f.binmode
-          f.write(Base64.strict_decode64(imgdata))
-          f.path
-        end
-      end
-
-      SVG_NS = "http://www.w3.org/2000/svg".freeze
-
-      def svgmap_rewrite(xmldoc, localdir = "")
-        n = Namespace.new(xmldoc)
-        xmldoc.xpath(n.ns("//svgmap")).each do |s|
-          next unless svgmap_rewrite0(s, n)
-          next if s.at(n.ns("./target/eref"))
-          s.replace(s.at(n.ns("./figure")))
-        end
-      end
-
-      def svgmap_rewrite0(s, n)
-        if i = s.at(n.ns(".//image")) and src = i["src"]
-          path = /^data:/.match(src) ? save_dataimage(src) : File.file?(src) ? src : localdir + src
-          File.file?(path) or return false
-          svgmap_rewrite1(s, Nokogiri::XML(File.read(path, encoding: "utf-8")), path, n)
-          /^data:/.match(src) and i["src"] = datauri(path)
-        elsif i = s.at(".//m:svg", "m" => SVG_NS)
-          svgmap_rewrite1(s, i, nil, n)
-        else
-          return false
-        end                 
-        true
-      end
-
-      def svgmap_rewrite1(s, svg, path, n)
-        targets = s.xpath(n.ns("./target")).each_with_object({}) do |t, m|
-          x = t.at(n.ns("./xref")) and m[File.expand_path(t["href"])] = "##{x['target']}"
-          x = t.at(n.ns("./link")) and m[File.expand_path(t["href"])] = x['target']
-          t.remove if t.at(n.ns("./xref | ./link"))
-        end
-        svg.xpath(".//m:a", "m" => SVG_NS).each do |a|
-          a["xlink:href"] and x = targets[File.expand_path(a["xlink:href"])] and a["xlink:href"] = x
-          a["href"] and x = targets[File.expand_path(a["href"])] and a["href"] = x
-        end
-        path and File.open(path, "w", encoding: "utf-8") { |f| f.write(svg.to_xml) }
-      end
-
       # not currently used
       def flatten_rawtext_lines(node, result)
         node.lines.each do |x|
-          if node.respond_to?(:context) && (node.context == :literal || node.context == :listing)
+          if node.respond_to?(:context) &&
+              (node.context == :literal || node.context == :listing)
             result << x.gsub(/</, "&lt;").gsub(/>/, "&gt;")
           else
             # strip not only HTML <tag>, and Asciidoc xrefs <<xref>>
@@ -174,42 +118,6 @@ module Metanorma
         end
         result.reject(&:empty?)
       end
-
-      # FIXME: nested uri path error(
-      #   sources/plantuml/plantuml20200524-90467-1iqek5i.png ->
-      #   sources/sources/plantuml/plantuml20200524-90467-1iqek5i.png)
-      def datauri(uri, localdir = ".")
-        return uri if /^data:/.match(uri)
-        path = %r{^([A-Z]:)?/}.match?(uri) ? uri : File.join(localdir, uri)
-        types = MIME::Types.type_for(path)
-        type = types ? types.first.to_s : 'text/plain; charset="utf-8"'
-        bin = File.open(path, 'rb', &:read)
-        data = Base64.strict_encode64(bin)
-        "data:#{type};base64,#{data}"
-      end
-
-      def datauri2mime(uri)
-        %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ uri
-        type = nil
-        imgtype = "png" unless /^[a-z0-9]+$/.match imgtype
-        ::Tempfile.open(["imageuri", ".#{imgtype}"]) do |file|
-          type = datauri2mime1(file, imgdata)
-        end
-        [type]
-      end
-
-      def datauri2mime1(file, imgdata)
-        type = nil
-        begin
-          file.binmode
-          file.write(Base64.strict_decode64(imgdata))
-          file.rewind
-          type = MimeMagic.by_magic(file)
-        ensure
-          file.close!
-        end
-        type
-      end
     end
   end
 end