metanorma-cli 1.3.3.1 → 1.3.4

data/gems/ruby/2.6.0/gems/relaton-cli-1.2.1/lib/relaton/cli.rb

@@ -0,0 +1,55 @@
+require "thor"
+require "relaton"
+require_relative "cli/command"
+
+module Relaton
+  def self.db
+    Cli.relaton
+  end
+
+  module Cli
+    class RelatonDb
+      include Singleton
+
+      def db
+        @db ||= Relaton::Db.new("#{Dir.home}/.relaton/cache", nil)
+      end
+    end
+
+    def self.start(arguments)
+      Relaton::Cli::Command.start(arguments)
+    end
+
+    # Relaton
+    #
+    # Based on current setup, we need to initiate a Db instance to
+    # register all of it's supported processor backends. To make it
+    # easier we have added it as a class method so we can use this
+    # whenever necessary.
+    #
+    def self.relaton
+      RelatonDb.instance.db
+    end
+
+    # @param content [Nokogiri::XML::Document]
+    # @return [RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliongraphicItem]
+    def self.parse_xml(doc)
+      if (proc = Cli.processor(doc))
+        proc.from_xml(doc.to_s)
+      else
+        RelatonBib::XMLParser.from_xml(doc.to_s)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String] Type prefix
+    def self.processor(doc)
+      docid = doc.at "docidentifier"
+      if docid && docid[:type]
+        proc = Relaton::Registry.instance.by_type(docid[:type])
+        return proc if proc
+      end
+      Relaton::Registry.instance.by_type(docid&.text&.match(/^\w+/)&.to_s)
+    end
+  end
+end

data/gems/ruby/2.6.0/gems/relaton-cli-1.2.1/lib/relaton/cli/base_convertor.rb

@@ -0,0 +1,141 @@
+require "fileutils"
+require "relaton/bibdata"
+require "relaton/bibcollection"
+require "relaton/cli/xml_to_html_renderer"
+
+module Relaton
+  module Cli
+    class BaseConvertor
+      def initialize(file, options = {})
+        @file = file
+        @options = options
+        @outdir = options.fetch(:outdir, nil)
+        @writable = options.fetch(:write, true)
+        @overwrite = options.fetch(:overwrite, true)
+        @default_filelabel = 0
+
+        install_dependencies(options[:require] || [])
+      end
+
+      # @return [String] HTML
+      def to_html
+        content = convert_to_html
+        write_to_a_file(content)
+      end
+
+      # Convert to HTML
+      #
+      # This interface expect us to provide Relaton collection XML
+      # as XML/RXL, and necessary styels / templates then it will be
+      # used convert that collection to HTML.
+      #
+      # @param file [String] Relaton collection file path
+      # @param style [String] Stylesheet file path for styles
+      # @param template [String] The liquid tempalte directory
+      #
+      # @return [String] HTML
+      def self.to_html(file, style = nil, template = nil)
+        new(
+          file,
+          style: style || File.join(File.dirname(__FILE__), "../../../templates/index-style.css"),
+          template: template || File.join(File.dirname(__FILE__), "../../../templates/"),
+          extension: "html",
+        ).to_html
+      end
+
+      private
+
+      attr_reader :file, :outdir, :options, :writable, :overwrite
+
+      # @return [String] HTML
+      def convert_to_html
+        Relaton::Cli::XmlToHtmlRenderer.render(
+          xml_content(file),
+          stylesheet: options[:style],
+          liquid_dir: options[:template],
+        )
+      end
+
+      # @param file [String] path to a file
+      # @return [String] the file's content
+      # @return [String] HTML
+      def xml_content(file)
+        File.read(file, encoding: "utf-8")
+      end
+
+      def install_dependencies(dependencies)
+        dependencies.each { |dependency| require(dependency) }
+      end
+
+      def item_output(content, format)
+        case format.to_sym
+        when :to_yaml then content.to_yaml
+        when :to_xml then content.to_xml(date_format: :full, bibdata: true)
+        end
+      end
+
+      def convert_and_write(content, format)
+        content = convert_content(content)
+        write_to_a_file(item_output(content, format))
+        write_to_file_collection(content, format.to_sym)
+      end
+
+      def write_to_a_file(content, outfile = nil)
+        outfile ||= Pathname.new(file).sub_ext(extension).to_s
+
+        if !File.exists?(outfile) || overwrite
+          File.open(outfile, "w:utf-8") do |file|
+            file.write(content)
+          end
+        end
+      end
+
+      def write_to_file_collection(content, format)
+        # if outdir && (content.is_a?(Relaton::Bibcollection))
+        #   FileUtils.mkdir_p(outdir)
+        #   content.items_flattened.each do |item|
+        #     collection = collection_filename(item.docidentifier_code)
+        #     write_to_a_file(item_output(item, format), collection)
+        #   end
+        # end
+        if outdir && content.is_a?(Relaton::Bibcollection)
+          FileUtils.mkdir_p(outdir)
+          content.items_flattened.each do |item|
+            collection = collection_filename(extract_docid(item))
+            write_to_a_file(item_output(item, format), collection)
+          end
+        end
+      end
+
+      def extract_docid(item)
+        @default_filelabel += 1
+        item.docidentifier.nil? && (return @default_filelabel.to_s)
+        # item.docidentifier.is_a?(Array) or return @default_filelabel.to_s
+        item.docidentifier.empty? && (return @default_filelabel.to_s)
+        docidentifier_code(item.docidentifier)
+      end
+
+      # From http://gavinmiller.io/2016/creating-a-secure-sanitization-function/
+      FILENAME_BAD_CHARS = ["/", '\\', "?", '%", "*", ":', "|", '"', "<", ">",
+                            ".", " "].freeze
+
+      def docidentifier_code(docidentifier)
+        return "" if docidentifier.nil?
+
+        FILENAME_BAD_CHARS.reduce(docidentifier.downcase) do |result, bad_char|
+          result.gsub(bad_char, "-")
+        end
+      end
+
+      def extension
+        @extension ||= [".", options.fetch(:extension, default_ext)].join
+      end
+
+      def collection_filename(identifier)
+        File.join(
+          outdir, [@options[:prefix], identifier, extension].compact.join("")
+        )
+      end
+    end
+  end
+end

data/gems/ruby/2.6.0/gems/relaton-cli-1.2.1/lib/relaton/cli/command.rb

@@ -0,0 +1,115 @@
+require "relaton/cli/relaton_file"
+require "relaton/cli/xml_convertor"
+require "relaton/cli/yaml_convertor"
+require "fcntl"
+
+module Relaton
+  module Cli
+    class Command < Thor
+      desc "fetch CODE", "Fetch Relaton XML for Standard identifier CODE"
+      option :type, aliases: :t, required: true, desc: "Type of standard to get bibliographic entry for"
+      option :format, aliases: :f, desc: "Output format (xml, bibtex). Default xml."
+      option :year, aliases: :y, type: :numeric, desc: "Year the standard was published"
+
+      def fetch(code)
+        Relaton.db
+        io = IO.new(STDOUT.fcntl(::Fcntl::F_DUPFD), mode: 'w:UTF-8')
+        io.puts(fetch_document(code, options) || supported_type_message)
+      end
+
+      desc "extract Metanorma-XML-File / Directory Relaton-XML-Directory", "Extract Relaton XML from Metanorma XML file / directory"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, defaults to 'rxl'"
+
+      def extract(source_dir, outdir)
+        Relaton::Cli::RelatonFile.extract(source_dir, outdir, options)
+      end
+
+      desc "concatenate SOURCE-DIR COLLECTION-FILE", "Concatenate entries in DIRECTORY (containing Relaton-XML or YAML) into a Relaton Collection"
+      option :title, aliases: :t,  desc: "Title of resulting Relaton collection"
+      option :organization, aliases: :g, desc: "Organization owner of Relaton collection"
+      option :new, aliases: :n, type: :boolean, desc: "Use the new Relaton YAML format"
+      option :extension, aliases: :x, desc: "File extension of destination Relaton file, defaults to 'rxl'"
+
+      def concatenate(source_dir, outfile)
+        Relaton::Cli::RelatonFile.concatenate(source_dir, outfile, options)
+      end
+
+      desc "split Relaton-Collection-File Relaton-XML-Directory", "Split a Relaton Collection into multiple files"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, defaults to 'rxl'"
+      option :new, aliases: :n, type: :boolean, desc: "Use the new Relaton YAML format"
+
+      def split(source, outdir)
+        Relaton::Cli::RelatonFile.split(source, outdir, options)
+      end
+
+      desc "yaml2xml YAML", "Convert Relaton YAML into Relaton Collection XML or separate files"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, defaults to 'rxl'"
+      option :prefix, aliases: :p, desc: "Filename prefix of individual Relaton XML files, defaults to empty"
+      option :outdir, aliases: :o,  desc: "Output to the specified directory with individual Relaton Bibdata XML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY prior to execution"
+      option :overwrite, aliases: :f, type: :boolean, default: false, desc: "Overwrite the existing file"
+
+      def yaml2xml(filename)
+        Relaton::Cli::YAMLConvertor.to_xml(filename, options)
+      end
+
+      desc "xml2yamlnew XML", "Convert Relaton XML into Relaton Bibdata / Bibcollection YAML (and separate files)"
+      option :extension, aliases: :x, default: "yaml", desc: "File extension of Relaton YAML files, defaults to 'yaml'"
+      option :prefix, aliases: :p, desc: "Filename prefix of Relaton XML files, defaults to empty"
+      option :outdir, aliases: :o, desc: "Output to the specified directory with individual Relaton Bibdata YAML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY prior to execution"
+      option :overwrite, aliases: :f, type: :boolean, default: false, desc: "Overwrite the existing file"
+
+      def xml2yaml(filename)
+        Relaton::Cli::XMLConvertor.to_yaml(filename, options)
+      end
+
+      desc "xml2html RELATON-INDEX-XML", "Convert Relaton Collection XML into HTML"
+      option :stylesheet, aliases: :s, desc: "Stylesheet file path for rendering HTML index"
+      option :templatedir, aliases: :t, desc: "Liquid template directory for rendering Relaton items and collection"
+      option :overwrite, aliases: :f, type: :boolean, default: false, desc: "Overwrite the existing file"
+
+      def xml2html(file, style = nil, template = nil)
+        Relaton::Cli::XMLConvertor.to_html(file, style, template)
+      end
+
+      desc "yaml2html RELATON-INDEX-YAML", "Concatenate Relaton Collection YAML into HTML"
+      option :stylesheet, aliases: :s, desc: "Stylesheet file path for rendering HTML index"
+      option :templatedir, aliases: :t, desc: "Liquid template directory for rendering Relaton items and collection"
+      option :overwrite, aliases: :f, type: :boolean, default: false, desc: "Overwrite the existing file"
+
+      def yaml2html(file, style = nil, template = nil)
+        Relaton::Cli::YAMLConvertor.to_html(file, style, template)
+      end
+
+      private
+
+      # @param code [String]
+      # @param options [Hash]
+      # @option options [String] :type
+      # @option options [String, NilClass] :format
+      # @option options [Integer, NilClass] :year
+      def fetch_document(code, options)
+        if registered_types.include?(options[:type])
+          doc = Cli.relaton.fetch(code, options[:year]&.to_s)
+          if doc
+            options[:format] == "bibtex" ? doc.to_bibtex : doc.to_xml
+          else
+            "No matching bibliographic entry found"
+          end
+        end
+      rescue RelatonBib::RequestError => e
+        e.message
+      end
+
+      def supported_type_message
+        ["Recognised types:", registered_types.sort.join(", ")].join(" ")
+      end
+
+      def registered_types
+        @registered_types ||=
+          Relaton::Registry.instance.processors.each.map { |_n, pr| pr.prefix }
+      end
+    end
+  end
+end

data/gems/ruby/2.6.0/gems/relaton-cli-1.2.1/lib/relaton/cli/relaton_file.rb

@@ -0,0 +1,276 @@
+require "nokogiri"
+require "pathname"
+
+module Relaton
+  module Cli
+    class RelatonFile
+      def initialize(source, options = {})
+        @source = source
+        @options = options
+        @outdir = options.fetch(:outdir, nil)
+        @outfile = options.fetch(:outfile, nil)
+      end
+
+      def extract
+        extract_and_write_to_files
+      end
+
+      def concatenate
+        concatenate_and_write_to_files
+      end
+
+      def split
+        split_and_write_to_files
+      end
+
+      # Extract files
+      #
+      # This interface expect us to provide a source file / directory,
+      # output directory and custom configuration options. Then it wll
+      # extract Relaton XML file / files to output directory from the
+      # source file / directory. During this process it will use custom
+      # options when available.
+      #
+      # @param source [Dir] The source directory for files
+      # @param outdir [Dir] The output directory for files
+      # @param options [Hash] Options as hash key value pair
+      #
+      def self.extract(source, outdir, options = {})
+        new(source, options.merge(outdir: outdir)).extract
+      end
+
+      # Concatenate files
+      #
+      # This interface expect us to provide a source directory, output
+      # file and custom configuration options. Normally, this expect the
+      # source directory to contain RXL fles, but it also converts any
+      # YAML files to RXL and then finally combines those together.
+      #
+      # This interface also allow us to provdie options like title and
+      # organization and then it usage those details to generate the
+      # collection file.
+      #
+      # @param source [Dir] The source directory for files
+      # @param output [String] The collection output file
+      # @param options [Hash] Options as hash key value pair
+      #
+      def self.concatenate(source, outfile, options = {})
+        new(source, options.merge(outfile: outfile)).concatenate
+      end
+
+      # Split collection
+      #
+      # This interface expects us to provide a Relaton Collection
+      # file and also an output directory, then it will split that
+      # collection into multiple files.
+      #
+      # By default it usages `rxl` extension for these new files,
+      # but we can also customize that by providing the correct
+      # one as `extension` option parameter.
+      #
+      # @param source [File] The source collection file
+      # @param output [Dir] The output directory for files
+      # @param options [Hash] Options as hash key value pair
+      #
+      def self.split(source, outdir = nil, options = {})
+        new(source, options.merge(outdir: outdir)).split
+      end
+
+      private
+
+      attr_reader :source, :options, :outdir, :outfile
+
+      def bibcollection
+        Bibcollection.new(
+          title: options[:title],
+          items: concatenate_files,
+          doctype: options[:doctype],
+          author: options[:organization],
+        )
+      end
+
+      # @param document [String] XML
+      # @param file [String, nil] path to file
+      # @return [Nokogiri::XML::Document]
+      def nokogiri_document(document, file = nil)
+        document ||= File.read(file, encoding: "utf-8")
+        Nokogiri.XML(document)
+      end
+
+      def select_source_files
+        if File.file?(source)
+          [source]
+        else
+          select_files_with("xml")
+        end
+      end
+
+      def relaton_collection
+        @relaton_collection ||= Bibcollection.from_xml(nokogiri_document(nil, source))
+      end
+
+      def extract_and_write_to_files
+        select_source_files.each do |file|
+          xml = nokogiri_document(nil, file)
+          xml.remove_namespaces!
+
+          if (bib = xml.at("//bibdata"))
+            bib = nokogiri_document(bib.to_xml)
+          elsif (rfc = xml.at("//rfc"))
+            require "relaton_ietf/scrapper"
+            ietf = RelatonIetf::Scrapper.fetch_rfc rfc
+            bib = nokogiri_document ietf.to_xml(bibdata: true)
+          else
+            next
+          end
+
+          bib.remove_namespaces!
+          bib.root.add_namespace(nil, "xmlns")
+
+          bibdata = Relaton::Bibdata.from_xml(bib.root)
+          if bibdata
+            build_bibdata_relaton(bibdata, file)
+
+            write_to_file(bibdata.send(output_type), outdir, build_filename(file))
+          end
+        end
+      end
+
+      def concatenate_files
+        xml_files = [convert_rxl_to_xml, convert_yamls_to_xml, convert_xml_to_xml]
+
+        xml_files.flatten.reduce([]) do |mem, xml|
+          doc = nokogiri_document(xml[:content])
+          if (rfc = doc.at("/rfc"))
+            require "relaton_ietf/scrapper"
+            ietf = RelatonIetf::Scrapper.fetch_rfc rfc
+            d = nokogiri_document ietf.to_xml(bibdata: true)
+            mem << bibdata_instance(d, xml[:file])
+          elsif %w[bibitem bibdata].include? doc&.root&.name
+            mem << bibdata_instance(doc, xml[:file])
+          else mem
+          end
+        end.uniq &:id
+      end
+
+      def concatenate_and_write_to_files
+        write_to_file(bibcollection.send(output_type))
+      end
+
+      def split_and_write_to_files
+        output_dir = outdir || build_dirname(source)
+
+        relaton_collection.items.each do |content|
+          name = build_filename(nil, content.docidentifier)
+          find_available_bibrxl_file(name, output_dir, content)
+          write_to_file(content.send(output_type), output_dir, name)
+        end
+      end
+
+      def find_available_bibrxl_file(name, _ouputdir, content)
+        if options[:extension] == "yaml" || options[:extension] == "yml"
+          bib_rxl = Pathname.new([outdir, name].join("/")).sub_ext(".rxl")
+          content.bib_rxl = bib_rxl.to_s if File.file?(bib_rxl)
+        end
+      end
+
+      def output_type(ext = options[:extension])
+        ext ||= File.extname(outfile)[1..-1] if outfile
+        case ext
+        when "rxl", "xml"
+          :to_xml
+        when "yml", "yaml"
+          :to_yaml
+        else
+          puts "[relaton-cli] the given extension of '#{ext}' is "\
+          "not supported. Use 'rxl'."
+          :to_xml
+        end
+      end
+
+      # @param document [Nokogiri::XML::Document]
+      # @param file [String] path to file
+      # @return [Relaton::Bibdata]
+      def bibdata_instance(document, file)
+        document = clean_nokogiri_document(document)
+        bibdata = Relaton::Bibdata.from_xml document.root
+        build_bibdata_relaton(bibdata, file) if bibdata
+
+        bibdata
+      end
+
+      # @param bibdata [Relaton::Bibdata]
+      # @param file [String] path to file
+      def build_bibdata_relaton(bibdata, file)
+        ["xml", "pdf", "doc", "html", "rxl", "txt"].each do |type|
+          file = Pathname.new(file).sub_ext(".#{type}")
+          bibdata.send("#{type}=", file.to_s) if File.file?(file)
+        end
+      end
+
+      # Force a namespace otherwise Nokogiri won't parse.
+      # The reason is we use Bibcollection's from_xml, but that one
+      # has an xmlns. We don't want to change the code for bibdata
+      # hence this hack #bibdata_doc.root['xmlns'] = "xmlns"
+      #
+      # @param document [Nokogiri::XML::Document]
+      # @return [Nokogiri::XML::Document]
+      def clean_nokogiri_document(document)
+        document.remove_namespaces!
+        document.root.add_namespace(nil, "xmlns")
+        nokogiri_document(document.to_xml)
+      end
+
+      def convert_rxl_to_xml
+        select_files_with("{rxl}").map do |file|
+          { file: file, content: File.read(file, encoding: "utf-8") }
+        end
+      end
+
+      def convert_yamls_to_xml
+        select_files_with("yaml").map do |file|
+          { file: file, content: YAMLConvertor.to_xml(file, write: false) }
+        end
+      end
+
+      def convert_xml_to_xml
+        select_files_with("{xml}").map do |file|
+          { file: file, content: File.read(file, encoding: "utf-8") }
+        end
+      end
+
+      def select_files_with(extension)
+        files = File.join(source, "**", "*.#{extension}")
+        Dir[files].reject { |file| File.directory?(file) }
+      end
+
+      def write_to_file(content, directory = nil, output_file = nil)
+        file_with_dir = [directory, output_file || outfile].compact.join("/")
+        File.open(file_with_dir, "w:utf-8") { |file| file.write(content) }
+      end
+
+      def build_dirname(filename)
+        basename = File.basename(filename)&.gsub(/.(xml|rxl)/, "")
+        directory_name = sanitize_string(basename)
+        Dir.mkdir(directory_name) unless File.exists?(directory_name)
+
+        directory_name
+      end
+
+      def build_filename(file, identifier = nil, ext = "rxl")
+        identifier ||= Pathname.new(File.basename(file.to_s, ".xml")).to_s
+        [sanitize_string(identifier), options[:extension] || ext].join(".")
+      end
+
+      def sanitize_string(string)
+        clean_string = replace_bad_characters(string.downcase)
+        clean_string.gsub(/^\s+/, "").gsub(/\s+$/, "").gsub(/\s+/, "-")
+      end
+
+      def replace_bad_characters(string)
+        bad_chars = ["/", "\\", "?", "%", "*", ":", "|", '"', "<", ">", ".", " "]
+        bad_chars.reduce(string.downcase) { |res, char| res.gsub(char, "-") }
+      end
+    end
+  end
+end