relaton-cli 0.1.2 → 0.1.4

data/lib/relaton/bibdata.rb

@@ -1,8 +1,9 @@
+require "date"
 
 module Relaton
   class Bibdata
     ATTRIBS = %i[
-      docid
+      docidentifier
       doctype
       title
       stage
@@ -16,6 +17,16 @@ module Relaton
       revdate
       abstract
       technical_committee
+      copyright_from
+      copyright_owner
+      contributor_author_role
+      contributor_author_organization
+      contributor_publisher_role
+      contributor_publisher_organization
+      language
+      script
+      edition
+      datetype
     ]
 
     attr_accessor *ATTRIBS
@@ -31,68 +42,120 @@ module Relaton
       options.each_pair do |k,v|
         send("#{k.to_s}=", v)
       end
+      self
+    end
 
-      puts "*+"*30
-      puts self.inspect
+    # From http://gavinmiller.io/2016/creating-a-secure-sanitization-function/
+    FILENAME_BAD_CHARS = [ '/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ' ]
 
-      self
+    def docidentifier_code
+      return "" if docidentifier.nil?
+      a = FILENAME_BAD_CHARS.inject(docidentifier.downcase) do |result, bad_char|
+        result.gsub(bad_char, '-')
+      end
     end
 
-    def docid_code
-      docid.downcase.gsub(/[\s\/]/, "-") || ""
+    DOC_NUMBER_REGEX = /([\w\/]+)\s+(\d+):?(\d*)/
+    def doc_number
+      docidentifier&.match(DOC_NUMBER_REGEX) ? $2.to_i : 999999
     end
 
     def self.from_xml(source)
 
       # bib.relaton_xml_path = URI.escape("#{relaton_root}/#{id_code}.xml")
-
-      datetype = source.at(ns("./date[@type]")).text
-      revdate = source.at(ns("./date/on")).text
+      revdate = source.at(ns("./date[@type = 'published']")) ||
+        source.at(ns("./date[@type = 'circulated']")) || source.at(ns("./date"))
+      datetype = "circulated"
+      datetype = revdate["type"] if revdate
 
       new({
-        uri: source.at(ns("./uri"))&.text,
+        uri: source.at(ns("./uri[not(@type)]"))&.text,
         xml: source.at(ns("./uri[@type='xml']"))&.text,
         pdf: source.at(ns("./uri[@type='pdf']"))&.text,
         html: source.at(ns("./uri[@type='html']"))&.text,
         relaton: source.at(ns("./uri[@type='relaton']"))&.text,
         doc: source.at(ns("./uri[@type='doc']"))&.text,
-        docid: source.at(ns("./docidentifier"))&.text,
+        docidentifier: source.at(ns("./docidentifier"))&.text,
         title: source.at(ns("./title"))&.text,
         doctype: source.at(ns("./@type"))&.text,
         stage: source.at(ns("./status"))&.text,
-        technical_committee: source.at(ns("./technical-committee"))&.text,
+        technical_committee: source.at(ns("./editorialgroup/technical-committee"))&.text,
         abstract: source.at(ns("./abstract"))&.text,
-        revdate: Date.parse(revdate)
-        # revdate TODO
+        revdate: revdate ? Date.parse(revdate.text) : nil,
+        language: source.at(ns("./language"))&.text,
+        script: source.at(ns("./script"))&.text,
+        edition: source.at(ns("./edition"))&.text,
+        copyright_from: source.at(ns("./copyright/from"))&.text,
+        copyright_owner: source.at(ns("./copyright/owner/organization/name"))&.text,
+        contributor_author_role: source.at(ns("./contributor/role[@type='author']"))&.text,
+        contributor_author_organization: source.at(ns("./contributor/role[@type='author']"))&.parent&.at(ns("./organization/name"))&.text,
+        contributor_publisher_role: source.at(ns("./contributor/role[@type='publisher']"))&.text,
+        contributor_publisher_organization: source.at(ns("./contributor/role[@type='publisher']"))&.parent&.at(ns("./organization/name"))&.text,
+        datetype: datetype
       })
     end
 
     def to_xml
-      datetype = stage.casecmp("published") == 0 ? "published" : "updated"
+      #datetype = stage&.casecmp("published") == 0 ? "published" : "circulated"
 
       ret = "<bibdata type='#{doctype}'>\n"
+      ret += "<fetched>#{Date.today.to_s}</fetched>\n"
       ret += "<title>#{title}</title>\n"
+      ret += "<docidentifier>#{docidentifier}</docidentifier>\n" if docidentifier
       ret += "<uri>#{uri}</uri>\n" if uri
       ret += "<uri type='xml'>#{xml}</uri>\n" if xml
       ret += "<uri type='html'>#{html}</uri>\n" if html
       ret += "<uri type='pdf'>#{pdf}</uri>\n" if pdf
       ret += "<uri type='doc'>#{doc}</uri>\n" if doc
       ret += "<uri type='relaton'>#{relaton}</uri>\n" if relaton
-      ret += "<docidentifier>#{docid}</docidentifier>\n"
+
+      ret += "<language>#{language}</language>\n"
+      ret += "<script>#{script}</script>\n"
+
+      if copyright_from
+        ret += "<copyright>"
+        ret += "<from>#{copyright_from}</from>\n" if copyright_from
+        ret += "<owner><organization><name>#{copyright_owner}</name></organization></owner>\n" if copyright_owner
+        ret += "</copyright>"
+      end
+
+      if contributor_author_role
+        ret += "<contributor>\n"
+        ret += "<role type='author'/>\n"
+        ret += "<organization><name>#{contributor_author_organization}</name></organization>\n"
+        ret += "</contributor>\n"
+      end
+
+      if contributor_publisher_role
+        ret += "<contributor>\n"
+        ret += "<role type='publisher'/>\n"
+        ret += "<organization><name>#{contributor_publisher_organization}</name></organization>\n"
+        ret += "</contributor>\n"
+      end
+
       ret += "<date type='#{datetype}'><on>#{revdate}</on></date>\n" if revdate
+      # ret += "<contributor><role type='author'/><organization><name>#{agency}</name></organization></contributor>" if agency
+      # ret += "<contributor><role type='publisher'/><organization><name>#{agency}</name></organization></contributor>" if agency
+      ret += "<edition>#{edition}</edition>\n" if edition
+      ret += "<language>#{language}</language>\n" if language
+      ret += "<script>#{script}</script>\n" if script
       ret += "<abstract>#{abstract}</abstract>\n" if abstract
       ret += "<status>#{stage}</status>\n" if stage
-      ret += "<technical-committee>#{technical_committee}</technical-committee>\n" if technical_committee
+      ret += "<editorialgroup><technical-committee>#{technical_committee}</technical-committee></editorialgroup>\n" if technical_committee
       ret += "</bibdata>\n"
     end
 
     def to_h
       ATTRIBS.inject({}) do |acc, k|
         value = send(k)
-        acc[k] = value unless value.nil?
+        acc[k.to_s] = value unless value.nil?
         acc
       end
     end
 
+    def to_yaml
+      to_h.to_yaml
+    end
+
   end
 end

data/lib/relaton/cli/_document.liquid

@@ -8,9 +8,9 @@
     <div class="doc-identifier">
       <h{{ depth }}>
       {% if document.html == "" %}
-        {{ document.docid }}
+        {{ document.docidentifier }}
       {% else %}
-        <a href="{{ document.html }}">{{ document.docid }}</a>
+        <a href="{{ document.html }}">{{ document.docidentifier }}</a>
       {% endif %}
       </h{{ depth }}>
     </div>

data/lib/relaton/cli/base_convertor.rb

@@ -0,0 +1,94 @@
+require "fileutils"
+require "relaton/bibdata"
+require "relaton/bibcollection"
+require "relaton/cli/xml_to_html_renderer"
+
+module Relaton
+  module Cli
+    class BaseConvertor
+      def initialize(file, options = {})
+        @file = file
+        @options = options
+        @outdir = options.fetch(:outdir, nil)
+        @writable = options.fetch(:write, true)
+
+        install_dependencies(options[:require] || [])
+      end
+
+      def to_html
+        content = convert_to_html
+        write_to_a_file(content)
+      end
+
+      # Convert to HTML
+      #
+      # This interface expect us to provide Relaton collection XML
+      # as XML/RXL, and necessary styels / templates then it will be
+      # used convert that collection to HTML.
+      #
+      # @param file [String] Relaton collection file path
+      # @param style [String] Stylesheet file path for styles
+      # @param template [String] The liquid tempalte directory
+      #
+      def self.to_html(file, style, template)
+        new(file, style: style, template: template, extension: "html").to_html
+      end
+
+      private
+
+      attr_reader :file, :outdir, :options, :writable
+
+      def default_ext
+        raise "Override this method"
+      end
+
+      def convert_to_html
+        Relaton::Cli::XmlToHtmlRenderer.render(
+          xml_content(file),
+          stylesheet: options[:style],
+          liquid_dir: options[:template],
+        )
+      end
+
+      def xml_content(file)
+        File.read(file, encoding: "utf-8")
+      end
+
+      def install_dependencies(dependencies)
+        dependencies.each { |dependency| require(dependency) }
+      end
+
+      def convert_and_write(content, format)
+        content = convert_content(content)
+        write_to_a_file(content.send(format.to_sym))
+        write_to_file_collection(content, format.to_sym)
+      end
+
+      def write_to_a_file(content, outfile = nil)
+        outfile ||= Pathname.new(file).sub_ext(extension).to_s
+        File.open(outfile, "w:utf-8") { |file| file.write(content) }
+      end
+
+      def write_to_file_collection(content, format)
+        if outdir && content.is_a?(Relaton::Bibcollection)
+          FileUtils.mkdir_p(outdir)
+
+          content.items_flattened.each do |item|
+            collection = collection_filename(item.docidentifier_code)
+            write_to_a_file(item.send(format.to_sym), collection)
+          end
+        end
+      end
+
+      def extension
+        @extension ||= [".", options.fetch(:extension, default_ext)].join
+      end
+
+      def collection_filename(identifier)
+        File.join(
+          outdir, [@options[:prefix], identifier, extension].compact.join("")
+        )
+      end
+    end
+  end
+end

data/lib/relaton/cli/command.rb

@@ -0,0 +1,83 @@
+require "relaton/cli/relaton_file"
+require "relaton/cli/xml_convertor"
+require "relaton/cli/yaml_convertor"
+
+module Relaton
+  module Cli
+    class Command < Thor
+      desc "fetch CODE", "Fetch Relaton XML for Standard identifier CODE"
+      option :type, aliases: :t, required: true, desc: "Type of standard to get bibliographic entry for"
+      option :year, aliases: :y, type: :numeric, desc: "Year the standard was published"
+
+      def fetch(code)
+        Relaton::Cli.relaton
+        say(fetch_document(code, options) || supported_type_message)
+      end
+
+      desc "extract Metanorma-XML-Directory Relaton-XML-Directory", "Extract Relaton XML from folder of Metanorma XML"
+      option :extension, aliases: :x, desc: "File extension of Relaton XML files, defaults to 'rxl'"
+
+      def extract(source_dir, outdir)
+        Relaton::Cli::RelatonFile.extract(source_dir, outdir, options)
+      end
+
+      desc "concatenate SOURCE-DIR COLLECTION-FILE", "Concatenate entries in DIRECTORY (containing Relaton-XML or YAML) into a Relaton Collection"
+      option :title, aliases: :t,  desc: "Title of resulting Relaton collection"
+      option :organization, aliases: :g, desc: "Organization owner of Relaton collection"
+
+      def concatenate(source_dir, outfile)
+        Relaton::Cli::RelatonFile.concatenate(source_dir, outfile, options)
+      end
+
+      desc "yaml2xml YAML", "Convert Relaton YAML into Relaton Collection XML or separate files"
+      option :extension, aliases: :x, desc: "File extension of Relaton XML files, defaults to 'rxl'"
+      option :prefix, aliases: :p, desc: "Filename prefix of individual Relaton XML files, defaults to empty"
+      option :outdir, aliases: :o,  desc: "Output to the specified directory with individual Relaton Bibdata XML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY prior to execution"
+
+      def yaml2xml(filename)
+        Relaton::Cli::YAMLConvertor.to_xml(filename, options)
+      end
+
+      desc "xml2yaml XML", "Convert Relaton YAML into Relaton Bibcollection YAML (and separate files)"
+      option :extension, aliases: :x, desc: "File extension of Relaton YAML files, defaults to 'yaml'"
+      option :prefix, aliases: :p, desc: "Filename prefix of Relaton XML files, defaults to empty"
+      option :outdir, aliases: :o, desc: "Output to the specified directory with individual Relaton Bibdata YAML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY prior to execution"
+
+      def xml2yaml(filename)
+        Relaton::Cli::XMLConvertor.to_yaml(filename, options)
+      end
+
+      desc "xml2html RELATON-INDEX-XML STYLESHEET LIQUID-TEMPLATE-DIR", "Convert Relaton Collection XML into HTML"
+
+      def xml2html(file, style, template)
+        Relaton::Cli::XMLConvertor.to_html(file, style, template)
+      end
+
+      desc "yaml2html YAML STYLESHEET LIQUID-TEMPLATE-DIR", "Concatenate Relaton YAML into HTML"
+
+      def yaml2html(file, style, template)
+        Relaton::Cli::YAMLConvertor.to_html(file, style, template)
+      end
+
+      private
+
+      def fetch_document(code, options)
+        if registered_types.include?(options[:type])
+          doc = Cli.relaton.fetch_std(code, options[:year], options[:type])
+          doc ? doc.to_xml : "No matching bibliographic entry found"
+        end
+      end
+
+      def supported_type_message
+        ["Recognised types:", registered_types.sort.join(", ")].join(" ")
+      end
+
+      def registered_types
+        @registered_types ||=
+          Relaton::Registry.instance.processors.each.map { |_n, pr| pr.prefix }
+      end
+    end
+  end
+end

data/lib/relaton/cli/relaton_file.rb

@@ -0,0 +1,153 @@
+require "nokogiri"
+require "pathname"
+
+module Relaton
+  module Cli
+    class RelatonFile
+      def initialize(source, options = {})
+        @source = source
+        @options = options
+        @outdir = options.fetch(:outdir, nil)
+        @outfile = options.fetch(:outfile, nil)
+      end
+
+      def extract
+        extract_and_write_to_files
+      end
+
+      def concatenate
+        write_to_file(bibcollection.to_xml)
+      end
+
+      # Extract files
+      #
+      # This interface expect us to provide a source directory, output
+      # directory and custom configuration options. Then it wll extract
+      # Relaton XML files to output directory from the source directory
+      # During this process it will use custom options when available.
+      #
+      # @param source [Dir] The source directory for files
+      # @param outdir [Dir] The output directory for files
+      # @param options [Hash] Options as hash key value pair
+      #
+      def self.extract(source, outdir, options = {})
+        new(source, options.merge(outdir: outdir)).extract
+      end
+
+      # Concatenate files
+      #
+      ## This interface expect us to provide a source directory, output
+      # file and custom configuration options. Normally, this expect the
+      # source directory to contain RXL fles, but it also converts any
+      # YAML files to RXL and then finally combines those together.
+      #
+      # This interface also allow us to provdie options like title and
+      # organization and then it usage those details to generate the
+      # collection file.
+      #
+      # @param source [Dir] The source directory for files
+      # @param output [String] The collection output file
+      # @param options [Hash] Options as hash key value pair
+      #
+      def self.concatenate(source, outfile, options = {})
+        new(source, options.merge(outfile: outfile)).concatenate
+      end
+
+      private
+
+      attr_reader :source, :options, :outdir, :outfile
+
+      def bibcollection
+        ::Relaton::Bibcollection.new(
+          title: options[:title],
+          items: concatenate_files,
+          doctype: options[:doctype],
+          author: options[:organization],
+        )
+      end
+
+      def nokogiri_document(document, file = nil)
+        document ||= File.read(file, encoding: "utf-8")
+        Nokogiri.XML(document)
+      end
+
+      def extract_and_write_to_files
+        select_files_with("xml").each do |file|
+          xml = nokogiri_document(nil, file)
+          xml.remove_namespaces!
+
+          bib = xml.at("//bibdata") || next
+          bib.add_namespace(nil, "")
+
+          outfile = [outdir, build_filename(file, bib)].join("/")
+          write_to_file(bib.to_xml, outfile)
+        end
+      end
+
+      def concatenate_files
+        xml_files = [convert_rxl_to_xml, convert_yamls_to_xml]
+
+        xml_files.flatten.map do |xml|
+          doc = nokogiri_document(xml[:content])
+          bibdata_instance(doc, xml[:file]) if doc.root.name == "bibdata"
+        end.compact
+      end
+
+      def bibdata_instance(document, file)
+        document = clean_nokogiri_document(document)
+        bibdata = Relaton::Bibdata.from_xml(document.root)
+        build_bibdata_relaton(bibdata, file)
+
+        bibdata
+      end
+
+      def build_bibdata_relaton(bibdata, file)
+        ["xml", "pdf", "doc", "html"].each do |type|
+          file = Pathname.new(file).sub_ext(".#{type}")
+          bibdata.send("#{type}=", file) if File.file?(file)
+        end
+      end
+
+      # Force a namespace otherwise Nokogiri won't parse.
+      # The reason is we use Bibcollection's from_xml, but that one
+      # has an xmlns. We don't want to change the code for bibdata
+      # hence this hack #bibdata_doc.root['xmlns'] = "xmlns"
+      #
+      def clean_nokogiri_document(document)
+        document.remove_namespaces!
+        document.root.add_namespace(nil, "xmlns")
+        nokogiri_document(document.to_xml)
+      end
+
+      def convert_rxl_to_xml
+        select_files_with("{rxl}").map do |file|
+          { file: file, content: File.read(file, encoding: "utf-8") }
+        end
+      end
+
+      def convert_yamls_to_xml
+        select_files_with("yaml").map do |file|
+          { file: file, content: YAMLConvertor.to_xml(file, write: false) }
+        end
+      end
+
+      def select_files_with(extension)
+        files = File.join(source, "**", "*.#{extension}")
+        Dir[files].reject { |file| File.directory?(file) }
+      end
+
+      def write_to_file(content, output_file = nil)
+        output_file ||= outfile
+        File.open(output_file, "w:utf-8") { |file| file.write(content) }
+      end
+
+      def build_filename(file, document)
+        identifier = document&.at("./docidentifier")&.text ||
+          Pathname.new(File.basename(file, ".xml")).to_s
+
+        filename = identifier.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-")
+        [filename, options[:extension] || "rxl"].join(".")
+      end
+    end
+  end
+end