relaton 3.0.7 → 3.1.0

data/lib/relaton/cli/base_convertor.rb

@@ -0,0 +1,134 @@
+require "fileutils"
+require_relative "../bibdata"
+require_relative "../bibcollection"
+require_relative "xml_to_html_renderer"
+
+module Relaton
+  module Cli
+    class BaseConvertor
+      def initialize(file, options = {})
+        @file = file
+        @options = options
+        @outdir = options.fetch(:outdir, nil)
+        @writable = options.fetch(:write, true)
+        @overwrite = options.fetch(:overwrite, true)
+        @default_filelabel = 0
+
+        install_dependencies(options[:require] || [])
+      end
+
+      # @return [String] HTML
+      def to_html
+        content = convert_to_html
+        write_to_a_file(content)
+      end
+
+      # Convert to HTML
+      #
+      # This interface expect us to provide Relaton collection XML
+      # as XML/RXL, and necessary styels / templates then it will be
+      # used convert that collection to HTML.
+      #
+      # @param file [String] Relaton collection file path
+      # @param style [String] Stylesheet file path for styles
+      # @param template [String] The liquid tempalte directory
+      #
+      # @return [String] HTML
+      def self.to_html(file, style = nil, template = nil)
+        new(
+          file,
+          style: style || File.join(File.dirname(__FILE__), "templates/index-style.css"),
+          template: template || File.join(File.dirname(__FILE__), "templates/"),
+          extension: "html"
+        ).to_html
+      end
+
+      private
+
+      attr_reader :file, :outdir, :options, :writable, :overwrite
+
+      # @return [String] HTML
+      def convert_to_html
+        Relaton::Cli::XmlToHtmlRenderer.render(
+          xml_content(file),
+          stylesheet: options[:style],
+          liquid_dir: options[:template]
+        )
+      end
+
+      # @param file [String] path to a file
+      # @return [String] the file's content
+      # @return [String] HTML
+      def xml_content(file)
+        File.read(file, encoding: "utf-8")
+      end
+
+      def install_dependencies(dependencies)
+        dependencies.each { |dependency| require(dependency) }
+      end
+
+      def item_output(content, format)
+        case format.to_sym
+        when :to_yaml then content.to_yaml
+        when :to_xml then content.to_xml(date_format: :full, bibdata: true)
+        end
+      end
+
+      def convert_and_write(content, format)
+        content = convert_content(content)
+        write_to_a_file(item_output(content, format))
+        write_to_file_collection(content, format.to_sym)
+      end
+
+      def write_to_a_file(content, outfile = nil)
+        outfile ||= Pathname.new(file).sub_ext(extension).to_s
+
+        if !File.exist?(outfile) || overwrite
+          File.open(outfile, "w:utf-8") do |file|
+            file.write(content)
+          end
+        end
+      end
+
+      def write_to_file_collection(content, format)
+        if outdir && content.is_a?(Relaton::Bibcollection)
+          FileUtils.mkdir_p(outdir)
+          content.items_flattened.each do |item|
+            collection = collection_filename(extract_docid(item))
+            write_to_a_file(item_output(item, format), collection)
+          end
+        end
+      end
+
+      def extract_docid(item)
+        @default_filelabel += 1
+        item.docidentifier.nil? && (return @default_filelabel.to_s)
+        # item.docidentifier.is_a?(Array) or return @default_filelabel.to_s
+        item.docidentifier.empty? && (return @default_filelabel.to_s)
+        docidentifier_code(item.docidentifier)
+      end
+
+      # From http://gavinmiller.io/2016/creating-a-secure-sanitization-function/
+      FILENAME_BAD_CHARS = ["/", '\\', "?", '%", "*", ":', "|", '"', "<", ">",
+                            ".", " "].freeze
+
+      def docidentifier_code(docidentifier)
+        return "" if docidentifier.nil?
+
+        FILENAME_BAD_CHARS.reduce(docidentifier.downcase) do |result, bad_char|
+          result.gsub(bad_char, "-")
+        end
+      end
+
+      def extension
+        @extension ||= [".", options.fetch(:extension, default_ext)].join
+      end
+
+      def collection_filename(identifier)
+        File.join(
+          outdir, [@options[:prefix], identifier, extension].compact.join("")
+        )
+      end
+    end
+  end
+end

data/lib/relaton/cli/command.rb

@@ -0,0 +1,396 @@
+require "date"
+require "tmpdir"
+require "shellwords"
+require "pubid"
+require_relative "relaton_file"
+require_relative "xml_convertor"
+require_relative "yaml_convertor"
+require_relative "data_fetcher"
+require_relative "index_site_generator"
+require_relative "subcommand_collection"
+require_relative "subcommand_db"
+require "fcntl"
+
+module Relaton
+  module Cli
+    class Command < Thor
+      include Relaton::Cli
+      class_before :relaton_config
+      class_option :verbose, aliases: :v, type: :boolean, desc: "Output warnings"
+
+      # Exit non-zero when Thor itself rejects an invocation (unknown argument,
+      # wrong arity, a raised Thor::Error). Thor's legacy default is to print the
+      # error and exit **0**, which in CI reads as success: a caller passing a
+      # removed flag would write no site at all and still go green. The Pages
+      # deploy in relaton/support runs unattended across ~29 data repos, so a
+      # silent green there publishes an empty index nobody notices.
+      def self.exit_on_failure?
+        true
+      end
+
+      desc "version", "Show Relaton version"
+
+      def version
+        Relaton::Cli.version
+      end
+
+      desc "types", "List the supported flavors (--type values of relaton fetch)"
+
+      def types
+        puts registered_types.sort.join(", ")
+      end
+
+      desc "fetch CODE", "Fetch Relaton XML for Standard identifier CODE"
+      option :type, aliases: :t,
+                    desc: "Type of standard to get bibliographic entry for"
+      option :format, aliases: :f,
+                      desc: "Output format (xml, yaml, bibtex). Default xml."
+      option :year, aliases: :y, type: :numeric, desc: "Year the standard was published"
+      option :"all-parts", type: :boolean, desc: "Fetch all parts"
+      option :"keep-year", type: :boolean,
+                           desc: "Undated reference should return actual reference with year"
+      option :retries, aliases: :r, type: :numeric,
+                       desc: "Number of network retries. Default 1."
+      option :"no-cache", type: :boolean, desc: "Ignore cache"
+      option :"publication-date-before",
+        desc: "Fetch only documents published before the specified date " \
+          "(e.g. 2008, 2008-02, or 2008-02-02)"
+      option :"publication-date-after",
+        desc: "Fetch only documents published after the specified date " \
+          "(e.g. 2002, 2002-01, or 2002-01-01)"
+
+      def fetch(code)
+        io = IO.new($stdout.fcntl(::Fcntl::F_DUPFD), mode: "w:UTF-8")
+        io.puts(fetch_document(code, options) || supported_type_message)
+      end
+
+      desc "Metanorma-XML-File / Directory Relaton-XML-Directory",
+           "Export Relaton XML from a Metanorma XML file / directory"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
+                                                            "defaults to 'rxl'"
+
+      def extract(source_dir, outdir)
+        Relaton::Cli::RelatonFile.extract(source_dir, outdir, options)
+      end
+
+      desc "concatenate SOURCE-DIR COLLECTION-FILE",
+           "Concatenate entries in DIRECTORY (containing Relaton-XML or YAML) into a Relaton Collection"
+      option :title, aliases: :t,  desc: "Title of resulting Relaton collection"
+      option :organization, aliases: :g, desc: "Organization owner of Relaton collection"
+      option :extension, aliases: :x, desc: "File extension of destination " \
+                                            "Relaton file, defaults to 'rxl'"
+
+      def concatenate(source_dir, outfile)
+        Relaton::Cli::RelatonFile.concatenate(source_dir, outfile, options)
+      end
+
+      desc "split Relaton-Collection-File Relaton-XML-Directory",
+           "Split a Relaton Collection into multiple files"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
+                                                            "defaults to 'rxl'"
+
+      def split(source, outdir)
+        Relaton::Cli::RelatonFile.split(source, outdir, options)
+      end
+
+      desc "yaml2xml YAML", "Convert Relaton YAML into Relaton Collection XML " \
+                            "or separate files"
+      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
+                                                            "defaults to 'rxl'"
+      option :prefix, aliases: :p, desc: "Filename prefix of individual " \
+                                         "Relaton XML files, defaults to empty"
+      option :outdir, aliases: :o, desc: "Output to the specified directory " \
+                                         "with individual Relaton Bibdata XML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY " \
+                                                        "prior to execution"
+      option :overwrite, aliases: :f, type: :boolean, default: false,
+                         desc: "Overwrite the existing file"
+
+      def yaml2xml(filename)
+        Relaton::Cli::YAMLConvertor.to_xml(filename, options)
+      end
+
+      desc "xml2yaml XML", "Convert Relaton XML into Relaton Bibdata / " \
+                           "Bibcollection YAML (and separate files)"
+      option :extension, aliases: :x, default: "yaml", desc: "File extension of Relaton YAML files, " \
+                                                             "defaults to 'yaml'"
+      option :prefix, aliases: :p, desc: "Filename prefix of Relaton XML " \
+                                         "files, defaults to empty"
+      option :outdir, aliases: :o, desc: "Output to the specified directory " \
+                                         "with individual Relaton Bibdata YAML files"
+      option :require, aliases: :r, type: :array, desc: "Require LIBRARY " \
+                                                        "prior to execution"
+      option :overwrite, aliases: :f, type: :boolean, default: false,
+                         desc: "Overwrite the existing file"
+
+      def xml2yaml(filename)
+        Relaton::Cli::XMLConvertor.to_yaml(filename, options)
+      end
+
+      desc "xml2html RELATON-INDEX-XML", "Convert Relaton Collection XML into HTML"
+      option :stylesheet, aliases: :s, desc: "Stylesheet file path for " \
+                                             "rendering HTML index"
+      option :templatedir, aliases: :t, desc: "Liquid template directory for " \
+                                              "rendering Relaton items and collection"
+      option :overwrite, aliases: :f, type: :boolean, default: false,
+                         desc: "Overwrite the existing file"
+
+      def xml2html(file, style = nil, template = nil)
+        Relaton::Cli::XMLConvertor.to_html(file, style, template)
+      end
+
+      desc "yaml2html RELATON-INDEX-YAML", "Concatenate Relaton Collection " \
+                                           "YAML into HTML"
+      option :stylesheet, aliases: :s, desc: "Stylesheet file path for " \
+                                             "rendering HTML index"
+      option :templatedir, aliases: :t, desc: "Liquid template directory for " \
+                                              "rendering Relaton items and collection"
+      option :overwrite, aliases: :f, type: :boolean, default: false,
+                         desc: "Overwrite the existing file"
+
+      def yaml2html(file, style = nil, template = nil)
+        Relaton::Cli::YAMLConvertor.to_html(file, style, template)
+      end
+
+      desc "index [DATA-DIR]",
+           "Build a browsable HTML index site from a folder of Relaton YAML docs (default ./data)"
+      option :output, aliases: :o, default: "_site",
+                      desc: "Output directory for the generated site"
+      option :flavor,
+             desc: "Build from relaton/relaton-data-<flavor> (shallow-cloned) instead of a local folder"
+      option :repo,
+             desc: "Build from a GitHub repo ORG/NAME (shallow-cloned) instead of a local folder"
+      option :branch, desc: "Branch to clone when --flavor/--repo is used"
+      option :title, aliases: :t, desc: "Index page title"
+      option :description,
+             desc: "Index page description (<meta name=\"description\"> and the header subtitle)"
+      option :favicon,
+             desc: "Favicon href for <link rel=\"icon\">, used verbatim (a relative " \
+                   "path must be placed in the deployed site by the caller)"
+      option :"base-url",
+             desc: "Base URL for raw YAML links (defaults to the raw GitHub URL when cloning)"
+      option :"shard-size", type: :numeric, default: 5000,
+                            desc: "Summary records per search-NNNN.json shard"
+      option :"detail-shard-size", type: :numeric, default: 500,
+                                   desc: "Detail records per detail-NNNN.json shard"
+      option :detail, type: :boolean, default: true,
+                      desc: "Emit the detail shards the document panel fetches; --no-detail disables"
+      option :static, type: :boolean, default: true,
+                      desc: "Also index a sibling static/ folder next to DATA-DIR (auto-detected); --no-static disables"
+      option :overwrite, aliases: :f, type: :boolean, default: true,
+                         desc: "Overwrite existing output files"
+      option :"publish-data", type: :boolean, default: false,
+                              desc: "Copy the corpus onto the site so document fetches " \
+                                    "share the Pages origin (for repos whose data is " \
+                                    "served only from there); --no-publish-data disables"
+      option :"pubid-flavor",
+             desc: "Flavor whose pubid Identifier parses docids and whose INDEXFILE " \
+                   "names the published index (e.g. iso, iho, iec). Required unless " \
+                   "--no-machine-index"
+      option :"machine-index", type: :boolean, default: true,
+                               desc: "Emit the machine-consumable index (index/manifest.json, " \
+                                     "index/shard-NNNNN.json and the index-vN monolith); " \
+                                     "--no-machine-index builds the human site only"
+      option :"index-name",
+             desc: "Base name for the published index, overriding the flavor's " \
+                   "INDEXFILE (e.g. index-v2). For a corpus that is not a relaton flavor"
+
+      def index(data_dir = nil)
+        with_index_source(data_dir) do |dir, default_base_url, default_title|
+          Relaton::Cli::IndexSiteGenerator.generate(
+            dir,
+            output: options[:output],
+            shard_size: options[:"shard-size"].to_i,
+            detail_shard_size: options[:"detail-shard-size"].to_i,
+            detail: options[:detail],
+            title: options[:title] || default_title,
+            description: options[:description],
+            favicon: options[:favicon],
+            base_url: options[:"base-url"] || default_base_url,
+            static: options[:static],
+            overwrite: options[:overwrite],
+            publish_data: options[:"publish-data"],
+            flavor: options[:"pubid-flavor"],
+            machine_index: options[:"machine-index"],
+            index_name: options[:"index-name"],
+          )
+        end
+      end
+
+      desc "convert XML", "Convert Relaton XML document"
+      option :format, aliases: :f, required: true,
+             desc: "Output format (yaml, bibtex, asciibib, citation)"
+      # Plain-text citation rendering via relaton-render's CitationStyle
+      # instances (relaton-cli#94): the style names a Style YAML, the engine
+      # stays style-free.
+      option :style, desc: "Citation style for --format citation " \
+             "(author-date, apa, chicago, mla, ieee-sa, lncs; a Style YAML path " \
+             "also works). Default: author-date"
+      option :output, aliases: :o, desc: "Output to the specified file"
+
+      def convert(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"))
+        item = Relaton::Cli.parse_xml xml
+        result = if /yaml|yml/.match?(options[:format])
+                   item.to_yaml
+                 elsif options[:format] == "citation"
+                   # The paths that exist in every published relaton-render;
+                   # the gem's conventional "relaton/render" entry is newer.
+require_relative "../render/general"
+require_relative "../render/iso690"
+                   Relaton::Render::General
+                     .new(style: options[:style] || "author-date")
+                     .render(item, embedded: true)
+                 else item.send "to_#{options[:format]}"
+                 end
+        ext = case options[:format]
+              when "bibtex" then "bib"
+              when "asciibib" then "adoc"
+              when "citation" then "txt"
+              else options[:format]
+              end
+        output = options[:output] || file.sub(/(?<=\.)[^.]+$/, ext)
+        File.write output, result, encoding: "UTF-8"
+      end
+
+      desc "fetch-data SOURCE", "Fetch all the documents from a source"
+      option :output, aliases: :o, desc: "Output dir. Default: ./data/"
+      option :format, aliases: :f, desc: "Output format (yaml, xml, bibxml). Default: yaml"
+
+      def fetch_data(source)
+        DataFetcher.fetch source, options
+      end
+
+      desc "collection SUBCOMMAND", "Collection manipulations"
+      subcommand "collection", SubcommandCollection
+
+      desc "db SUBCOMMAND", "Cache DB manipulation"
+      subcommand "db", SubcommandDb
+
+      no_commands do
+        def relaton_config
+          Relaton::Logger.configure do |conf|
+            if options[:verbose]
+              conf.logger_pool[:default].add_level :info
+            else
+              conf.logger_pool[:default].remove_level :info
+            end
+          end
+        end
+
+        # Resolve the source folder for `index`. Locally it's just DATA-DIR
+        # (default ./data). With --flavor/--repo it shallow-clones the GitHub
+        # repo into a temp dir and yields its data/ subfolder plus sensible
+        # defaults for the raw-YAML base URL and page title. The temp clone is
+        # removed after the block returns.
+        def with_index_source(data_dir)
+          subdir = data_dir || "data"
+          if options[:flavor] || options[:repo]
+            repo = options[:repo] || "relaton/relaton-data-#{options[:flavor]}"
+            flavor = options[:flavor] ||
+              repo.split("/").last.sub(/\Arelaton-data-/, "")
+            Dir.mktmpdir("relaton-index-") do |tmp|
+              branch = clone_data_repo(repo, options[:branch], tmp)
+              base = "https://raw.githubusercontent.com/#{repo}/#{branch}"
+              yield File.join(tmp, subdir), base, "#{flavor.upcase} Index"
+            end
+          else
+            yield subdir, nil, nil
+          end
+        end
+
+        # Shallow-clone ORG/NAME into `dest`; return the checked-out branch name.
+        def clone_data_repo(repo, branch, dest)
+          url = "https://github.com/#{repo}.git"
+          cmd = ["git", "clone", "--depth", "1", "--single-branch"]
+          cmd += ["--branch", branch] if branch
+          cmd += [url, dest]
+          Relaton::Cli::Util.info "Cloning #{url}…"
+          unless system(*cmd, out: File::NULL, err: File::NULL)
+            raise Thor::Error,
+                  "Failed to clone #{url}. Check the repo/branch name and that git is installed."
+          end
+          branch || `git -C #{Shellwords.escape(dest)} rev-parse --abbrev-ref HEAD`.strip
+        end
+      end
+    end
+
+    private
+
+    DATE_FILTER_FORMAT = /\A\d{4}(-\d{2}(-\d{2})?)?\z/
+
+    def parse_date_option(value, name)
+      return unless value
+
+      unless value.match?(DATE_FILTER_FORMAT)
+        raise ArgumentError,
+          "Invalid #{name}: #{value.inspect}. Expected YYYY, YYYY-MM, or YYYY-MM-DD."
+      end
+      parts = value.split("-").map(&:to_i)
+      Date.new(*parts.concat([1] * (3 - parts.size)))
+    rescue Date::Error
+      raise ArgumentError,
+        "Invalid #{name}: #{value.inspect}. Date components are out of range."
+    end
+
+    def validate_date_range(date_after, date_before)
+      return unless date_after && date_before
+      return if date_after < date_before
+
+      raise ArgumentError,
+        "Invalid date range: --publication-date-after (#{date_after}) must be before --publication-date-before (#{date_before})."
+    end
+
+    # @param code [String]
+    # @param options [Hash]
+    # @option options [String] :type
+    # @option options [String, NilClass] :format
+    # @option options [Integer, NilClass] :year
+    # @return [String, nil]
+    def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/MethodLength
+      year = options[:year]&.to_s
+      dup_opts = options.dup.transform_keys { |k| k.to_s.gsub("-", "_").to_sym }
+      %i[publication_date_before publication_date_after].each do |key|
+        dup_opts[key] = parse_date_option(dup_opts[key], key.to_s.tr("_", "-").prepend("--")) if dup_opts[key]
+      end
+      validate_date_range dup_opts[:publication_date_after], dup_opts[:publication_date_before]
+      if (processor = Relaton::Db::Registry.instance.by_type options[:type]&.upcase)
+        doc = Relaton.db.fetch_std code, year, processor.short, **dup_opts
+      elsif options[:type] then return
+      else doc = Relaton.db.fetch(code, year, **dup_opts)
+      end
+      return "No matching bibliographic entry found" unless doc
+
+      serialize doc, options[:format]
+    rescue Relaton::UnknownReferenceError => e
+      # No flavor recognizes the reference: log it, return no document (the
+      # command then prints the supported types).
+      Util.warn e.message
+      nil
+    rescue Pubid::Errors::Error
+      %("#{code}" is not a recognized standards identifier)
+    rescue Relaton::RequestError => e
+      e.message
+    end
+
+    # @param doc [Relaton::Bib::ItemData]
+    # @param format [String]
+    # @return [String]
+    def serialize(doc, format)
+      case format
+      when "yaml", "yml" then doc.to_yaml
+      when "bibtex" then doc.to_bibtex
+      else doc.to_xml bibdata: true
+      end
+    end
+
+    def supported_type_message
+      ["Recognised types:", registered_types.sort.join(", ")].join(" ")
+    end
+
+    def registered_types
+      @registered_types ||=
+        Relaton::Db::Registry.instance.processors.each.map { |_n, pr| pr.prefix }
+    end
+  end
+end

data/lib/relaton/cli/data_fetcher.rb

@@ -0,0 +1,20 @@
+module Relaton
+  module Cli
+    module DataFetcher
+      def fetch(source, options)
+        processor = Relaton::Db::Registry.instance.find_processor_by_dataset source
+        unless processor
+          Util.warn "no processor found for `#{source}`"
+          return
+        end
+
+        opts = {}
+        opts[:output] = options[:output] if options[:output]
+        opts[:format] = options[:format] if options[:format]
+        processor.fetch_data source, opts
+      end
+
+      extend DataFetcher
+    end
+  end
+end

data/lib/relaton/cli/frontend/dist/404.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Redirecting…</title>
+    <script>
+      // GitHub Pages has no URL rewrites, so path-based document URLs
+      // (/doc/<id>) land on this 404 fallback. Hand the id to the app as
+      // ?doc=<id>, preserving the site's base path wherever it is hosted.
+      (function () {
+        var m = location.pathname.match(/\/doc\/([^/?#]+)$/);
+        var root = m ? location.pathname.slice(0, m.index) : location.pathname.replace(/[^/]*$/, "");
+        location.replace(root + "?doc=" + (m ? m[1] : ""));
+      })();
+    </script>
+  </head>
+  <body></body>
+</html>