relaton-cli 3.0.7 → 3.1.0

data/lib/relaton/cli/base_convertor.rb

@@ -1,134 +0,0 @@
-require "fileutils"
-require "relaton/bibdata"
-require "relaton/bibcollection"
-require "relaton/cli/xml_to_html_renderer"
-
-module Relaton
-  module Cli
-    class BaseConvertor
-      def initialize(file, options = {})
-        @file = file
-        @options = options
-        @outdir = options.fetch(:outdir, nil)
-        @writable = options.fetch(:write, true)
-        @overwrite = options.fetch(:overwrite, true)
-        @default_filelabel = 0
-
-        install_dependencies(options[:require] || [])
-      end
-
-      # @return [String] HTML
-      def to_html
-        content = convert_to_html
-        write_to_a_file(content)
-      end
-
-      # Convert to HTML
-      #
-      # This interface expect us to provide Relaton collection XML
-      # as XML/RXL, and necessary styels / templates then it will be
-      # used convert that collection to HTML.
-      #
-      # @param file [String] Relaton collection file path
-      # @param style [String] Stylesheet file path for styles
-      # @param template [String] The liquid tempalte directory
-      #
-      # @return [String] HTML
-      def self.to_html(file, style = nil, template = nil)
-        new(
-          file,
-          style: style || File.join(File.dirname(__FILE__), "../../../templates/index-style.css"),
-          template: template || File.join(File.dirname(__FILE__), "../../../templates/"),
-          extension: "html"
-        ).to_html
-      end
-
-      private
-
-      attr_reader :file, :outdir, :options, :writable, :overwrite
-
-      # @return [String] HTML
-      def convert_to_html
-        Relaton::Cli::XmlToHtmlRenderer.render(
-          xml_content(file),
-          stylesheet: options[:style],
-          liquid_dir: options[:template]
-        )
-      end
-
-      # @param file [String] path to a file
-      # @return [String] the file's content
-      # @return [String] HTML
-      def xml_content(file)
-        File.read(file, encoding: "utf-8")
-      end
-
-      def install_dependencies(dependencies)
-        dependencies.each { |dependency| require(dependency) }
-      end
-
-      def item_output(content, format)
-        case format.to_sym
-        when :to_yaml then content.to_yaml
-        when :to_xml then content.to_xml(date_format: :full, bibdata: true)
-        end
-      end
-
-      def convert_and_write(content, format)
-        content = convert_content(content)
-        write_to_a_file(item_output(content, format))
-        write_to_file_collection(content, format.to_sym)
-      end
-
-      def write_to_a_file(content, outfile = nil)
-        outfile ||= Pathname.new(file).sub_ext(extension).to_s
-
-        if !File.exist?(outfile) || overwrite
-          File.open(outfile, "w:utf-8") do |file|
-            file.write(content)
-          end
-        end
-      end
-
-      def write_to_file_collection(content, format)
-        if outdir && content.is_a?(Relaton::Bibcollection)
-          FileUtils.mkdir_p(outdir)
-          content.items_flattened.each do |item|
-            collection = collection_filename(extract_docid(item))
-            write_to_a_file(item_output(item, format), collection)
-          end
-        end
-      end
-
-      def extract_docid(item)
-        @default_filelabel += 1
-        item.docidentifier.nil? && (return @default_filelabel.to_s)
-        # item.docidentifier.is_a?(Array) or return @default_filelabel.to_s
-        item.docidentifier.empty? && (return @default_filelabel.to_s)
-        docidentifier_code(item.docidentifier)
-      end
-
-      # From http://gavinmiller.io/2016/creating-a-secure-sanitization-function/
-      FILENAME_BAD_CHARS = ["/", '\\', "?", '%", "*", ":', "|", '"', "<", ">",
-                            ".", " "].freeze
-
-      def docidentifier_code(docidentifier)
-        return "" if docidentifier.nil?
-
-        FILENAME_BAD_CHARS.reduce(docidentifier.downcase) do |result, bad_char|
-          result.gsub(bad_char, "-")
-        end
-      end
-
-      def extension
-        @extension ||= [".", options.fetch(:extension, default_ext)].join
-      end
-
-      def collection_filename(identifier)
-        File.join(
-          outdir, [@options[:prefix], identifier, extension].compact.join("")
-        )
-      end
-    end
-  end
-end

data/lib/relaton/cli/command.rb

@@ -1,390 +0,0 @@
-require "date"
-require "tmpdir"
-require "shellwords"
-require "pubid"
-require "relaton/cli/relaton_file"
-require "relaton/cli/xml_convertor"
-require "relaton/cli/yaml_convertor"
-require "relaton/cli/data_fetcher"
-require "relaton/cli/index_site_generator"
-require "relaton/cli/subcommand_collection"
-require "relaton/cli/subcommand_db"
-require "fcntl"
-
-module Relaton
-  module Cli
-    class Command < Thor
-      include Relaton::Cli
-      class_before :relaton_config
-      class_option :verbose, aliases: :v, type: :boolean, desc: "Output warnings"
-
-      # Exit non-zero when Thor itself rejects an invocation (unknown argument,
-      # wrong arity, a raised Thor::Error). Thor's legacy default is to print the
-      # error and exit **0**, which in CI reads as success: a caller passing a
-      # removed flag would write no site at all and still go green. The Pages
-      # deploy in relaton/support runs unattended across ~29 data repos, so a
-      # silent green there publishes an empty index nobody notices.
-      def self.exit_on_failure?
-        true
-      end
-
-      desc "version", "Show Relaton version"
-
-      def version
-        Relaton::Cli.version
-      end
-
-      desc "fetch CODE", "Fetch Relaton XML for Standard identifier CODE"
-      option :type, aliases: :t,
-                    desc: "Type of standard to get bibliographic entry for"
-      option :format, aliases: :f,
-                      desc: "Output format (xml, yaml, bibtex). Default xml."
-      option :year, aliases: :y, type: :numeric, desc: "Year the standard was published"
-      option :"all-parts", type: :boolean, desc: "Fetch all parts"
-      option :"keep-year", type: :boolean,
-                           desc: "Undated reference should return actual reference with year"
-      option :retries, aliases: :r, type: :numeric,
-                       desc: "Number of network retries. Default 1."
-      option :"no-cache", type: :boolean, desc: "Ignore cache"
-      option :"publication-date-before",
-        desc: "Fetch only documents published before the specified date " \
-          "(e.g. 2008, 2008-02, or 2008-02-02)"
-      option :"publication-date-after",
-        desc: "Fetch only documents published after the specified date " \
-          "(e.g. 2002, 2002-01, or 2002-01-01)"
-
-      def fetch(code)
-        io = IO.new($stdout.fcntl(::Fcntl::F_DUPFD), mode: "w:UTF-8")
-        io.puts(fetch_document(code, options) || supported_type_message)
-      end
-
-      desc "Metanorma-XML-File / Directory Relaton-XML-Directory",
-           "Export Relaton XML from a Metanorma XML file / directory"
-      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
-                                                            "defaults to 'rxl'"
-
-      def extract(source_dir, outdir)
-        Relaton::Cli::RelatonFile.extract(source_dir, outdir, options)
-      end
-
-      desc "concatenate SOURCE-DIR COLLECTION-FILE",
-           "Concatenate entries in DIRECTORY (containing Relaton-XML or YAML) into a Relaton Collection"
-      option :title, aliases: :t,  desc: "Title of resulting Relaton collection"
-      option :organization, aliases: :g, desc: "Organization owner of Relaton collection"
-      option :extension, aliases: :x, desc: "File extension of destination " \
-                                            "Relaton file, defaults to 'rxl'"
-
-      def concatenate(source_dir, outfile)
-        Relaton::Cli::RelatonFile.concatenate(source_dir, outfile, options)
-      end
-
-      desc "split Relaton-Collection-File Relaton-XML-Directory",
-           "Split a Relaton Collection into multiple files"
-      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
-                                                            "defaults to 'rxl'"
-
-      def split(source, outdir)
-        Relaton::Cli::RelatonFile.split(source, outdir, options)
-      end
-
-      desc "yaml2xml YAML", "Convert Relaton YAML into Relaton Collection XML " \
-                            "or separate files"
-      option :extension, aliases: :x, default: "rxl", desc: "File extension of Relaton XML files, " \
-                                                            "defaults to 'rxl'"
-      option :prefix, aliases: :p, desc: "Filename prefix of individual " \
-                                         "Relaton XML files, defaults to empty"
-      option :outdir, aliases: :o, desc: "Output to the specified directory " \
-                                         "with individual Relaton Bibdata XML files"
-      option :require, aliases: :r, type: :array, desc: "Require LIBRARY " \
-                                                        "prior to execution"
-      option :overwrite, aliases: :f, type: :boolean, default: false,
-                         desc: "Overwrite the existing file"
-
-      def yaml2xml(filename)
-        Relaton::Cli::YAMLConvertor.to_xml(filename, options)
-      end
-
-      desc "xml2yaml XML", "Convert Relaton XML into Relaton Bibdata / " \
-                           "Bibcollection YAML (and separate files)"
-      option :extension, aliases: :x, default: "yaml", desc: "File extension of Relaton YAML files, " \
-                                                             "defaults to 'yaml'"
-      option :prefix, aliases: :p, desc: "Filename prefix of Relaton XML " \
-                                         "files, defaults to empty"
-      option :outdir, aliases: :o, desc: "Output to the specified directory " \
-                                         "with individual Relaton Bibdata YAML files"
-      option :require, aliases: :r, type: :array, desc: "Require LIBRARY " \
-                                                        "prior to execution"
-      option :overwrite, aliases: :f, type: :boolean, default: false,
-                         desc: "Overwrite the existing file"
-
-      def xml2yaml(filename)
-        Relaton::Cli::XMLConvertor.to_yaml(filename, options)
-      end
-
-      desc "xml2html RELATON-INDEX-XML", "Convert Relaton Collection XML into HTML"
-      option :stylesheet, aliases: :s, desc: "Stylesheet file path for " \
-                                             "rendering HTML index"
-      option :templatedir, aliases: :t, desc: "Liquid template directory for " \
-                                              "rendering Relaton items and collection"
-      option :overwrite, aliases: :f, type: :boolean, default: false,
-                         desc: "Overwrite the existing file"
-
-      def xml2html(file, style = nil, template = nil)
-        Relaton::Cli::XMLConvertor.to_html(file, style, template)
-      end
-
-      desc "yaml2html RELATON-INDEX-YAML", "Concatenate Relaton Collection " \
-                                           "YAML into HTML"
-      option :stylesheet, aliases: :s, desc: "Stylesheet file path for " \
-                                             "rendering HTML index"
-      option :templatedir, aliases: :t, desc: "Liquid template directory for " \
-                                              "rendering Relaton items and collection"
-      option :overwrite, aliases: :f, type: :boolean, default: false,
-                         desc: "Overwrite the existing file"
-
-      def yaml2html(file, style = nil, template = nil)
-        Relaton::Cli::YAMLConvertor.to_html(file, style, template)
-      end
-
-      desc "index [DATA-DIR]",
-           "Build a browsable HTML index site from a folder of Relaton YAML docs (default ./data)"
-      option :output, aliases: :o, default: "_site",
-                      desc: "Output directory for the generated site"
-      option :flavor,
-             desc: "Build from relaton/relaton-data-<flavor> (shallow-cloned) instead of a local folder"
-      option :repo,
-             desc: "Build from a GitHub repo ORG/NAME (shallow-cloned) instead of a local folder"
-      option :branch, desc: "Branch to clone when --flavor/--repo is used"
-      option :title, aliases: :t, desc: "Index page title"
-      option :description,
-             desc: "Index page description (<meta name=\"description\"> and the header subtitle)"
-      option :favicon,
-             desc: "Favicon href for <link rel=\"icon\">, used verbatim (a relative " \
-                   "path must be placed in the deployed site by the caller)"
-      option :"base-url",
-             desc: "Base URL for raw YAML links (defaults to the raw GitHub URL when cloning)"
-      option :"shard-size", type: :numeric, default: 5000,
-                            desc: "Summary records per search-NNNN.json shard"
-      option :"detail-shard-size", type: :numeric, default: 500,
-                                   desc: "Detail records per detail-NNNN.json shard"
-      option :detail, type: :boolean, default: true,
-                      desc: "Emit the detail shards the document panel fetches; --no-detail disables"
-      option :static, type: :boolean, default: true,
-                      desc: "Also index a sibling static/ folder next to DATA-DIR (auto-detected); --no-static disables"
-      option :overwrite, aliases: :f, type: :boolean, default: true,
-                         desc: "Overwrite existing output files"
-      option :"publish-data", type: :boolean, default: false,
-                              desc: "Copy the corpus onto the site so document fetches " \
-                                    "share the Pages origin (for repos whose data is " \
-                                    "served only from there); --no-publish-data disables"
-      option :"pubid-flavor",
-             desc: "Flavor whose pubid Identifier parses docids and whose INDEXFILE " \
-                   "names the published index (e.g. iso, iho, iec). Required unless " \
-                   "--no-machine-index"
-      option :"machine-index", type: :boolean, default: true,
-                               desc: "Emit the machine-consumable index (index/manifest.json, " \
-                                     "index/shard-NNNNN.json and the index-vN monolith); " \
-                                     "--no-machine-index builds the human site only"
-      option :"index-name",
-             desc: "Base name for the published index, overriding the flavor's " \
-                   "INDEXFILE (e.g. index-v2). For a corpus that is not a relaton flavor"
-
-      def index(data_dir = nil)
-        with_index_source(data_dir) do |dir, default_base_url, default_title|
-          Relaton::Cli::IndexSiteGenerator.generate(
-            dir,
-            output: options[:output],
-            shard_size: options[:"shard-size"].to_i,
-            detail_shard_size: options[:"detail-shard-size"].to_i,
-            detail: options[:detail],
-            title: options[:title] || default_title,
-            description: options[:description],
-            favicon: options[:favicon],
-            base_url: options[:"base-url"] || default_base_url,
-            static: options[:static],
-            overwrite: options[:overwrite],
-            publish_data: options[:"publish-data"],
-            flavor: options[:"pubid-flavor"],
-            machine_index: options[:"machine-index"],
-            index_name: options[:"index-name"],
-          )
-        end
-      end
-
-      desc "convert XML", "Convert Relaton XML document"
-      option :format, aliases: :f, required: true,
-             desc: "Output format (yaml, bibtex, asciibib, citation)"
-      # Plain-text citation rendering via relaton-render's CitationStyle
-      # instances (relaton-cli#94): the style names a Style YAML, the engine
-      # stays style-free.
-      option :style, desc: "Citation style for --format citation " \
-             "(author-date, apa, chicago, mla, ieee-sa, lncs; a Style YAML path " \
-             "also works). Default: author-date"
-      option :output, aliases: :o, desc: "Output to the specified file"
-
-      def convert(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-        xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"))
-        item = Relaton::Cli.parse_xml xml
-        result = if /yaml|yml/.match?(options[:format])
-                   item.to_yaml
-                 elsif options[:format] == "citation"
-                   # The paths that exist in every published relaton-render;
-                   # the gem's conventional "relaton/render" entry is newer.
-                   require "relaton/render/general"
-                   require "relaton/render/iso690"
-                   Relaton::Render::General
-                     .new(style: options[:style] || "author-date")
-                     .render(item, embedded: true)
-                 else item.send "to_#{options[:format]}"
-                 end
-        ext = case options[:format]
-              when "bibtex" then "bib"
-              when "asciibib" then "adoc"
-              when "citation" then "txt"
-              else options[:format]
-              end
-        output = options[:output] || file.sub(/(?<=\.)[^.]+$/, ext)
-        File.write output, result, encoding: "UTF-8"
-      end
-
-      desc "fetch-data SOURCE", "Fetch all the documents from a source"
-      option :output, aliases: :o, desc: "Output dir. Default: ./data/"
-      option :format, aliases: :f, desc: "Output format (yaml, xml, bibxml). Default: yaml"
-
-      def fetch_data(source)
-        DataFetcher.fetch source, options
-      end
-
-      desc "collection SUBCOMMAND", "Collection manipulations"
-      subcommand "collection", SubcommandCollection
-
-      desc "db SUBCOMMAND", "Cache DB manipulation"
-      subcommand "db", SubcommandDb
-
-      no_commands do
-        def relaton_config
-          Relaton::Logger.configure do |conf|
-            if options[:verbose]
-              conf.logger_pool[:default].add_level :info
-            else
-              conf.logger_pool[:default].remove_level :info
-            end
-          end
-        end
-
-        # Resolve the source folder for `index`. Locally it's just DATA-DIR
-        # (default ./data). With --flavor/--repo it shallow-clones the GitHub
-        # repo into a temp dir and yields its data/ subfolder plus sensible
-        # defaults for the raw-YAML base URL and page title. The temp clone is
-        # removed after the block returns.
-        def with_index_source(data_dir)
-          subdir = data_dir || "data"
-          if options[:flavor] || options[:repo]
-            repo = options[:repo] || "relaton/relaton-data-#{options[:flavor]}"
-            flavor = options[:flavor] ||
-              repo.split("/").last.sub(/\Arelaton-data-/, "")
-            Dir.mktmpdir("relaton-index-") do |tmp|
-              branch = clone_data_repo(repo, options[:branch], tmp)
-              base = "https://raw.githubusercontent.com/#{repo}/#{branch}"
-              yield File.join(tmp, subdir), base, "#{flavor.upcase} Index"
-            end
-          else
-            yield subdir, nil, nil
-          end
-        end
-
-        # Shallow-clone ORG/NAME into `dest`; return the checked-out branch name.
-        def clone_data_repo(repo, branch, dest)
-          url = "https://github.com/#{repo}.git"
-          cmd = ["git", "clone", "--depth", "1", "--single-branch"]
-          cmd += ["--branch", branch] if branch
-          cmd += [url, dest]
-          Relaton::Cli::Util.info "Cloning #{url}…"
-          unless system(*cmd, out: File::NULL, err: File::NULL)
-            raise Thor::Error,
-                  "Failed to clone #{url}. Check the repo/branch name and that git is installed."
-          end
-          branch || `git -C #{Shellwords.escape(dest)} rev-parse --abbrev-ref HEAD`.strip
-        end
-      end
-    end
-
-    private
-
-    DATE_FILTER_FORMAT = /\A\d{4}(-\d{2}(-\d{2})?)?\z/
-
-    def parse_date_option(value, name)
-      return unless value
-
-      unless value.match?(DATE_FILTER_FORMAT)
-        raise ArgumentError,
-          "Invalid #{name}: #{value.inspect}. Expected YYYY, YYYY-MM, or YYYY-MM-DD."
-      end
-      parts = value.split("-").map(&:to_i)
-      Date.new(*parts.concat([1] * (3 - parts.size)))
-    rescue Date::Error
-      raise ArgumentError,
-        "Invalid #{name}: #{value.inspect}. Date components are out of range."
-    end
-
-    def validate_date_range(date_after, date_before)
-      return unless date_after && date_before
-      return if date_after < date_before
-
-      raise ArgumentError,
-        "Invalid date range: --publication-date-after (#{date_after}) must be before --publication-date-before (#{date_before})."
-    end
-
-    # @param code [String]
-    # @param options [Hash]
-    # @option options [String] :type
-    # @option options [String, NilClass] :format
-    # @option options [Integer, NilClass] :year
-    # @return [String, nil]
-    def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize,Metrics/MethodLength
-      year = options[:year]&.to_s
-      dup_opts = options.dup.transform_keys { |k| k.to_s.gsub("-", "_").to_sym }
-      %i[publication_date_before publication_date_after].each do |key|
-        dup_opts[key] = parse_date_option(dup_opts[key], key.to_s.tr("_", "-").prepend("--")) if dup_opts[key]
-      end
-      validate_date_range dup_opts[:publication_date_after], dup_opts[:publication_date_before]
-      if (processor = Relaton::Db::Registry.instance.by_type options[:type]&.upcase)
-        doc = Relaton.db.fetch_std code, year, processor.short, **dup_opts
-      elsif options[:type] then return
-      else doc = Relaton.db.fetch(code, year, **dup_opts)
-      end
-      return "No matching bibliographic entry found" unless doc
-
-      serialize doc, options[:format]
-    rescue Relaton::UnknownReferenceError => e
-      # No flavor recognizes the reference: log it, return no document (the
-      # command then prints the supported types).
-      Util.warn e.message
-      nil
-    rescue Pubid::Errors::Error
-      %("#{code}" is not a recognized standards identifier)
-    rescue Relaton::RequestError => e
-      e.message
-    end
-
-    # @param doc [Relaton::Bib::ItemData]
-    # @param format [String]
-    # @return [String]
-    def serialize(doc, format)
-      case format
-      when "yaml", "yml" then doc.to_yaml
-      when "bibtex" then doc.to_bibtex
-      else doc.to_xml bibdata: true
-      end
-    end
-
-    def supported_type_message
-      ["Recognised types:", registered_types.sort.join(", ")].join(" ")
-    end
-
-    def registered_types
-      @registered_types ||=
-        Relaton::Db::Registry.instance.processors.each.map { |_n, pr| pr.prefix }
-    end
-  end
-end

data/lib/relaton/cli/data_fetcher.rb

@@ -1,20 +0,0 @@
-module Relaton
-  module Cli
-    module DataFetcher
-      def fetch(source, options)
-        processor = Relaton::Db::Registry.instance.find_processor_by_dataset source
-        unless processor
-          Util.warn "no processor found for `#{source}`"
-          return
-        end
-
-        opts = {}
-        opts[:output] = options[:output] if options[:output]
-        opts[:format] = options[:format] if options[:format]
-        processor.fetch_data source, opts
-      end
-
-      extend DataFetcher
-    end
-  end
-end

data/lib/relaton/cli/frontend_assets.rb

@@ -1,58 +0,0 @@
-module Relaton
-  module Cli
-    # Resolves the compiled frontend bundle (`app.iife.js` + `style.css`) that is
-    # built from `gems/relaton-cli/frontend/` at gem-build time and shipped inside
-    # the gem under `frontend/dist/`. Both the installed gem and a git checkout
-    # place `frontend/dist` at the gem root, so one relative path covers both.
-    #
-    # If the bundle is missing (a fresh checkout that hasn't run the frontend
-    # build), raise a clear, actionable error rather than emitting a broken page.
-    module FrontendAssets
-      class BuildMissingError < StandardError; end
-
-      # gem root = up three from lib/relaton/cli
-      DEFAULT_DIST = File.expand_path("../../../frontend/dist", __dir__).freeze
-
-      class << self
-        # Overridable so specs can point at a fixture dist without a Node build.
-        attr_writer :dist_dir
-
-        def dist_dir
-          @dist_dir || DEFAULT_DIST
-        end
-
-        # Run a block with a temporary dist dir, restoring the previous one.
-        def with_dist_dir(dir)
-          previous = @dist_dir
-          @dist_dir = dir
-          yield
-        ensure
-          @dist_dir = previous
-        end
-
-        def iife
-          read("app.iife.js")
-        end
-
-        def stylesheet
-          read("style.css")
-        end
-
-        def built?
-          File.exist?(File.join(dist_dir, "app.iife.js"))
-        end
-
-        def read(name)
-          path = File.join(dist_dir, name)
-          unless File.exist?(path)
-            raise BuildMissingError,
-                  "Frontend build missing (#{name} not found in #{dist_dir}). " \
-                  "Run `bundle exec rake build_frontend` in gems/relaton-cli " \
-                  "(or `cd frontend && npm install && npm run build`) first."
-          end
-          File.read(path, encoding: "utf-8")
-        end
-      end
-    end
-  end
-end