relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/bipm/si_brochure_parser.rb

@@ -0,0 +1,186 @@
+require "nokogiri"
+require_relative "id_parser"
+
+module Relaton::Bipm
+  class SiBrochureParser
+    #
+    # Create new parser
+    #
+    # @param [Relaton::Bipm::DataFetcher] data_fetcher data fetcher
+    #
+    def initialize(data_fetcher)
+      @data_fetcher = WeakRef.new data_fetcher
+    end
+
+    #
+    # Parse documents from SI brochure dataset and write thems to YAML files
+    #
+    # @param [Relaton::Bipm::DataFetcher] data_fetcher data fetcher
+    #
+    def self.parse(data_fetcher)
+      new(data_fetcher).parse
+    end
+
+    #
+    # Parse SI brochure and write them to YAML files
+    #
+    def parse # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # metanorma site generate writes per-document outputs into a subdirectory
+      # named after the source path (e.g. _site/documents/si-brochure/3.01/
+      # si-brochure-en.rxl). The legacy top-level *.rxl glob is kept for
+      # backwards compatibility with any older flow that flattened outputs.
+      si_brochure_rxls.each do |f|
+        puts "Parsing #{f}"
+        xml = File.read(f, encoding: "UTF-8")
+        xml = xml.force_encoding("UTF-8") if xml.encoding != Encoding::UTF_8
+        item1 = Bibdata.from_xml(xml)
+        # Workaround for relaton-bib Version#content bug: whitespace between
+        # legacy <revision-date>/<draft> children gets captured as @content,
+        # blocking the legacy-fold path. Clear it so the getter recomputes.
+        item1.version.each do |v|
+          c = v.instance_variable_get(:@content)
+          next unless c.is_a?(Array) || (c.is_a?(String) && c.strip.empty?)
+          v.instance_variable_set(:@content, nil)
+        end
+        @data_fetcher.errors[:si_brochure_title] &&= item1.title.empty?
+        @data_fetcher.errors[:si_brochure_docidentifier] &&= item1.docidentifier.empty?
+        unless has_committee_contributor?(item1)
+          contribs = extract_editorialgroup(xml)
+          contribs.each { |c| item1.contributor << c }
+        end
+        fix_si_brochure_id item1
+        basename = File.join @data_fetcher.output, File.basename(f).sub(/(?:-(?:en|fr))?\.rxl$/, "")
+        outfile = "#{basename}.#{@data_fetcher.ext}"
+        key = item1.docnumber || basename
+        @data_fetcher.index.add_or_update Id.new.parse(key).to_hash, outfile
+        item =
+          if File.exist? outfile
+            warn_duplicate = false
+            item2 = Item.from_yaml File.read(outfile, encoding: "UTF-8")
+            fix_si_brochure_id item2
+            hash1 = YAML.safe_load item1.to_yaml
+            hash2 = YAML.safe_load item2.to_yaml
+            Item.from_yaml deep_merge(hash1, hash2).to_yaml
+          else
+            warn_duplicate = true
+            item1
+          end
+        @data_fetcher.write_file outfile, item, warn_duplicate: warn_duplicate
+        puts "Saved to #{outfile}"
+      end
+    end
+
+    #
+    # @return [Array<String>] paths to SI Brochure RXL files. Looks at the
+    #   legacy flat layout first, then the metanorma-cli subdirectory layout
+    #   (`<source_path_without_'sources'>/<doc>.rxl`) used by current
+    #   `metanorma site generate` output.
+    #
+    def si_brochure_rxls
+      flat = Dir["bipm-si-brochure/_site/documents/*.rxl"]
+      return flat if flat.any?
+
+      Dir["bipm-si-brochure/_site/documents/**/si-brochure-{en,fr}.rxl"]
+    end
+
+    #
+    # Update ID of SI brochure
+    #
+    # @param [ItemData] item bibliographic item
+    #
+    # @return [void]
+    #
+    def fix_si_brochure_id(item)
+      # isbn = hash["docid"].detect { |id| id["type"] == "ISBN" }
+      # num = isbn && isbn["id"] == "978-92-822-2272-0" ?  "SI Brochure" : "SI Brochure, Appendix 4"
+
+      update_id item
+
+      prid = primary_id item
+      if item.docnumber
+        item.docnumber.sub!(/^Brochure(?:\sConcise|\sFAQ)?$/i, prid.sub(/^BIPM\s/, ""))
+      else
+        item.docnumber = prid.sub(/^BIPM\s/, "")
+      end
+      item.id = prid.gsub(/[,\s-]/, "")
+    end
+
+    def update_id(item)
+      item.docidentifier.each do |id|
+        next unless id.type == "BIPM" && id.content&.match?(/BIPM Brochure/i)
+
+        id.primary = true
+        id.content.sub!(/(?<=^BIPM\s)(Brochure)/i, "SI \\1")
+      end
+    end
+
+    def primary_id(item)
+      item.docidentifier.detect do |id|
+        id.primary && (id.language == "en" || id.language.nil?)
+      end.content
+    end
+
+    def has_committee_contributor?(item)
+      item.contributor.any? do |c|
+        c.role.any? { |r| r.description.any? { |d| d.content == "committee" } }
+      end
+    end
+
+    def extract_editorialgroup(xml)
+      doc = Nokogiri::XML(xml)
+      doc.xpath("//editorialgroup/committee").map do |committee|
+        acronym = committee["acronym"]
+        names = committee.xpath("variant").map do |v|
+          Relaton::Bib::TypedLocalizedString.new(
+            content: v.text, language: v["language"], script: v["script"],
+          )
+        end
+        subdiv = Relaton::Bib::Subdivision.new(
+          type: "committee", name: names,
+          abbreviation: Relaton::Bib::LocalizedString.new(content: acronym),
+        )
+        bipm_name = [Relaton::Bib::TypedLocalizedString.new(content: acronym)]
+        org = Relaton::Bib::Organization.new(name: bipm_name, subdivision: [subdiv])
+        desc = Relaton::Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Relaton::Bib::Contributor::Role.new(type: "author", description: [desc])
+        Relaton::Bib::Contributor.new(organization: org, role: [role])
+      end
+    end
+
+    #
+    # Deep merge two hashes
+    #
+    # @param [Hash] hash1
+    # @param [Hash] hash2
+    #
+    # @return [Hash] Merged hash
+    #
+    def deep_merge(hash1, hash2) # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+      hash1.merge(hash2) do |_, oldval, newval|
+        if oldval.is_a?(Hash) && newval.is_a?(Hash)
+          deep_merge(oldval, newval)
+        elsif oldval.is_a?(Array) && newval.is_a?(Array)
+          (oldval + newval).uniq { |i| downcase_all i }
+        else
+          newval || oldval
+        end
+      end
+    end
+
+    #
+    # Downcase all values in hash or array
+    #
+    # @param [Array, Hash, String] content hash, array or string
+    #
+    # @return [Array, Hash, String] hash, array or string with downcased values
+    #
+    def downcase_all(content)
+      case content
+      when Hash then content.transform_values { |v| downcase_all v }
+      when Array then content.map { |v| downcase_all v }
+      when String then content.downcase
+      else content
+      end
+    end
+  end
+end

data/lib/relaton/bipm/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bipm
+    module Util
+      extend Bib::Util
+      PROGNAME = "relaton-bipm".freeze
+    end
+  end
+end

data/lib/relaton/bipm.rb

@@ -0,0 +1,32 @@
+require "zip"
+require "fileutils"
+require "parslet"
+require "relaton/bib"
+require "relaton/index"
+require "relaton/core"
+require_relative "version"
+require_relative "bipm/util"
+require_relative "bipm/item_data"
+require_relative "bipm/model/item"
+require_relative "bipm/model/bibitem"
+require_relative "bipm/model/bibdata"
+require_relative "bipm/bibliography"
+
+module Relaton
+  module Bipm
+    class Error < StandardError; end
+
+    INDEXFILE = "index-v1.yaml".freeze
+
+    # Returns hash of gems versions used to generate the data model.
+    # @return [String]
+    def grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+
+    extend self
+  end
+end

data/lib/relaton/bsi/bibliography.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Relaton::Bsi
+  # Class methods for search ISO standards.
+  class Bibliography
+    class << self
+      # @param text [String]
+      # @return [Relaton::Bsi::HitCollection]
+      def search(text, year = nil)
+        code = text.sub(/^BSI\s/, "").sub(/ExComm|Expert commentary/, "Ex")
+        HitCollection.search(code, year)
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Algolia::AlgoliaUnreachableHostError => e
+        raise Relaton::RequestError, e.message
+      end
+
+      #
+      # @param code [String] the BSI standard Code to look up
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options
+      # @option opts [Boolean] :all_parts if all-parts reference is required
+      # @option opts [Boolean] :no_year if last published document is required
+      #
+      # @return [String] Relaton XML serialisation of reference
+      def get(code, year = nil, opts = {})
+        # y = code.split(":")[1]
+        year ||= code_parts(code)[:year]
+        ret = bib_get(code, year, opts)
+        return nil if ret.nil?
+
+        ret = ret.to_most_recent_reference unless year || opts[:keep_year]
+        # ret = ret.to_all_parts if opts[:all_parts]
+        ret
+      end
+
+      #
+      # Destruct code to its parts.
+      #
+      # @param [String] code document identifier
+      #
+      # @return [MatchData] parts of the code
+      #
+      def code_parts(code)
+        %r{
+          ^(?:BSI\s)?(?<code>(?:[A-Z]+\s)*[^:\s+]+(?:\s\d+)?)
+          (?::(?<year>\d{4})(?:-\d{2})?)?
+          (?:\+(?<a>[^:\s]+)(?::(?<y>\d{4}))?)?
+          (?:\s(?<rest>.+))?
+        }x.match code
+      end
+
+      private
+
+      def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
+        # y = code_parts(code)[:year]
+        # id = year && !y ? "#{code}:#{year}" : code
+        # Util.warn "WARNING: no match found online for `#{id}`. " \
+        #           "The code must be exactly like it is on the standards website."
+        unless missed_years.empty?
+          Util.info "There was no match for `#{year}`, though there " \
+                    "were matches found for `#{missed_years.join('`, `')}`."
+        end
+        # if /\d-\d/.match? code
+        #   warn "[relaton-bsi] The provided document part may not exist, or "\
+        #     "the document may no longer be published in parts."
+        # else
+        #   warn "[relaton-bsi] If you wanted to cite all document parts for "\
+        #     "the reference, use \"#{code} (all parts)\".\nIf the document "\
+        #     "is not a standard, use its document type abbreviation (TS, TR, "\
+        #     "PAS, Guide)."
+        # end
+        nil
+      end
+
+      #
+      # Search for a BSI standard.
+      #
+      # @param [String] code the BSI standard Code to look up
+      #
+      # @return [Relaton::Bsi::HitCollection] a collection of hits
+      #
+      def search_filter(code)
+        cp = code_parts code
+        Util.info "Fetching from shop.bsigroup.com ...", key: code
+        unless cp
+          Util.info "Could not parse the reference", key: code
+          return []
+        end
+
+        search(code).filter_hits!(cp)
+      end
+
+      # Sort through the results from Isobib, fetching them three at a time,
+      # and return the first result that matches the code,
+      # matches the year (if provided), and which # has a title (amendments do not).
+      # Only expects the first page of results to be populated.
+      # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+      # If no match, returns any years which caused mismatch, for error reporting
+      def results_filter(result, year)
+        missed_years = []
+        result.each do |r|
+          pyear = code_parts(r.hit[:code])[:year]
+          return { ret: r.item } if (!year || year == pyear) && r.item
+
+          missed_years << pyear
+        end
+        { years: missed_years }
+      end
+
+      def bib_get(code, year, _opts)
+        result = search_filter(code) || return
+        ret = results_filter(result, year)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first&.content}`", key: code
+          ret[:ret]
+        else
+          Util.info "Not found", key: code
+          fetch_ref_err(code, year, ret[:years])
+        end
+      end
+    end
+  end
+end

data/lib/relaton/bsi/hit.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Relaton::Bsi
+  # Hit.
+  class Hit < Relaton::Core::Hit
+    attr_writer :fetch
+
+    # Parse page.
+    # @return [Relaton::Bsi::ItemData]
+    def item
+      @item ||= Scraper.parse_page self
+    end
+  end
+end

data/lib/relaton/bsi/hit_collection.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "algolia"
+require_relative "hit"
+
+module Relaton::Bsi
+  # Page of hit collection.
+  class HitCollection < Relaton::Core::HitCollection
+    DOMAIN = "https://shop.bsigroup.com"
+
+    #
+    # Initialize a new HitCollection object and perform search.
+    #
+    # @param ref [String] reference
+    # @param year [String] year
+    #
+    # @return [Relaton::Bsi::HitCollection] hit collection
+    #
+    def self.search(ref, year = nil)
+      new(ref, year).search
+    end
+
+    #
+    # Perform search.
+    # @return [self] hit collection
+    def search
+      resp = algolia_index.search ref # , facetFilters: "product_type:standard"
+      @array = create_hits resp[:hits]
+      self
+    end
+
+    #
+    # Algolia index.
+    #
+    # @return [Algolia::Search::Index] Algolia index
+    #
+
+    def algolia_index
+      @algolia_index ||= begin
+        config = Algolia::Search::Config.new(
+          application_id: "575YE157G9",
+          api_key: "a057b4e74099445df2eddb7940828a10",
+        )
+        client = Algolia::Search::Client.new config, logger: ::Logger.new($stderr)
+        client.init_index "shopify_products"
+      end
+    end
+
+    #
+    # Filter the search results for a BSI standard.
+    #
+    # @param [MatchData] code_parts parts of document identifier
+    #
+    # @return [self] filtered search results
+    #
+    def filter_hits!(code_parts)
+      hits = filter code_parts
+      hits = filter code_parts, skip_rest: true if hits.empty?
+      hits = filter code_parts, drop_amd: true if hits.empty?
+      @array = hits
+      self
+    end
+
+    private
+
+    #
+    # Create hits from search results.
+    #
+    # @param hits [Array<Hash>] search results
+    #
+    # @return [Array<Relaton::Bsi::Hit>] hits
+    #
+    def create_hits(hits) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      hits.each_with_object([]) do |h, obj|
+        next unless h[:meta][:global][:publishedDate]
+
+        code = h[:meta][:global][:primaryDesignator]
+        code = code.is_a?(Array) ? code.first : code
+        code.sub!(/\s?(?:LOOSELEAF|\(A5 LAMINATED\)|-\s?TC$)/, "")
+        obj << Hit.new(
+          {
+            code: code,
+            title: h[:title],
+            url: h[:handle],
+            date: h[:meta][:global][:publishedDate],
+            publisher: h[:meta][:global][:publisher],
+            status: h[:meta][:global][:status],
+            ics: h[:meta][:global][:icsCodesAlgoliaStringArray],
+            doctype: h[:product_type],
+          }, self
+        )
+      end.sort_by { |h| h.hit[:date] }.reverse
+    end
+
+    #
+    # Select hits that match the document identifier.
+    #
+    # @param [MatchData] code_parts parts of document identifier
+    # @param [Boolean] drop_amd drop amendments and corrigendums
+    # @param [Boolean] skip_rest skip rest suffix of document identifier
+    #
+    def filter(code_parts, drop_amd: false, skip_rest: false) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      @array.select do |i|
+        code = drop_amd ? i.hit[:code].sub(/\+[AC]\d+.*$/, "") : i.hit[:code]
+        cp = Bibliography.code_parts code
+        match = cp[:code] == code_parts[:code] && cp[:a] == code_parts[:a] &&
+          (!code_parts[:y] || cp[:y] == code_parts[:y]) &&
+          (skip_rest || cp[:rest] == code_parts[:rest])
+        i.hit[:code] = code if drop_amd && match
+        match
+      end
+    end
+  end
+end

data/lib/relaton/bsi/item_data.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Bsi
+    class ItemData < Iso::ItemData
+      def create_id(without_date: false)
+        docid = docidentifier.find(&:primary) || docidentifier.first
+        return unless docid
+
+        id_content = without_date ? docid.content.sub(/:\d{4}$/, "") : docid.content
+        self.id = id_content.gsub(/\W+/, "")
+      end
+    end
+  end
+end

data/lib/relaton/bsi/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bsi
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/bsi/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bsi
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/bsi/model/docidentifier.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Bsi
+    class Docidentifier < Bib::Docidentifier
+      def remove_date!
+        content&.sub!(/:\d{4}(?:-\d{2})?/, "")
+      end
+    end
+  end
+end

data/lib/relaton/bsi/model/doctype.rb

@@ -0,0 +1,14 @@
+module Relaton
+  module Bsi
+    class Doctype < Bib::Doctype
+      TYPES = %w[
+        british-standard draft-for-development published-document privately-subscribed-standard
+        publicly-available-specification flex-standard international-standard technical-specification
+        technical-report guide international-workshop-agreement industry-technical-agreement
+        standard european-workshop-agreement fast-track-standard expert-commentary
+      ].freeze
+
+      attribute :content, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/bsi/model/ext.rb

@@ -0,0 +1,17 @@
+require_relative "doctype"
+
+module Relaton
+  module Bsi
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :ics, Bib::ICS, collection: true
+      attribute :structuredidentifier, Iso::StructuredIdentifier, collection: true, initialize_empty: true
+      attribute :stagename, Iso::Stagename
+
+      xml { map_element "stagename", to: :stagename }
+      key_value { map_element "stagename", to: :stagename }
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-bsi"]
+    end
+  end
+end

data/lib/relaton/bsi/model/item.rb

@@ -0,0 +1,17 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Bsi
+    class Relation < Bib::Relation
+    end
+
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :relation, Relation, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/bsi/model/item_base.rb

@@ -0,0 +1,22 @@
+require_relative "docidentifier"
+
+module Relaton
+  module Bsi
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :relation, Relation, collection: true, initialize_empty: true
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/bsi/model/relation.rb

@@ -0,0 +1,9 @@
+require_relative "item_base"
+
+module Relaton
+  module Bsi
+    class Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/bsi/processor.rb

@@ -0,0 +1,42 @@
+require "relaton/core/processor"
+
+module Relaton::Bsi
+  class Processor < Relaton::Core::Processor
+    def initialize
+      @short = :relaton_bsi
+      @prefix = "BSI"
+      @defaultprefix = %r{^(BSI|BS|PD)\s}
+      @idtype = "BSI"
+    end
+
+    # @param code [String]
+    # @param date [String, nil] year
+    # @param opts [Hash]
+    # @return [Relaton::Bsi::ItemData]
+    def get(code, date, opts)
+      require_relative "../bsi"
+      ::Relaton::Bsi::Bibliography.get(code, date, opts)
+    end
+
+    # @param xml [String]
+    # @return [Relaton::Bsi::ItemData]
+    def from_xml(xml)
+      require_relative "../bsi"
+      ::Relaton::Bsi::Item.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [Relaton::Bsi::ItemData]
+    def from_yaml(yaml)
+      require_relative "../bsi"
+      ::Relaton::Bsi::Item.from_yaml yaml
+    end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def grammar_hash
+      require_relative "../bsi"
+      @grammar_hash ||= ::Relaton::Bsi.grammar_hash
+    end
+  end
+end