relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/bipm/si_brochure_parser.rb

@@ -0,0 +1,188 @@
+require "nokogiri"
+
+module Relaton::Bipm
+  class SiBrochureParser
+    #
+    # Create new parser
+    #
+    # @param [Relaton::Bipm::DataFetcher] data_fetcher data fetcher
+    #
+    def initialize(data_fetcher)
+      @data_fetcher = WeakRef.new data_fetcher
+    end
+
+    #
+    # Parse documents from SI brochure dataset and write thems to YAML files
+    #
+    # @param [Relaton::Bipm::DataFetcher] data_fetcher data fetcher
+    #
+    def self.parse(data_fetcher)
+      new(data_fetcher).parse
+    end
+
+    #
+    # Parse SI brochure and write them to YAML files
+    #
+    def parse # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # metanorma site generate writes per-document outputs into a subdirectory
+      # named after the source path (e.g. _site/documents/si-brochure/3.01/
+      # si-brochure-en.rxl). The legacy top-level *.rxl glob is kept for
+      # backwards compatibility with any older flow that flattened outputs.
+      si_brochure_rxls.each do |f|
+        puts "Parsing #{f}"
+        xml = File.read(f, encoding: "UTF-8")
+        xml = xml.force_encoding("UTF-8") if xml.encoding != Encoding::UTF_8
+        item1 = Bibdata.from_xml(xml)
+        # Workaround for relaton-bib Version#content bug: whitespace between
+        # legacy <revision-date>/<draft> children gets captured as @content,
+        # blocking the legacy-fold path. Clear it so the getter recomputes.
+        item1.version.each do |v|
+          c = v.instance_variable_get(:@content)
+          next unless c.is_a?(Array) || (c.is_a?(String) && c.strip.empty?)
+          v.instance_variable_set(:@content, nil)
+        end
+        @data_fetcher.errors[:si_brochure_title] &&= item1.title.empty?
+        @data_fetcher.errors[:si_brochure_docidentifier] &&= item1.docidentifier.empty?
+        unless has_committee_contributor?(item1)
+          contribs = extract_editorialgroup(xml)
+          contribs.each { |c| item1.contributor << c }
+        end
+        fix_si_brochure_id item1
+        basename = File.join @data_fetcher.output, File.basename(f).sub(/(?:-(?:en|fr))?\.rxl$/, "")
+        outfile = "#{basename}.#{@data_fetcher.ext}"
+        # Index by the primary docidentifier content ("BIPM SI Brochure 9e
+        # v3.01 (2019/2024, E)"), which `Pubid::Bipm` parses — unlike the bare
+        # `docnumber` ("SI Brochure 9e …", no "BIPM" prefix), which it rejects.
+        key = primary_id(item1) || item1.docnumber || basename
+        @data_fetcher.add_to_index key, outfile
+        item =
+          if File.exist? outfile
+            warn_duplicate = false
+            item2 = Item.from_yaml File.read(outfile, encoding: "UTF-8")
+            fix_si_brochure_id item2
+            hash1 = YAML.safe_load item1.to_yaml
+            hash2 = YAML.safe_load item2.to_yaml
+            Item.from_yaml deep_merge(hash1, hash2).to_yaml
+          else
+            warn_duplicate = true
+            item1
+          end
+        @data_fetcher.write_file outfile, item, warn_duplicate: warn_duplicate
+        puts "Saved to #{outfile}"
+      end
+    end
+
+    #
+    # @return [Array<String>] paths to SI Brochure RXL files. Looks at the
+    #   legacy flat layout first, then the metanorma-cli subdirectory layout
+    #   (`<source_path_without_'sources'>/<doc>.rxl`) used by current
+    #   `metanorma site generate` output.
+    #
+    def si_brochure_rxls
+      flat = Dir["bipm-si-brochure/_site/documents/*.rxl"]
+      return flat if flat.any?
+
+      Dir["bipm-si-brochure/_site/documents/**/si-brochure-{en,fr}.rxl"]
+    end
+
+    #
+    # Update ID of SI brochure
+    #
+    # @param [ItemData] item bibliographic item
+    #
+    # @return [void]
+    #
+    def fix_si_brochure_id(item)
+      # isbn = hash["docid"].detect { |id| id["type"] == "ISBN" }
+      # num = isbn && isbn["id"] == "978-92-822-2272-0" ?  "SI Brochure" : "SI Brochure, Appendix 4"
+
+      update_id item
+
+      prid = primary_id item
+      if item.docnumber
+        item.docnumber.sub!(/^Brochure(?:\sConcise|\sFAQ)?$/i, prid.sub(/^BIPM\s/, ""))
+      else
+        item.docnumber = prid.sub(/^BIPM\s/, "")
+      end
+      item.id = prid.gsub(/[,\s-]/, "")
+    end
+
+    def update_id(item)
+      item.docidentifier.each do |id|
+        next unless id.type == "BIPM" && id.content&.match?(/BIPM Brochure/i)
+
+        id.primary = true
+        id.content.sub!(/(?<=^BIPM\s)(Brochure)/i, "SI \\1")
+      end
+    end
+
+    def primary_id(item)
+      item.docidentifier.detect do |id|
+        id.primary && (id.language == "en" || id.language.nil?)
+      end.content
+    end
+
+    def has_committee_contributor?(item)
+      item.contributor.any? do |c|
+        c.role.any? { |r| r.description.any? { |d| d.content == "committee" } }
+      end
+    end
+
+    def extract_editorialgroup(xml)
+      doc = Nokogiri::XML(xml)
+      doc.xpath("//editorialgroup/committee").map do |committee|
+        acronym = committee["acronym"]
+        names = committee.xpath("variant").map do |v|
+          Relaton::Bib::TypedLocalizedString.new(
+            content: v.text, language: v["language"], script: v["script"],
+          )
+        end
+        subdiv = Relaton::Bib::Subdivision.new(
+          type: "committee", name: names,
+          abbreviation: Relaton::Bib::LocalizedString.new(content: acronym),
+        )
+        bipm_name = [Relaton::Bib::TypedLocalizedString.new(content: acronym)]
+        org = Relaton::Bib::Organization.new(name: bipm_name, subdivision: [subdiv])
+        desc = Relaton::Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Relaton::Bib::Contributor::Role.new(type: "author", description: [desc])
+        Relaton::Bib::Contributor.new(organization: org, role: [role])
+      end
+    end
+
+    #
+    # Deep merge two hashes
+    #
+    # @param [Hash] hash1
+    # @param [Hash] hash2
+    #
+    # @return [Hash] Merged hash
+    #
+    def deep_merge(hash1, hash2) # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+      hash1.merge(hash2) do |_, oldval, newval|
+        if oldval.is_a?(Hash) && newval.is_a?(Hash)
+          deep_merge(oldval, newval)
+        elsif oldval.is_a?(Array) && newval.is_a?(Array)
+          (oldval + newval).uniq { |i| downcase_all i }
+        else
+          newval || oldval
+        end
+      end
+    end
+
+    #
+    # Downcase all values in hash or array
+    #
+    # @param [Array, Hash, String] content hash, array or string
+    #
+    # @return [Array, Hash, String] hash, array or string with downcased values
+    #
+    def downcase_all(content)
+      case content
+      when Hash then content.transform_values { |v| downcase_all v }
+      when Array then content.map { |v| downcase_all v }
+      when String then content.downcase
+      else content
+      end
+    end
+  end
+end

data/lib/relaton/bipm/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bipm
+    module Util
+      extend Bib::Util
+      PROGNAME = "relaton-bipm".freeze
+    end
+  end
+end

data/lib/relaton/bipm.rb

@@ -0,0 +1,38 @@
+require "zip"
+require "fileutils"
+require "parslet"
+require "relaton/bib"
+require "relaton/index"
+require "relaton/core"
+require "pubid"
+require_relative "version"
+require_relative "bipm/util"
+require_relative "bipm/id_parser"
+require_relative "bipm/item_data"
+require_relative "bipm/model/item"
+require_relative "bipm/model/bibitem"
+require_relative "bipm/model/bibdata"
+require_relative "bipm/bibliography"
+
+module Relaton
+  module Bipm
+    class Error < StandardError; end
+
+    # The runtime index is the pubid-backed `index-v2` (`_type: pubid:bipm:*`).
+    # The legacy bespoke `{group,type,number,year}` `index-v1` is still published
+    # by `relaton-data-bipm`'s crawler (using the retained `Relaton::Bipm::Id`),
+    # but this flavor no longer reads or writes it.
+    INDEXFILE = "index-v2".freeze
+
+    # Returns hash of gems versions used to generate the data model.
+    # @return [String]
+    def grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+
+    extend self
+  end
+end

data/lib/relaton/bsi/bibliography.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Relaton::Bsi
+  # Class methods for search ISO standards.
+  class Bibliography
+    # pubid identifier types referenced when navigating the parsed object graph.
+    Ids = ::Pubid::Bsi::Identifiers
+
+    class << self
+      # @param text [String]
+      # @return [Relaton::Bsi::HitCollection]
+      def search(text, year = nil)
+        code = text.sub(/^BSI\s/, "").sub(/ExComm|Expert commentary/, "Ex")
+        HitCollection.search(code, year)
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Algolia::AlgoliaUnreachableHostError => e
+        raise Relaton::RequestError, e.message
+      end
+
+      #
+      # @param code [String] the BSI standard Code to look up
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options
+      # @option opts [Boolean] :all_parts if all-parts reference is required
+      # @option opts [Boolean] :no_year if last published document is required
+      #
+      # @return [String] Relaton XML serialisation of reference
+      def get(code, year = nil, opts = {})
+        year ||= publication_year(code)
+        ret = bib_get(code, year, opts)
+        return nil if ret.nil?
+
+        ret = ret.to_most_recent_reference unless year || opts[:keep_year]
+        ret
+      end
+
+      # Parse a BSI reference with pubid. An unrecognized reference raises;
+      # like ISO and 3GPP we let it propagate -- relaton-cli rescues
+      # Pubid::Errors::Error and renders "... is not a recognized standards
+      # identifier". A catalogue hit code is a different case: `Hit#pubid`
+      # rescues, because one odd Algolia row must not abort the search.
+      # @param [String] code document identifier
+      # @return [Pubid::Bsi::Identifier]
+      # @raise [Pubid::Errors::ParseError]
+      def parse(code)
+        ::Pubid::Bsi::Identifier.parse(code)
+      end
+
+      # Publication year of a reference's base document.
+      # @param [String] code document identifier
+      # @return [String, nil]
+      def publication_year(code)
+        year_of parse(code)
+      end
+
+      # Whether a query and a candidate hit denote the same BSI document, at the
+      # fuzziness the hit filter asks for. Built entirely on pubid: `base_document`
+      # peels supplements/wrappers, `matches?` compares ignoring the publication
+      # date, and the `Amendment`/`Corrigendum` `#supplement_type`, `#number` and
+      # `#year` supply the amendment type, number and year.
+      #
+      # @param query [Pubid::Bsi::Identifier]
+      # @param hit   [Pubid::Bsi::Identifier]
+      # @param skip_rest [Boolean] ignore the free-text suffix (ExComm / Flex version)
+      # @param drop_amd  [Boolean] reduce the hit to its base document before
+      #   matching, so a query for the base document matches a catalogue entry
+      #   that only exists in amended form (the returned id is de-amended too)
+      # @return [Boolean]
+      def same_reference?(query, hit, skip_rest: false, drop_amd: false)
+        hit = hit.base_document if drop_amd
+
+        base_match?(query, hit) &&
+          same_supplement?(supplement_of(query), supplement_of(hit)) &&
+          (skip_rest || rest_marker(query) == rest_marker(hit))
+      end
+
+      private
+
+      # The year of the dated document at the root of a reference. pubid holds
+      # an adopted standard (`BS EN ISO 8848:2021`) in `base`, so its
+      # `base_document.year` is nil; `#root` reaches the dated document for
+      # every form, on the pubid before that change and after it.
+      # @param id [Pubid::Bsi::Identifier, nil]
+      # @return [String, nil]
+      def year_of(id)
+        id&.root&.year&.to_s
+      end
+
+      # Same base document, ignoring the publication date. `:month` is excluded
+      # alongside `:date` because Flex identifiers store the month separately, so
+      # `exclude(:date)` alone leaves them unequal.
+      def base_match?(query, hit)
+        query.base_document.matches?(hit.base_document, ignore: %i[date month])
+      end
+
+      # The `Amendment`/`Corrigendum` supplement of a reference, if any — peeling
+      # an Expert-commentary wrapper first. Both supplements expose
+      # `#supplement_type`, `#number` and `#year`.
+      def supplement_of(id)
+        while id.is_a?(Ids::ExpertCommentary)
+          inner = id.base
+          break if inner.nil? || inner.equal?(id)
+
+          id = inner
+        end
+        return nil unless id.respond_to?(:identifiers) && id.identifiers
+
+        id.identifiers.find { |i| i.respond_to?(:supplement_type) }
+      end
+
+      # Same supplement, with the amendment year treated as optional when the
+      # query omits it (a query for "…+A2" matches a hit for "…+A2:2020").
+      def same_supplement?(query_supp, hit_supp)
+        return hit_supp.nil? if query_supp.nil?
+        return false if hit_supp.nil?
+
+        query_supp.supplement_type == hit_supp.supplement_type &&
+          supplement_number(query_supp) == supplement_number(hit_supp) &&
+          (supplement_year(query_supp).nil? ||
+            supplement_year(query_supp) == supplement_year(hit_supp))
+      end
+
+      # The number and the year of a supplement. pubid removes the
+      # `#supplement_number` / `#supplement_year` aliases and stores the values
+      # in `#number` / `#year`. An older pubid still answers the aliases and
+      # leaves `#number` / `#year` nil, so the alias wins while it exists.
+      # TODO: read `#number` / `#year` directly once pubid `main` has no alias.
+      def supplement_number(supp)
+        supp.respond_to?(:supplement_number) ? supp.supplement_number : supp.number
+      end
+
+      def supplement_year(supp)
+        supp.respond_to?(:supplement_year) ? supp.supplement_year : supp.year
+      end
+
+      # Canonical marker for the free-text suffix pubid parses into a typed
+      # wrapper (so an `ExComm` query matches an `Expert Commentary` hit).
+      def rest_marker(id)
+        return :expert_commentary if id.is_a?(Ids::ExpertCommentary)
+        return :"flex_#{id.edition}" if id.is_a?(Ids::Flex) && id.edition
+
+        nil
+      end
+
+      def fetch_ref_err(_code, year, missed_years)
+        unless missed_years.empty?
+          Util.info "There was no match for `#{year}`, though there " \
+                    "were matches found for `#{missed_years.join('`, `')}`."
+        end
+        nil
+      end
+
+      #
+      # Search for a BSI standard.
+      #
+      # @param [String] code the BSI standard Code to look up
+      #
+      # @return [Relaton::Bsi::HitCollection] a collection of hits
+      #
+      def search_filter(code)
+        query = parse(code)
+        Util.info "Fetching from shop.bsigroup.com ...", key: code
+        search(code).filter_hits!(query)
+      end
+
+      # Sort through the results, and return the first result that matches the
+      # code and matches the year (if provided), and which has a title.
+      # If no match, returns any years which caused mismatch, for error reporting.
+      def results_filter(result, year)
+        missed_years = []
+        result.each do |r|
+          # The hit's own parse: a kept hit always parsed, and its base
+          # document's year is the same after a `drop_amd` rewrite.
+          pyear = year_of r.pubid
+          return { ret: r.item } if (!year || year == pyear) && r.item
+
+          missed_years << pyear
+        end
+        { years: missed_years }
+      end
+
+      def bib_get(code, year, _opts)
+        result = search_filter(code) || return
+        ret = results_filter(result, year)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first&.content}`", key: code
+          ret[:ret]
+        else
+          Util.info "Not found", key: code
+          fetch_ref_err(code, year, ret[:years])
+        end
+      end
+    end
+  end
+end

data/lib/relaton/bsi/hit.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Relaton::Bsi
+  # Hit.
+  class Hit < Relaton::Core::Hit
+    attr_writer :fetch
+
+    # Parse page.
+    # @return [Relaton::Bsi::ItemData]
+    def item
+      @item ||= Scraper.parse_page self
+    end
+
+    # The hit's reference parsed with pubid, memoized so the (up to three) hit
+    # filter passes don't re-parse it. `nil` when pubid can't parse the code:
+    # the Algolia catalogue carries codes like `BS 381C SET:1996 (R2002)`, and
+    # one such row must not abort the search. (A malformed *query* raises; see
+    # `Bibliography.parse`.)
+    # @return [Pubid::Bsi::Identifier, nil]
+    def pubid
+      return @pubid if defined?(@pubid)
+
+      @pubid = Bibliography.parse(hit[:code])
+    rescue StandardError
+      @pubid = nil
+    end
+  end
+end

data/lib/relaton/bsi/hit_collection.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "algolia"
+require_relative "hit"
+
+module Relaton::Bsi
+  # Page of hit collection.
+  class HitCollection < Relaton::Core::HitCollection
+    DOMAIN = "https://shop.bsigroup.com"
+
+    #
+    # Initialize a new HitCollection object and perform search.
+    #
+    # @param ref [String] reference
+    # @param year [String] year
+    #
+    # @return [Relaton::Bsi::HitCollection] hit collection
+    #
+    def self.search(ref, year = nil)
+      new(ref, year).search
+    end
+
+    #
+    # Perform search.
+    # @return [self] hit collection
+    def search
+      resp = algolia_index.search ref # , facetFilters: "product_type:standard"
+      @array = create_hits resp[:hits]
+      self
+    end
+
+    #
+    # Algolia index.
+    #
+    # @return [Algolia::Search::Index] Algolia index
+    #
+
+    def algolia_index
+      @algolia_index ||= begin
+        config = Algolia::Search::Config.new(
+          application_id: "575YE157G9",
+          api_key: "a057b4e74099445df2eddb7940828a10",
+        )
+        client = Algolia::Search::Client.new config, logger: ::Logger.new($stderr)
+        client.init_index "shopify_products"
+      end
+    end
+
+    #
+    # Filter the search results for a BSI standard.
+    #
+    # @param [Pubid::Bsi::Identifier] query the parsed query reference
+    #
+    # @return [self] filtered search results
+    #
+    def filter_hits!(query)
+      hits = filter query
+      hits = filter query, skip_rest: true if hits.empty?
+      hits = filter query, drop_amd: true if hits.empty?
+      @array = hits
+      self
+    end
+
+    private
+
+    #
+    # Create hits from search results.
+    #
+    # @param hits [Array<Hash>] search results
+    #
+    # @return [Array<Relaton::Bsi::Hit>] hits
+    #
+    def create_hits(hits) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      hits.each_with_object([]) do |h, obj|
+        next unless h[:meta][:global][:publishedDate]
+
+        code = h[:meta][:global][:primaryDesignator]
+        code = code.is_a?(Array) ? code.first : code
+        code.sub!(/\s?(?:LOOSELEAF|\(A5 LAMINATED\)|-\s?TC$)/, "")
+        obj << Hit.new(
+          {
+            code: code,
+            title: h[:title],
+            url: h[:handle],
+            date: h[:meta][:global][:publishedDate],
+            publisher: h[:meta][:global][:publisher],
+            status: h[:meta][:global][:status],
+            ics: h[:meta][:global][:icsCodesAlgoliaStringArray],
+            doctype: h[:product_type],
+          }, self
+        )
+      end.sort_by { |h| h.hit[:date] }.reverse
+    end
+
+    #
+    # Select hits that match the query reference.
+    #
+    # @param [Pubid::Bsi::Identifier] query the parsed query reference
+    # @param [Boolean] drop_amd match base documents only (ignore supplements)
+    # @param [Boolean] skip_rest ignore the free-text suffix (ExComm / Flex version)
+    #
+    def filter(query, drop_amd: false, skip_rest: false)
+      @array.select do |i|
+        hit = i.pubid or next false
+        match = Bibliography.same_reference?(query, hit, skip_rest: skip_rest, drop_amd: drop_amd)
+        # When matching base documents only, drop the amendment from the hit's
+        # rendered id too, so the returned reference is the base document.
+        i.hit[:code] = hit.base_document.to_s if drop_amd && match
+        match
+      end
+    end
+  end
+end

data/lib/relaton/bsi/item_data.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Bsi
+    class ItemData < Iso::ItemData
+      def create_id(without_date: false)
+        docid = docidentifier.find(&:primary) || docidentifier.first
+        return unless docid
+
+        id_content = without_date ? docid.content.sub(/:\d{4}$/, "") : docid.content
+        self.id = id_content.gsub(/\W+/, "")
+      end
+    end
+  end
+end

data/lib/relaton/bsi/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bsi
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/bsi/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bsi
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end