relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/iso/data_parser.rb

@@ -0,0 +1,458 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "../iso"
+require_relative "scraper"
+
+module Relaton
+  module Iso
+    #
+    # Parses one ISO Open Data record (`iso_deliverables_metadata.jsonl` line)
+    # into an `Relaton::Iso::ItemData`.
+    #
+    # See https://www.iso.org/open-data.html for the field reference.
+    #
+    class DataParser
+      ATTRS = %i[
+        type docidentifier docnumber edition language script title status ics
+        date contributor abstract copyright source relation place
+        structuredidentifier ext
+      ].freeze
+
+      DOCTYPES = {
+        "IS" => "international-standard",
+        "TS" => "technical-specification",
+        "TR" => "technical-report",
+        "PAS" => "publicly-available-specification",
+        "GUIDE" => "guide",
+        "IWA" => "international-workshop-agreement",
+        "R" => "recommendation",
+        "ISP" => "international-standard",
+        "DATA" => "international-standard",
+        "TTA" => "international-standard",
+      }.freeze
+
+      SUPPLEMENT_DOCTYPES = {
+        "Amd" => "amendment",
+        "Cor" => "technical-corrigendum",
+        "Add" => "addendum",
+        "Suppl" => "supplement",
+        "Ext" => "extract",
+      }.freeze
+
+      DOC_URL = "https://www.iso.org/standard/%d.html"
+      OBP_URL = "https://www.iso.org/obp/ui/en/#!iso:std:%d:en"
+      RSS_URL = "https://www.iso.org/contents/data/standard/%s/%s/%d.detail.rss"
+
+      #
+      # @param [Hash] pub one Open Data record
+      # @param [Hash{Integer=>String}] ref_index map of Open Data `id` ->
+      #   `reference`, used to resolve `replaces` / `replacedBy` (which are
+      #   numeric IDs in the source).
+      # @param [Hash] errors error accumulator (`Hash.new(true)`); fields are
+      #   AND-ed across all records by the `report_errors` machinery.
+      # @param [Hash{String=>Hash}] tc_index map of TC/SC reference ->
+      #   `{ "en" => title, "fr" => title }`, used to resolve the human
+      #   committee label from the Open Data technical-committees dataset.
+      # @param [Hash{String=>Array<String>}] amend_index map of base
+      #   reference -> list of supplement (Amd/Cor/Add) references that
+      #   target it. Open Data records the supplement -> base direction only
+      #   via the reference string, so we pre-build the reverse map.
+      # @param [Hash{String=>String}] date_index map of reference ->
+      #   `publicationDate`, used to attach a `published` date to each
+      #   emitted relation's bibitem when the related document is itself
+      #   present in the Open Data feed.
+      #
+      def initialize(pub, ref_index = {}, errors = {}, tc_index = {}, amend_index = {}, date_index = {})
+        @pub = pub
+        @ref_index = ref_index
+        @errors = errors
+        @tc_index = tc_index
+        @amend_index = amend_index
+        @date_index = date_index
+      end
+
+      def parse
+        ItemData.new(**ATTRS.each_with_object({}) { |a, h| h[a] = send(a) })
+      end
+
+      private
+
+      def type = "standard"
+
+      # ---- identifiers -----------------------------------------------------
+
+      def reference
+        @reference ||= @pub["reference"] || ""
+      end
+
+      def pubid
+        return @pubid if defined?(@pubid)
+
+        @pubid = begin
+          ::Pubid::Iso::Identifier.parse(reference)
+        rescue StandardError => e
+          Util.warn "Failed to parse pubid `#{reference}`: #{e.message}"
+          nil
+        end
+      end
+
+      def docidentifier
+        ids = []
+        if pubid
+          primary = Docidentifier.new(content: pubid, type: "ISO", primary: true)
+          ids << primary
+          if (undated = undated_docid(primary))
+            ids << undated
+          end
+          if (ref = iso_reference_pubid)
+            ids << Docidentifier.new(content: ref, type: "iso-reference")
+          end
+          if (urn = safe_urn_docid)
+            ids << urn
+          end
+        else
+          ids << Docidentifier.new(content: reference, type: "ISO", primary: true)
+        end
+        @errors[:docidentifier] &&= ids.empty?
+        ids
+      end
+
+      # Build the undated reference docid (e.g. `ISO 10303-52` from
+      # `ISO 10303-52:2011`). Returns nil when the reference carries no year
+      # (the undated form would just duplicate the primary).
+      def undated_docid(primary)
+        undated = Docidentifier.new(content: pubid, type: "iso-undated")
+        undated unless undated.to_s == primary.to_s
+      rescue StandardError
+        nil
+      end
+
+      def safe_urn_docid
+        return nil unless urn_pubid
+
+        Docidentifier.new(content: urn_pubid, type: "URN")
+      rescue StandardError
+        nil
+      end
+
+      def iso_reference_pubid
+        pubid.dup.tap do |id|
+          id.languages = [::Pubid::Components::Language.new(code: "en", original_code: "E")]
+        end
+      rescue StandardError
+        nil
+      end
+
+      def urn_pubid
+        return @urn_pubid if defined?(@urn_pubid)
+
+        @urn_pubid = begin
+          # Override stage even when the parsed pubid carries the default
+          # "published" stage — relaton's currentStage (e.g. 9092 = Withdrawn)
+          # is the authoritative source for URN stage.
+          stage_dotted ? pubid.with_harmonized_stage(stage_dotted) : pubid.dup
+        rescue StandardError
+          nil
+        end
+      end
+
+      def docnumber
+        pubid&.to_s&.match(/\d+/)&.to_s
+      end
+
+      def edition
+        return nil unless @pub["edition"]
+
+        Bib::Edition.new(content: @pub["edition"].to_s)
+      end
+
+      # ---- language / script ----------------------------------------------
+
+      def language
+        langs = Array(@pub["languages"]).dup
+        langs << "en" if langs.empty?
+        langs.uniq
+      end
+
+      def script
+        language.filter_map { |l| script_for(l) }.uniq
+      end
+
+      def script_for(lang)
+        case lang
+        when "en", "fr" then "Latn"
+        when "ru" then "Cyrl"
+        end
+      end
+
+      # ---- title -----------------------------------------------------------
+
+      def title
+        result = []
+        result += titles_for("en")
+        result += titles_for("fr")
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def titles_for(lang)
+        raw = @pub.dig("title", lang)
+        return [] if raw.nil? || raw.empty?
+
+        Bib::Title.from_string(normalize_dashes(raw), lang, script_for(lang))
+      end
+
+      def normalize_dashes(str)
+        str.gsub(/\s—\s/, " - ").gsub(/\s–\s/, " - ")
+      end
+
+      # ---- status ----------------------------------------------------------
+
+      # Open Data exposes a 4-digit stage code (e.g. 2098 = 20.98, 6060 = 60.60).
+      # Records occasionally come through with 2 or 3 digits (zero-padded).
+      def stage_dotted
+        return @stage_dotted if defined?(@stage_dotted)
+
+        @stage_dotted =
+          if @pub["currentStage"]
+            digits = format("%04d", @pub["currentStage"].to_i)
+            "#{digits[0, 2]}.#{digits[2, 2]}"
+          end
+      end
+
+      def status
+        return nil unless stage_dotted
+
+        stg, sub = stage_dotted.split(".")
+        Bib::Status.new(
+          stage: Bib::Status::Stage.new(content: stg),
+          substage: sub ? Bib::Status::Stage.new(content: sub) : nil,
+        )
+      end
+
+      # ---- ICS -------------------------------------------------------------
+
+      def ics
+        return [] unless @pub["icsCode"]
+
+        Array(@pub["icsCode"]).map do |code|
+          info = safe_isoics_fetch(code)
+          Bib::ICS.new(code: code, text: info&.description)
+        end
+      end
+
+      def safe_isoics_fetch(code)
+        Isoics.fetch code
+      rescue StandardError
+        nil
+      end
+
+      # ---- dates -----------------------------------------------------------
+
+      def date
+        pd = @pub["publicationDate"]
+        return [] if pd.nil? || pd.empty?
+
+        [Bib::Date.new(type: "published", at: pd)]
+      end
+
+      # ---- contributors ----------------------------------------------------
+
+      def contributor
+        publishers + Array(editorialgroup_contributor)
+      end
+
+      def publishers
+        reference.sub(/\s.*/, "").split("/").filter_map do |abbrev|
+          info = Scraper::PUBLISHERS[abbrev]
+          next unless info
+
+          name = Bib::TypedLocalizedString.new(content: info[:name])
+          abbr = Bib::LocalizedString.new(content: abbrev)
+          uri = Bib::Uri.new(content: info[:uri]) if info[:uri]
+          org = Bib::Organization.new(name: [name], abbreviation: abbr, uri: [uri].compact)
+          role = Bib::Contributor::Role.new(type: "publisher")
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+      end
+
+      def editorialgroup_contributor
+        wg = @pub["ownerCommittee"]
+        return nil if wg.nil? || wg.empty?
+
+        parts = wg.split("/")
+        prefix = parts[0]
+        type = parts[1]&.match(/^[A-Z]+/)&.to_s || "TC"
+
+        publisher = Scraper::PUBLISHERS[prefix]
+        name = if publisher
+                 [Bib::TypedLocalizedString.new(content: publisher[:name])]
+               elsif prefix
+                 [Bib::TypedLocalizedString.new(content: prefix)]
+               else
+                 []
+               end
+        abbreviation = (Bib::LocalizedString.new(content: prefix) if prefix)
+
+        label = @tc_index.dig(wg, "en") || wg
+        subdivision = Bib::Subdivision.new(
+          type: "technical-committee",
+          subtype: type,
+          name: [Bib::TypedLocalizedString.new(content: label)],
+          identifier: [Bib::OrganizationType::Identifier.new(content: wg)],
+        )
+
+        role = Bib::Contributor::Role.new(
+          type: "author",
+          description: [Bib::LocalizedMarkedUpString.new(content: "committee")],
+        )
+
+        Bib::Contributor.new(
+          role: [role],
+          organization: Bib::Organization.new(
+            name: name, subdivision: [subdivision], abbreviation: abbreviation,
+          ),
+        )
+      end
+
+      # ---- abstract --------------------------------------------------------
+
+      def abstract
+        %w[en fr].filter_map do |lang|
+          html = @pub.dig("scope", lang)
+          next if html.nil? || html.empty?
+
+          text = strip_html(html)
+          next if text.empty?
+
+          Bib::Abstract.new(content: text, language: lang, script: script_for(lang))
+        end
+      end
+
+      def strip_html(html)
+        Nokogiri::HTML.fragment(html).text.strip.gsub(/\s+/, " ")
+      end
+
+      # ---- copyright -------------------------------------------------------
+
+      def copyright
+        from = reference[/(?<=:)\d{4}/] ||
+               @pub["publicationDate"]&.match(/\d{4}/)&.to_s
+        return [] unless from && !from.empty?
+
+        owner_name = reference.match(/.*?(?=\s)/).to_s
+        name = Bib::TypedLocalizedString.new(content: owner_name)
+        org = Bib::Organization.new(name: [name])
+        contrib = Bib::ContributionInfo.new(organization: org)
+        [Bib::Copyright.new(owner: [contrib], from: from)]
+      end
+
+      # ---- source links ----------------------------------------------------
+
+      def source
+        id = @pub["id"]
+        return [] unless id
+
+        pad = format("%06d", id)
+        [
+          Bib::Uri.new(type: "src", content: format(DOC_URL, id)),
+          Bib::Uri.new(type: "obp", content: format(OBP_URL, id)),
+          Bib::Uri.new(type: "rss", content: format(RSS_URL, pad[0, 2], pad[2, 2], id)),
+        ]
+      end
+
+      # ---- relations -------------------------------------------------------
+
+      # Open Data semantics:
+      #   * `replaces`   - older docs THIS one supersedes -> `obsoletes`
+      #   * `replacedBy` - newer docs that supersede THIS one -> `obsoletedBy`
+      # Amendments/corrigenda/addenda are stitched in via two routes:
+      #   * on the BASE record, look up `@amend_index` for supplements
+      #     targeting it (-> `updatedBy`); the index is pre-built in
+      #     `DataFetcher#build_ref_index` because Open Data only records
+      #     the supplement -> base direction via the reference string.
+      #   * on the SUPPLEMENT record itself, derive the base from
+      #     `pubid.base` and emit the forward `updates` relation.
+      def relation
+        rels = []
+        rels += build_relations(@pub["replaces"], "obsoletes")
+        rels += build_relations(@pub["replacedBy"], "obsoletedBy")
+        rels += amendment_relations
+        rels += base_relation
+        rels
+      end
+
+      def build_relations(ids, type)
+        Array(ids).filter_map do |id|
+          ref = @ref_index[id] || @ref_index[id.to_s]
+          next unless ref
+
+          relation_for(ref, type)
+        end
+      end
+
+      def amendment_relations
+        Array(@amend_index[pubid&.to_s || reference]).map do |amend_ref|
+          relation_for(amend_ref, "updatedBy")
+        end
+      end
+
+      def base_relation
+        return [] unless pubid&.base
+
+        [relation_for(pubid.base.to_s, "updates")]
+      end
+
+      def relation_for(ref, type)
+        docid = Docidentifier.new(content: ref, type: "ISO", primary: true)
+        attrs = {
+          docidentifier: [docid],
+          formattedref: Bib::Formattedref.new(content: ref),
+        }
+        if (pub_date = @date_index[ref]) && !pub_date.empty?
+          attrs[:date] = [Bib::Date.new(type: "published", at: pub_date)]
+        end
+        Relation.new(type: type, bibitem: ItemData.new(**attrs))
+      end
+
+      # ---- structured identifier ------------------------------------------
+
+      def structuredidentifier
+        return nil unless @pub["id"]
+
+        pnum = ProjectNumber.new(content: @pub["id"].to_s)
+        publisher = pubid&.respond_to?(:publisher) ? pubid.publisher : nil
+        StructuredIdentifier.new(project_number: pnum, type: publisher || "ISO")
+      end
+
+      # ---- place -----------------------------------------------------------
+
+      def place
+        [Bib::Place.new(city: "Geneva")]
+      end
+
+      # ---- ext -------------------------------------------------------------
+
+      def ext
+        Ext.new(
+          doctype: doctype,
+          flavor: "iso",
+          ics: ics,
+          structuredidentifier: structuredidentifier,
+          stagename: nil,
+          updates_document_type: nil,
+          fast_track: nil,
+          price_code: nil,
+        )
+      end
+
+      def doctype
+        type = SUPPLEMENT_DOCTYPES[@pub["supplementType"]] ||
+               DOCTYPES[@pub["deliverableType"]] ||
+               "international-standard"
+        Doctype.new(content: type)
+      end
+    end
+  end
+end

data/lib/relaton/iso/hash_parser_v1.rb

@@ -0,0 +1,177 @@
+require "relaton/bib/hash_parser_v1"
+require_relative "../iso"
+
+module Relaton
+  module Iso
+    #
+    # This module is used to parse hash data from Relaton YAML version 1 files.
+    # It needs for trasition form Relaton v! to Relaton v2.
+    #
+    module HashParserV1
+      include Bib::HashParserV1
+      extend self
+
+      PUBLISHERS = {
+        "IEC" => "International Electrotechnical Commission",
+        "ISO" => "International Organization for Standardization",
+        "IEEE" => "Institute of Electrical and Electronics Engineers",
+        "SAE" => "SAE International",
+        "CIE" => " International Commission on Illumination",
+        "ASME" => "American Society of Mechanical Engineers",
+      }.freeze
+
+      private
+
+      def ext_hash_to_bib(ret) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+        ret[:ext] ||= {}
+        ret[:ext][:schema_version] = ret[:ext].delete(:"schema-version")
+        doctype_hash_to_bib ret
+        ret[:ext][:subdoctype] = ret.delete(:subdoctype) if ret[:subdoctype]
+        ret[:ext][:flavor] ||= flavor(ret)
+        ret[:ext][:horizontal] = ret.delete(:horizontal) unless ret[:horizontal].nil?
+        editorialgroup_hash_to_bib ret
+        approvalgroup_hash_to_bib ret
+        ics_hash_to_bib ret
+        structuredidentifier_hash_to_bib ret
+        stagename_hash_to_bib ret
+        ret[:ext][:fast_track] = ret.delete(:fast_track) unless ret[:fast_track].nil?
+        ret[:ext][:price_code] = ret.delete(:price_code) if ret[:price_code]
+        ret[:ext] = Ext.new(**ret[:ext]) if ret[:ext]
+      end
+
+      def create_docid(**args)
+        Docidentifier.new(**args)
+      end
+
+      #
+      # Ovverides superclass's method
+      #
+      # @param item [Hash]
+      # @retirn [RelatonIsoBib::IsoBibliographicItem]
+      def bib_item(item)
+        ItemData.new(**item)
+      end
+
+      #
+      # Ovverides superclass's method
+      #
+      # @param title [Hash]
+      # @return [RelatonBib::TypedTitleString]
+      def typed_title_strig(title)
+        Relaton::Bib::Title.new(**title)
+      end
+
+      # @param ret [Hash]
+      def editorialgroup_hash_to_bib(ret) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        eg = ret.dig(:ext, :editorialgroup) || ret[:editorialgroup]
+        return unless eg
+
+        ret[:ext]&.delete(:editorialgroup)
+        ret.delete(:editorialgroup)
+        ret[:contributor] ||= []
+        add_group_contributors(ret, eg, "committee")
+      end
+
+      def approvalgroup_hash_to_bib(ret) # rubocop:disable Metrics/AbcSize
+        ag = ret.dig(:ext, :approvalgroup) || ret[:approvalgroup]
+        return unless ag
+
+        ret[:ext]&.delete(:approvalgroup)
+        ret.delete(:approvalgroup)
+        ret[:contributor] ||= []
+        add_group_contributors(ret, ag, "authorizer", role_type: "authorizer")
+      end
+
+      def add_group_contributors(ret, group, description, role_type: "author") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+        subdiv_types = {
+          technical_committee: "technical-committee",
+          subcommittee: "subcommittee",
+          workgroup: "workgroup",
+        }
+        subdiv_types.each do |key, subdiv_type|
+          array(group[key]).each do |wg|
+            wg[:content] ||= wg.delete(:name)
+            next unless wg[:content]
+
+            prefix = wg[:prefix] || wg[:identifier]&.split("/")&.first || extract_prefix(wg[:content])
+            publisher_name = PUBLISHERS[prefix]
+            name = if publisher_name
+                     [Bib::TypedLocalizedString.new(content: publisher_name)]
+                   elsif prefix
+                     [Bib::TypedLocalizedString.new(content: prefix)]
+                   else
+                     [Bib::TypedLocalizedString.new(content: wg[:content])]
+                   end
+            abbreviation = prefix ? Bib::LocalizedString.new(content: prefix) : nil
+
+            subdivision = Bib::Subdivision.new(
+              type: subdiv_type, subtype: wg[:type],
+              name: [Bib::TypedLocalizedString.new(content: wg[:content])],
+              identifier: wg[:identifier] ? [Bib::OrganizationType::Identifier.new(content: wg[:identifier])] : [],
+            )
+
+            role = Bib::Contributor::Role.new(
+              type: role_type,
+              description: [Bib::LocalizedMarkedUpString.new(content: description)],
+            )
+
+            ret[:contributor] << Bib::Contributor.new(
+              role: [role],
+              organization: Bib::Organization.new(
+                name: name, subdivision: [subdivision], abbreviation: abbreviation,
+              ),
+            )
+          end
+        end
+      end
+
+      def extract_prefix(content)
+        match = content&.match(%r{^([A-Z]+)/})
+        match[1] if match
+      end
+
+      # @param ret [Hash]
+      def structuredidentifier_hash_to_bib(ret)
+        struct_id = ret.dig(:ext, :structuredidentifier) || ret[:structuredidentifier]
+        return unless struct_id
+
+        struct_id[:project_number] = project_number_hash_to_bib(struct_id)
+        ret[:ext][:structuredidentifier] = StructuredIdentifier.new(**struct_id)
+      end
+
+      def project_number_hash_to_bib(struct_id)
+        ProjectNumber.new(
+          part: struct_id.delete(:part),
+          subpart: struct_id.delete(:subpart),
+          amendment: struct_id.delete(:amendment),
+          corrigendum: struct_id.delete(:corrigendum),
+          origyr: struct_id.delete(:origyr),
+          content: struct_id.delete(:project_number),
+        )
+      end
+
+      def stagename_hash_to_bib(ret)
+        stagename = ret.dig(:ext, :stagename) || ret[:stagename]
+        return unless stagename
+
+        ret[:ext][:stagename] = Stagename.new(**stagename_args(stagename))
+      end
+
+      def stagename_args(stagename)
+        if stagename.is_a? Hash
+          stagename
+        else
+          { content: stagename }
+        end
+      end
+
+      def create_doctype(args)
+        Doctype.new(**args)
+      end
+
+      def create_relation(rel)
+        Relation.new(**rel)
+      end
+    end
+  end
+end

data/lib/relaton/iso/hit.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Iso
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      # @return [RelatonIsoBib::IsoBibliographicItem]
+      attr_writer :item
+
+      # @return [Pubid::Iso::Identifier] pubid
+      attr_writer :pubid
+
+      # Parse page.
+      # @return [Relaton::Iso::ItemData]
+      def item
+        @item ||= begin
+          url = "#{HitCollection::ENDPOINT}#{hit[:file]}"
+          resp = Net::HTTP.get_response URI(url)
+          item = Item.from_yaml resp.body
+          item.fetched = ::Date.today.to_s
+          item
+        end
+      end
+
+      # @return [Integer]
+      def sort_weight
+        case hit[:status] # && hit["publicationStatus"]["key"]
+        when "Published" then 0
+        when "Under development" then 1
+        when "Withdrawn" then 2
+        when "Deleted" then 3
+        else 4
+        end
+      end
+
+      # @return [Pubid::Iso::Identifier]
+      def pubid
+        return @pubid if defined? @pubid
+
+        @pubid = create_pubid hit[:id]
+      rescue StandardError
+        Util.warn "Unable to create an identifier from #{hit[:id]}"
+        @pubid = nil
+      end
+
+      private
+
+      def create_pubid(id)
+        if id.is_a?(Hash)
+          ::Pubid::Iso::Identifier.from_hash(id)
+        else
+          id
+        end
+      end
+    end
+  end
+end