relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/iec/data_fetcher.rb

@@ -0,0 +1,216 @@
+require "relaton/core"
+require_relative "../iec"
+require_relative "data_parser"
+
+module Relaton
+  module Iec
+    class DataFetcher < Relaton::Core::DataFetcher
+      ENTRYPOINT = "https://api.iec.ch/harmonized/publications?size=100&sortBy=urn&page=".freeze
+      CREDENTIAL = "https://api.iec.ch/oauth/client_credential/accesstoken?grant_type=client_credentials".freeze
+      LAST_CHANGE_FILE = "last_change.txt".freeze
+
+      #
+      # Fetch data from IEC.
+      #
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(source = "iec-harmonised-latest") # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        @all = source == "iec-harmonised-all"
+
+        if @all
+          FileUtils.rm_rf @output
+        end
+        FileUtils.mkdir_p @output
+        rebuild_index
+        fetch_all
+        index.save
+        save_last_change
+        report_errors
+      end
+
+      private
+
+      def last_change
+        return @last_change if defined? @last_change
+
+        @last_change = File.read(LAST_CHANGE_FILE, encoding: "UTF-8") if File.exist? LAST_CHANGE_FILE
+      end
+
+      def last_change_max
+        @last_change_max ||= last_change.to_s
+      end
+
+      # def last_change_max(date)
+      #   @last_change_max = date if last_change_max < date
+      # end
+
+      def save_last_change
+        return if last_change_max.empty?
+
+        File.write LAST_CHANGE_FILE, last_change_max, encoding: "UTF-8"
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :iec, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Iec::Identifier
+        )
+      end
+
+      #
+      # Rebuild index from existing YAML files.
+      #
+      # @return [void]
+      #
+      def rebuild_index
+        Dir.glob(File.join(@output, "*.yaml")).each do |file|
+          add_file_to_index(file)
+        end
+        add_static_files_to_index if Dir.exist?("static")
+      end
+
+      #
+      # Add a YAML file to the index.
+      #
+      # @param [String] file path to the YAML file
+      #
+      # @return [void]
+      #
+      def add_file_to_index(file)
+        item = Item.from_yaml(File.read(file, encoding: "UTF-8"))
+        did = find_primary_docidentifier item
+        return unless did
+
+        pubid = parse_pubid(did.to_s)
+        index.add_or_update pubid, file if pubid
+      rescue StandardError => e
+        Util.warn "Failed to index file `#{file}`: #{e.message}"
+      end
+
+      def find_primary_docidentifier(item)
+        item.docidentifier.detect(&:primary) ||
+          item.docidentifier.detect { |id| id.type == "IEC" } ||
+          item.docidentifier.first
+      end
+
+      #
+      # Add static files to index.
+      #
+      # @return [void]
+      #
+      def add_static_files_to_index
+        Dir.glob("static/*.yaml").each do |file|
+          add_file_to_index(file)
+        end
+      end
+
+      #
+      # Fetch documents from IEC API.
+      #
+      # @return [void]
+      #
+      def fetch_all # rubocop:disable Metrics/MethodLength
+        page = 0
+        next_page = true
+        while next_page
+          res = fetch_page_token page
+          unless res.code == "200"
+            Util.warn "#{res.body}"
+            break
+          end
+          json = JSON.parse res.body
+          json["publication"].each { |pub| fetch_pub pub }
+          page += 1
+          next_page = res["link"]&.include? "rel=\"last\""
+        end
+      end
+
+      #
+      # Fetch page. If response code is 401, then get new access token and try
+      #
+      # @param [Integer] page page number
+      #
+      # @return [Net::HTTP::Response] response
+      #
+      def fetch_page_token(page)
+        res = fetch_page page
+        if res.code == "401"
+          @access_token = nil
+          res = fetch_page page
+        end
+        res
+      end
+
+      #
+      # Fetch page from IEC API.
+      #
+      # @param [Integer] page page number
+      #
+      # @return [Net::HTTP::Response] response
+      #
+      def fetch_page(page)
+        url = "#{ENTRYPOINT}#{page}"
+        if !@all && last_change
+          url += "&lastChangeTimestampFrom=#{last_change}"
+        end
+        uri = URI url
+        req = Net::HTTP::Get.new uri
+        req["Authorization"] = "Bearer #{access_token}"
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request req
+        end
+      end
+
+      #
+      # Get access token.
+      #
+      # @return [String] access token
+      #
+      def access_token # rubocop:disable Metrics/AbcSize
+        @access_token ||= begin
+          uri = URI CREDENTIAL
+          req = Net::HTTP::Get.new uri
+          req.basic_auth ENV.fetch("IEC_HAPI_PROJ_PUBS_KEY"), ENV.fetch("IEC_HAPI_PROJ_PUBS_SECRET")
+          res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+            http.request req
+          end
+          JSON.parse(res.body)["access_token"]
+        end
+      end
+
+      #
+      # Fetch publication and save it to file.
+      #
+      # @param [Hash] pub publication
+      #
+      def fetch_pub(pub) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        bib = DataParser.new(pub, @errors).parse
+        did = bib.docidentifier.detect(&:primary)
+        file = output_file(did.to_s)
+        if @files.include? file then Util.warn "File #{file} exists."
+        else
+          @files << file
+          pubid = parse_pubid(did.to_s)
+          index.add_or_update pubid, file if pubid
+        end
+        ts = pub["lastChangeTimestamp"]
+        @last_change_max = ts if ts && last_change_max < ts
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      def parse_pubid(content)
+        ::Pubid::Iec::Identifier.parse(content)
+      rescue StandardError => e
+        Util.warn "Failed to parse pubid `#{content}`: #{e.message}"
+        nil
+      end
+
+      def to_xml(bib) = bib.to_xml bibdata: true
+
+      def to_yaml(bib) = bib.to_yaml
+
+      def to_bibxml(bib) = bib.to_rfcxml
+    end
+  end
+end

data/lib/relaton/iec/data_parser.rb

@@ -0,0 +1,391 @@
+module Relaton
+  module Iec
+    class DataParser
+      include Core::ArrayWrapper
+
+      DOMAIN = "https://webstore.iec.ch"
+
+      ATTRS = %i[
+        type docidentifier language script title date contributor
+        status edition abstract copyright source relation place ext
+      ].freeze
+
+      ABBREVS = {
+        "ISO" => ["International Organization for Standardization", "www.iso.org"],
+        "IEC" => ["International Electrotechnical Commission", "www.iec.ch"],
+        "IEEE" => ["Institute of Electrical and Electronics Engineers", "www.ieee.org"],
+        "ASTM" => ["American Society of Testing Materials", "www.astm.org"],
+        "CISPR" => ["International special committee on radio interference", "www.iec.ch"],
+      }.freeze
+
+      DOCTYPES = {
+        "IS" => "international-standard",
+        "TR" => "technical-report",
+        "TS" => "technical-specification",
+        "PAS" => "publicly-available-specification",
+        "SRD" => "system-reference-deliverable",
+      }.freeze
+
+      #
+      # Initialize new instance.
+      #
+      # @param [Hash] pub document data
+      #
+      def initialize(pub, errors = {})
+        @pub = pub
+        @errors = errors
+      end
+
+      #
+      # Parse document.
+      #
+      # @return [RelatonIec::IecBibliographicItem] bib item
+      #
+      def parse
+        args = ATTRS.each_with_object({}) { |a, h| h[a] = send a }
+        ItemData.new(**args)
+      end
+
+      private
+
+      def type = "standard"
+
+      #
+      # Parse document identifiers.
+      #
+      # @return [Array<Relaton::Bib::Docidentifier>] document identifiers
+      #
+      def docidentifier
+        result = [iec_docid, urn_docid].compact
+        @errors[:docidentifier] &&= result.empty?
+        result
+      end
+
+      def iec_docid
+        pubid = ::Pubid::Iec::Identifier.parse(@pub["reference"])
+        Docidentifier.new(content: pubid, type: "IEC", primary: true)
+      rescue StandardError => e
+        Util.warn "Failed to parse IEC identifier `#{@pub['reference']}`: #{e.message}"
+        Docidentifier.new(content: @pub["reference"], type: "IEC", primary: true)
+      end
+
+      def urn_docid
+        pubid = parse_urn_pubid
+        return unless pubid
+
+        Docidentifier.new(content: pubid, type: "URN")
+      end
+
+      def parse_urn_pubid
+        if @pub["urnAlt"]&.first
+          urnid = "urn:#{@pub['urnAlt'][0]}"
+          begin
+            return ::Pubid::Iec::Identifier.parse(urnid)
+          rescue StandardError => e
+            Util.warn "Failed to parse URN `#{urnid}`: #{e.message}"
+          end
+        end
+        ::Pubid::Iec::Identifier.parse(@pub["reference"])
+      rescue StandardError
+        nil
+      end
+
+      #
+      # Parse languages.
+      #
+      # @return [Array<String>] languages
+      #
+      def language
+        result = @pub["title"].map { |t| t["lang"] }.uniq
+        @errors[:language] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse scripts.
+      #
+      # @return [Array<String>] scripts
+      #
+      def script
+        result = language.each_with_object([]) do |l, s|
+          scr = lang_to_script l
+          s << scr if scr && !s.include?(scr)
+        end
+        @errors[:script] &&= result.empty?
+        result
+      end
+
+      #
+      # Detect script.
+      #
+      # @param [String] lang language
+      #
+      # @return [String] script
+      #
+      def lang_to_script(lang)
+        case lang
+        when "en", "fr", "es" then "Latn"
+        end
+      end
+
+      #
+      # Parse titles.
+      #
+      # @return [Array<Relaton::Bib::Title>] titles
+      #
+      def title
+        result = @pub["title"].reduce([]) do |acc, t|
+          acc + Bib::Title.from_string(t["value"], t["lang"], lang_to_script(t["lang"]))
+        end
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def status
+        result = begin
+          stage = Bib::Status::Stage.new content: @pub["status"]
+          Bib::Status.new stage: stage
+        end
+        @errors[:status] &&= result.nil?
+        result
+      end
+
+      def edition
+        result = Bib::Edition.new content: @pub["edition"]
+        @errors[:edition] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse abstract.
+      #
+      # @return [Array<Relaton::Bib::LocalizedMarkedUpString>] abstract
+      #
+      def abstract
+        result = Array(@pub["abstract"]).map do |a|
+          Bib::Abstract.new(
+            content: a["content"], language: a["lang"], script: lang_to_script(a["lang"]),
+          )
+        end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Copyright>] copyright
+      def copyright # rubocop:disable Metrics/AbcSize
+        from = @pub["reference"].match(/(?<=:)\d{4}/).to_s
+        from = @pub["releaseDate"]&.match(/\d{4}/).to_s if from.empty?
+        return [] if from.nil? || from.empty?
+
+        abbreviation = @pub["reference"].match(/.*?(?=\s)/).to_s
+        owner = abbreviation.split("/").map do |abbrev|
+          name, url = ABBREVS[abbrev]
+          orgname = Bib::TypedLocalizedString.new(content: name, language: "en", script: "Latn")
+          abbrev = Bib::LocalizedString.new(content: abbrev, language: "en", script: "Latn")
+          uri = Bib::Uri.new(content: url, type: "org")
+          org = Bib::Organization.new(name: [orgname], abbreviation: abbrev, uri: [uri])
+          Bib::ContributionInfo.new(organization: org)
+        end
+        result = [Bib::Copyright.new(owner: owner, from: from)]
+        @errors[:copyright] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse dates.
+      #
+      # @return [Array<Relaton::Bib::Date>] dates
+      #
+      def date
+        result = {
+          "published" => "publicationDate",
+          "stable-until" => "stabilityDate",
+          "confirmed" => "confirmationDate",
+          "obsoleted" => "dateOfWithdrawal",
+        }.reduce([]) do |a, (k, v)|
+          next a unless @pub[v]
+
+          a << Bib::Date.new(type: k, at: @pub[v])
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse contributors.
+      #
+      # @return [Array<Bib::Contributor>] contributors
+      #
+      def contributor
+        contribs = @pub["reference"].sub(/\s.*/, "").split("/").map do |abbrev|
+          name, url = ABBREVS[abbrev]
+          orgname = Bib::TypedLocalizedString.new(content: name, language: "en", script: "Latn")
+          abbr = Bib::LocalizedString.new(content: abbrev, language: "en", script: "Latn")
+          uri = Bib::Uri.new(content: url, type: "org")
+          org = Bib::Organization.new(name: [orgname], uri: [uri], abbreviation: abbr)
+          role = Bib::Contributor::Role.new(type: "publisher")
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+        result = contribs + editorialgroup_contributors
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      #
+      # Create contributors from editorial group (committee).
+      #
+      # @return [Array<Bib::Contributor>] editorial group contributors
+      #
+      def editorialgroup_contributors
+        return [] unless @pub["committee"]
+
+        wg = @pub["committee"]["reference"]
+        number = wg.match(/\d+/)&.to_s
+        orgname = Bib::TypedLocalizedString.new(content: "International Electrotechnical Commission",
+                                                language: "en", script: "Latn")
+        abbr = Bib::LocalizedString.new(content: "IEC", language: "en", script: "Latn")
+        tcname = Bib::TypedLocalizedString.new(content: wg, language: "en", script: "Latn")
+        identifiers = array(number).map { |n| Bib::OrganizationType::Identifier.new(content: n) }
+        subdivision = Bib::Subdivision.new(
+          type: "technical-committee",
+          name: [tcname],
+          identifier: identifiers,
+        )
+        org = Bib::Organization.new(name: [orgname], abbreviation: abbr, subdivision: [subdivision])
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(type: "author", description: [desc])
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+
+      #
+      # Parse links.
+      #
+      # @return [Array<Relaton::Bib::Uri>] links
+      #
+      def source
+        url = "#{DOMAIN}/publication/#{urn_id}"
+        l = [Bib::Uri.new(content: url, type: "src")]
+        result = array(@pub["releaseItems"]).each_with_object(l) do |r, a|
+          next unless r["type"] == "PREVIEW"
+
+          url = "#{DOMAIN}/preview/#{r['contentRef']['fileName']}"
+          a << Bib::Uri.new(content: url, type: "obp")
+        end
+        @errors[:source] &&= result.empty?
+        result
+      end
+
+      #
+      # Extract URN ID from URN.
+      #
+      # @return [String] URN ID
+      #
+      def urn_id
+        @pub["urn"].split(":").last
+      end
+
+      #
+      # Parse relation.
+      #
+      # @return [Array<Relaton::Bib::Relation>] relation
+      #
+      def relation # rubocop:disable Metrics/MethodLength
+        try = 0
+        result = begin
+          uri = URI "#{DOMAIN}/webstore/webstore.nsf/AjaxRequestXML?Openagent&url=#{urn_id}"
+          resp = Net::HTTP.get_response uri
+          doc = Nokogiri::XML resp.body
+          create_relations doc
+        rescue StandardError => e
+          try += 1
+          try < 3 ? retry : raise(e)
+        end
+        @errors[:relation] &&= result.empty?
+        result
+      end
+
+      #
+      # Create relations.
+      #
+      # @param [Nokogiri::XML::Document] doc XML document
+      #
+      # @return [Array<Relaton::Bib::Relation>] relations
+      #
+      def create_relations(doc) # rubocop:disable Metrics/MethodLength
+        doc.xpath('//ROW[STATUS[.!="PREPARING" and .!="PUBLISHED"]]')
+          .map do |r|
+          r_type = r.at("STATUS").text.downcase
+          type = case r_type
+                when "revised", "replaced" then "updates"
+                when "withdrawn" then "obsoletes"
+                else r_type
+                end
+          ref = r.at("FULL_NAME").text
+          docid = Docidentifier.new(content: ref, type: "IEC", primary: true)
+          bibitem = ItemData.new(formattedref: Bib::Formattedref.new(content: ref), docidentifier: [docid])
+          Relation.new type: type, bibitem: bibitem
+        end
+      end
+
+      def place
+        [Bib::Place.new(city: "Geneva")]
+      end
+
+      def ext
+        result = Ext.new(
+          doctype: doctype,
+          structuredidentifier: structuredidentifier,
+          flavor: "iec",
+          ics: ics,
+          price_code: @pub.dig("priceInfo", "priceCode"),
+        )
+        @errors[:ext] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse structured identifier.
+      #
+      # @return [Relaton::Iso::StructuredIdentifier] structured identifier
+      #
+      def structuredidentifier
+        urn = @pub.dig("project", "urn")
+        return unless urn
+
+        result = begin
+          pnum = Iso::ProjectNumber.new(content: urn.split(":").last)
+          Iso::StructuredIdentifier.new(project_number: pnum, type: "IEC")
+        end
+        @errors[:structuredidentifier] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse document type.
+      #
+      # @return [Relaton::Iec::Doctype] document type
+      #
+      def doctype
+        type = DOCTYPES[@pub["stdType"]] || @pub["stdType"].downcase
+        result = Doctype.new content: type
+        @errors[:doctype] &&= result.nil?
+        result
+      end
+
+      #
+      # Fetche ics.
+      #
+      # @return [Array<Relaton::Bib::ICS>] ics
+      #
+      def ics
+        return [] unless @pub["classifications"]
+
+        result = @pub["classifications"].select { |c| c["type"] == "ICS" }.map do |c|
+          Bib::ICS.new(code: c["value"])
+        end
+        @errors[:ics] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/iec/hit.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Iec
+    # Hit.
+    class Hit < Core::Hit
+      GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-iec/refs/heads/v2/"
+
+      attr_writer :item
+
+      # Parse page.
+      # @return [Relaton::Iec::ItemData]
+      def item
+        @item ||= begin
+          url = "#{GHURL}#{hit[:file]}"
+          resp = Net::HTTP.get URI(url)
+          Item.from_yaml(resp).tap { |it| it.fetched = Date.today.to_s }
+        end
+      end
+
+      def part
+        @part ||= hit[:id]&.part&.to_s || hit[:code]&.match(/(?<=-)[\w-]+/)&.to_s
+      end
+    end
+  end
+end