relaton-iec 1.14.0 → 1.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 471e060c4f00c9b6b9260ad1c35d300233b6953745e441d23b3dfdda66c6f84a
-  data.tar.gz: 5c44abbeae47429b403f2b90f030a87f6a5c3e4a20d02218c6c048ce43fa9be6
+  metadata.gz: e377e8eaef84193231a109087a5d036e68819065c3256179533011f7291e205e
+  data.tar.gz: c4657c57400cc95b4707ffa0132865f793bf973fa9733683576ec88cef3fcb5d
 SHA512:
-  metadata.gz: 3e39cb0ce0a1483b6081c0718bdd400f5ef614a92db579aaa2abb7092c355e3d47e832c3ccbdae718e7b161c8c0f9f8b4a1ee0a97023f98001f5abdaad89cd1a
-  data.tar.gz: f51cd3488b180a64804fc890b29ad430f4225ab069bc82037e16c996376f389b38ea59ae820782f460b9aa8b97ec252e749ff128b7240e4a71995149f1b58d32
+  metadata.gz: 21f13b927c895e376e245d678e80d3185d6281b98f86e1cdb190905651685579286562b7d88786c52af70c4f1cc25433e3f6af41c290908ac8a46211eefb0602
+  data.tar.gz: 5f1c9e9e5a0068704a4619f0ec2fe9e958bc18d8c776822072317d3d252c2fe67774186be69547d9cc8b5ad22fb96c7c4e60777c33e0215c99f609c78b540997

data/README.adoc

@@ -47,25 +47,24 @@ hit_collection = RelatonIec::IecBibliography.search("60050")
 => <RelatonIec::HitCollection:0x007fe0d7126f28 @ref=60050 @fetched=false>
 
 hit_collection.first
-=> <RelatonIec::Hit:0x007fe1068b4d10 @text="60050" @fetched="false" @fullIdentifier="" @title="IEC 60050-102:2007">
+=> <RelatonIec::Hit:0x000000000003e8 @text="60050" @fetched="false" @fullIdentifier="" @title="IEC 60050-05:1935">
 
 item = hit_collection[2].fetch
 => #<RelatonIec::IecBibliographicItem:0x007fe1171a06f8
 ...
 
 item.docidentifier
-=> [#<RelatonBib::DocumentIdentifier:0x00007f876539d1b0 @id="IEC 60050-112:2010", @language=nil, @primary=true, @scope=nil, @script=nil, @type="IEC">,
- #<RelatonBib::DocumentIdentifier:0x00007f876539d070
-  @id="urn:iec:std:iec:60050-112:2010:::en",
+=> [#<RelatonBib::DocumentIdentifier:0x00007ff1b52a2b60 @id="IEC 60050-07:1956", @language=nil, @primary=true, @scope=nil, @script=nil, @type="IEC">,
+ #<RelatonBib::DocumentIdentifier:0x00007ff1b52a2a98
+  @id="urn:iec:std:iec:60050-07:1956-01:::",
   @language=nil,
   @primary=nil,
   @scope=nil,
   @script=nil,
   @type="URN">]
-[6] pry(main)> 
 
 item.docidentifier.detect { |di| di.type == "URN" }.id
-=> "urn:iec:std:iec:60050-112:2010:::en"
+=> "urn:iec:std:iec:60050-07:1956-01:::"
 ----
 
 === Fetch document by keywords
@@ -79,8 +78,8 @@ item.docidentifier.detect { |di| di.type == "URN" }.id
 [source,ruby]
 ----
 item = RelatonIec::IecBibliography.get("IEC 60050-112:2010")
-[relaton-iec] ("IEC 60050-112") fetching...
-[relaton-iec] ("IEC 60050-112") found IEC 60050-112:2010
+[relaton-iec] ("IEC 60050-112:2010") fetching...
+[relaton-iec] ("IEC 60050-112:2010") found IEC 60050-112:2010
 => #<RelatonIec::IecBibliographicItem:0x00007f876525e8d0
 ...
 
@@ -104,15 +103,12 @@ item.docidentifier.first
 
 === Search for ISO/IEC Directives
 
-The ISO/IEC Directives are stored in a static cache in a relaton gem. It needs to use the relaton gem to fetch the ISO/IEC Directives. The following references are allowed to fetch:
+The following references are allowed to fetch:
 
-- ISO/IEC DIR 1 - Procedures for the technical work
-- ISO/IEC DIR 1 IEC SUP - Procedures for the technical work – Procedures specific to IEC
-- ISO/IEC DIR 1 ISO SUP - Consolidated ISO Supplement -- Procedures specific to ISO
-- ISO/IEC DIR 2 IEC - Principles and rules for the structure and drafting of ISO and IEC documents
-- ISO/IEC DIR 2 ISO - Principles and rules for the structure and drafting of ISO and IEC documents
-- ISO/IEC DIR IEC SUP - Procedures specific to IEC
-- ISO/IEC DIR JTC 1 SUP - Procedures specific to JTC 1
+- `IEC 61360-4 DB` - IEC/SC 3D - Common Data Dictionary (CDD - V2.0015.0001)
+- `ISO/IEC DIR 1 IEC SUP` - Procedures for the technical work - Procedures specific to IEC
+- `ISO/IEC DIR 2 IEC` - Principles and rules for the structure and drafting of ISO and IEC documents
+- 'ISO/IEC DIR IEC SUP' - Procedures specific to IEC
 
 === XML serialization
 
@@ -125,22 +121,22 @@ Possible options:
 ----
 item.to_xml
 => "<bibitem id="IEC60050-112-2010" type="standard" schema-version="v1.2.1">
-      <fetched>2022-12-03</fetched>
-      <title type="title-main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV)</title>
-      <title type="title-part" format="text/plain" language="en" script="Latn">Part 112: Quantities and units</title>
+      <fetched>2023-03-01</fetched>
       <title type="main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV) - Part 112: Quantities and units</title>
+      <title type="main" format="text/plain" language="fr" script="Latn">Vocabulaire Electrotechnique International (IEV) - Partie 112: Grandeurs et unités</title>
+      <title type="main" format="text/plain" language="es" script="Latn">Versión Oficial En español - Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
       <uri type="src">https://webstore.iec.ch/publication/162</uri>
       ...
     </bibitem>"
 
 item.to_xml bibdata: true
 => "<bibdata type="standard" schema-version="v1.2.1">
-      <fetched>2022-12-03</fetched>
-      <title type="title-main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV)</title>
-      <title type="title-part" format="text/plain" language="en" script="Latn">Part 112: Quantities and units</title>
+      <fetched>2023-03-01</fetched>
       <title type="main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV) - Part 112: Quantities and units</title>
+      <title type="main" format="text/plain" language="fr" script="Latn">Vocabulaire Electrotechnique International (IEV) - Partie 112: Grandeurs et unités</title>
+      <title type="main" format="text/plain" language="es" script="Latn">Versión Oficial En español - Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
       <uri type="src">https://webstore.iec.ch/publication/162</uri>
-      <uri type="obp">/preview/info_iec60050-112%7Bed1.0%7Db.pdf</uri>
+      <uri type="obp">https://webstore.iec.ch/preview/info_iec60050-112{ed1.0}b.pdf</uri>
       ...
       <ext schema-version="v1.0.0">
         <doctype>international-standard</doctype>
@@ -163,8 +159,16 @@ Each IEC document has `src` type link and optional `obp` type link.
 [source,ruby]
 ----
 item.link
-=> [#<RelatonBib::TypedUri:0x00007ffe9d9b6420 @content=#<Addressable::URI:0x80c URI:https://webstore.iec.ch/publication/162>, @type="src">,
- #<RelatonBib::TypedUri:0x00007ffe9d9af080 @content=#<Addressable::URI:0x820 URI:/preview/info_iec60050-112%7Bed1.0%7Db.pdf>, @type="obp">]
+=> [#<RelatonBib::TypedUri:0x00007ff1d50e9e20
+  @content=#<Addressable::URI:0x2260 URI:https://webstore.iec.ch/publication/162>,
+  @language=nil,
+  @script=nil,
+  @type="src">,
+ #<RelatonBib::TypedUri:0x00007ff1d50e9498
+  @content=#<Addressable::URI:0x2274 URI:https://webstore.iec.ch/preview/info_iec60050-112{ed1.0}b.pdf>,
+  @language=nil,
+  @script=nil,
+  @type="obp">]
 ----
 
 === Create bibliographic item from Hash
@@ -172,7 +176,8 @@ item.link
 [source,ruby]
 ----
 hash = YAML.load_file "spec/examples/hit.yaml"
-=> {"id"=>"IEC61058-2-4-1995+AMD1-2003CSV",
+=> {"schema-version"=>"v1.2.1",
+ "id"=>"IEC61058-2-4-1995+AMD1-2003CSV",
 ...
 
 RelatonIec::IecBibliographicItem.from_hash hash

data/lib/relaton_iec/data_fetcher.rb

@@ -0,0 +1,166 @@
+module RelatonIec
+  class DataFetcher
+    ENTRYPOINT = "https://api.iec.ch/harmonized/publications?size=100&sortBy=urn&page=".freeze
+    CREDENTIAL = "https://api.iec.ch/oauth/client_credential/accesstoken?grant_type=client_credentials".freeze
+
+    #
+    # Initialize new instance.
+    #
+    # @param [String] source source name (iec-harmonized-all, iec-harmonized-latest)
+    # @param [String] output output directory
+    # @param [String] format format of output files (xml, bibxml, yaml)
+    #
+    def initialize(source = "iec-harmonised-latest", output: "data", format: "yaml")
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      @files = []
+      @index = Index.new "index.yaml"
+      @all = source == "iec-harmonised-all"
+    end
+
+    #
+    # Fetch data from IEC.
+    #
+    def fetch # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      FileUtils.mkdir_p @output
+      if @all
+        FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+        @index.clear
+      end
+      fetch_all
+      add_static_files_to_index
+      @index.save
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+      warn e.backtrace.join("\n")
+    end
+
+    #
+    # Add static files to index.
+    #
+    # @return [void]
+    #
+    def add_static_files_to_index
+      Dir["static/*.yaml"].each do |file|
+        pub = RelatonBib.parse_yaml File.read(file, encoding: "UTF-8")
+        pubid = RelatonBib.array(pub["docid"]).detect { |id| id["primary"] }["id"]
+        @index.add pubid, file
+      end
+    end
+
+    #
+    # Fetch documents from IEC API.
+    #
+    # @return [void]
+    #
+    def fetch_all # rubocop:disable Metrics/MethodLength
+      page = 0
+      next_page = true
+      while next_page
+        res = fetch_page_token page
+        unless res.code == "200"
+          warn "[relaton-iec] #{res.body}"
+          break
+        end
+        json = JSON.parse res.body
+        json["publication"].each { |pub| fetch_pub pub }
+        page += 1
+        next_page = res["link"]&.include? "rel=\"last\""
+      end
+    end
+
+    #
+    # Fetch page. If response code is 401, then get new access token and try
+    #
+    # @param [Integer] page page number
+    #
+    # @return [Net::HTTP::Response] response
+    #
+    def fetch_page_token(page)
+      res = fetch_page page
+      if res.code == "401"
+        @access_token = nil
+        res = fetch_page page
+      end
+      res
+    end
+
+    #
+    # Fetch page from IEC API.
+    #
+    # @param [Integer] page page number
+    #
+    # @return [Net::HTTP::Response] response
+    #
+    def fetch_page(page)
+      url = "#{ENTRYPOINT}#{page}"
+      if !@all && @index.last_change
+        url += "&lastChangeTimestampFrom=#{@index.last_change}"
+      end
+      uri = URI url
+      req = Net::HTTP::Get.new uri
+      req["Authorization"] = "Bearer #{access_token}"
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+        http.request req
+      end
+    end
+
+    #
+    # Get access token.
+    #
+    # @return [String] access token
+    #
+    def access_token # rubocop:disable Metrics/AbcSize
+      @access_token ||= begin
+        uri = URI CREDENTIAL
+        req = Net::HTTP::Get.new uri
+        req.basic_auth ENV.fetch("IEC_HAPI_PROJ_PUBS_KEY"), ENV.fetch("IEC_HAPI_PROJ_PUBS_SECRET")
+        res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request req
+        end
+        JSON.parse(res.body)["access_token"]
+      end
+    end
+
+    #
+    # Fetch publication and save it to file.
+    #
+    # @param [Hash] pub publication
+    #
+    def fetch_pub(pub) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      bib = DataParser.new(pub).parse
+      did = bib.docidentifier.detect &:primary
+      file = File.join(@output, "#{did.id.downcase.gsub(/[:\s\/]/, '_')}.#{@ext}")
+      if @files.include? file then warn "File #{file} exists."
+      else
+        @files << file
+        @index.add index_id(pub), file, pub["lastChangeTimestamp"]
+      end
+      content = case @format
+                when "xml" then bib.to_xml bibdata: true
+                when "yaml", "yml" then bib.to_hash.to_yaml
+                when "bibxml" then bib.to_bibxml
+                end
+      File.write file, content, encoding: "UTF-8"
+    end
+
+    def index_id(pub)
+      /-(?<part>\d+)/ =~ pub["reference"]
+      title = pub.dig("title", 0, "value")
+      return pub["reference"] unless part && title
+
+      ids = title.scan(/(?<=-\sPart\s)#{part[0]}\d+(?=:)/).map do |m|
+        pub["reference"].sub(/-#{part}/, "-#{m}")
+      end
+      ids.size > 1 ? ids : pub["reference"]
+    end
+  end
+end

data/lib/relaton_iec/data_parser.rb

@@ -0,0 +1,287 @@
+module RelatonIec
+  class DataParser
+    DOMAIN = "https://webstore.iec.ch"
+
+    ATTRS = %i[
+      docid structuredidentifier language script title doctype
+      ics date contributor editorialgroup abstract copyright link relation
+    ].freeze
+
+    ABBREVS = {
+      "ISO" => ["International Organization for Standardization", "www.iso.org"],
+      "IEC" => ["International Electrotechnical Commission", "www.iec.ch"],
+      "IEEE" => ["Institute of Electrical and Electronics Engineers", "www.ieee.org"],
+      "ASTM" => ["American Society of Testing Materials", "www.astm.org"],
+      "CISPR" => ["International special committee on radio interference", "www.iec.ch"],
+    }.freeze
+
+    #
+    # Initialize new instance.
+    #
+    # @param [Hash] pub document data
+    #
+    def initialize(pub)
+      @pub = pub
+    end
+
+    #
+    # Parse document.
+    #
+    # @return [RelatonIec::IecBibliographicItem] bib item
+    #
+    def parse # rubocop:disable Metrics/AbcSize
+      args = ATTRS.each_with_object({}) { |a, h| h[a] = send a }
+      args[:docstatus] = RelatonBib::DocumentStatus.new stage: @pub["status"]
+      args[:edition] = @pub["edition"]
+      args[:price_code] = @pub["priceInfo"]["priceCode"]
+      args[:place] = ["Geneva"]
+      IecBibliographicItem.new(**args)
+    end
+
+    #
+    # Parse document identifiers.
+    #
+    # @return [Array<RelatonBib::DocumentIdentifier>] document identifiers
+    #
+    def docid
+      ids = []
+      ids << RelatonBib::DocumentIdentifier.new(id: @pub["reference"], type: "IEC", primary: true)
+      urnid = "urn:#{@pub['urnAlt'][0]}"
+      ids << RelatonBib::DocumentIdentifier.new(id: urnid, type: "URN")
+    end
+
+    #
+    # Parse structured identifier.
+    #
+    # @return [RelatonIsoBib::StructuredIdentifier] structured identifier
+    #
+    def structuredidentifier
+      m = @pub["reference"].match(
+        /(?<=\s)(?<project>\w+)(?:-(?<part>\w*)(?:-(?<subpart>\w*))?)?/,
+      )
+      RelatonIsoBib::StructuredIdentifier.new(
+        project_number: m[:project], part: m[:part], subpart: m[:subpart],
+        type: "IEC", id: @pub["reference"]
+      )
+    end
+
+    #
+    # Parse languages.
+    #
+    # @return [Array<String>] languages
+    #
+    def language
+      @pub["title"].map { |t| t["lang"] }.uniq
+    end
+
+    #
+    # Parse scripts.
+    #
+    # @return [Array<String>] scripts
+    #
+    def script
+      language.each_with_object([]) do |l, s|
+        scr = lang_to_script l
+        s << scr if scr && !s.include?(scr)
+      end
+    end
+
+    #
+    # Detect script.
+    #
+    # @param [String] lang language
+    #
+    # @return [String] script
+    #
+    def lang_to_script(lang)
+      case lang
+      when "en", "fr", "es" then "Latn"
+      end
+    end
+
+    #
+    # Parse titles.
+    #
+    # @return [Array<RelatonBib::TypedTitleString>] titles
+    #
+    def title
+      @pub["title"].map do |t|
+        RelatonBib::TypedTitleString.new(
+          content: t["value"], language: t["lang"], script: lang_to_script(t["lang"]), type: "main",
+        )
+      end
+    end
+
+    #
+    # Parse editorial group.
+    #
+    # @return [Hash] editorial group
+    #
+    def editorialgroup
+      return unless @pub["committee"]
+
+      wg = @pub["committee"]["reference"]
+      {
+        technical_committee: [{
+          name: wg,
+          type: "technicalCommittee",
+          number: wg.match(/\d+/)&.to_s&.to_i,
+        }],
+      }
+    end
+
+    #
+    # Parse abstract.
+    #
+    # @return [Array<RelatonBib::FormattedString>] abstract
+    #
+    def abstract
+      @pub["abstract"]&.map do |a|
+        RelatonBib::FormattedString.new(
+          content: a["content"], language: a["lang"], script: lang_to_script(a["lang"]),
+          format: a["format"]
+        )
+      end
+    end
+
+    # @return [Array<Hash>]
+    def copyright # rubocop:disable Metrics/AbcSize
+      from = @pub["reference"].match(/(?<=:)\d{4}/).to_s
+      from = @pub["releaseDate"]&.match(/\d{4}/).to_s if from.empty?
+      return [] if from.nil? || from.empty?
+
+      abbreviation = @pub["reference"].match(/.*?(?=\s)/).to_s
+      owner = abbreviation.split("/").map do |abbrev|
+        name, url = ABBREVS[abbrev]
+        { name: name, abbreviation: abbrev, url: url }
+      end
+      [{ owner: owner, from: from }]
+    end
+
+    #
+    # Fetche ics.
+    #
+    # @return [Array<RelatonIsoBib::Ics>] ics
+    #
+    def ics
+      return [] unless @pub["classifications"]
+
+      @pub["classifications"].select { |c| c["type"] == "ICS" }.map do |c|
+        RelatonIsoBib::Ics.new(c["value"])
+      end
+    end
+
+    #
+    # Parse dates.
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] dates
+    #
+    def date
+      date = []
+      date << create_date("published", @pub["releaseDate"]) if @pub["releaseDate"]
+      date << create_date("confirmed", @pub["confirmationDate"]) if @pub["confirmationDate"]
+      date
+    end
+
+    def create_date(type, date)
+      RelatonBib::BibliographicDate.new(type: type, on: date)
+    end
+
+    #
+    # Parse contributors.
+    #
+    # @return [Array<Hash>] contributors
+    #
+    def contributor
+      @pub["reference"].sub(/\s.*/, "").split("/").map do |abbrev|
+        name, url = ABBREVS[abbrev]
+        { entity: { name: name, url: url, abbreviation: abbrev },
+          role: [type: "publisher"] }
+      end
+    end
+
+    #
+    # Parse links.
+    #
+    # @return [Array<RelatonBib::TypedUri>] links
+    #
+    def link
+      url = "#{DOMAIN}/publication/#{urn_id}"
+      l = [RelatonBib::TypedUri.new(content: url, type: "src")]
+      RelatonBib.array(@pub["releaseItems"]).each_with_object(l) do |r, a|
+        next unless r["type"] == "PREVIEW"
+
+        url = "#{DOMAIN}/preview/#{r['contentRef']['fileName']}"
+        a << RelatonBib::TypedUri.new(content: url, type: "obp")
+      end
+    end
+
+    #
+    # Extract URN ID from URN.
+    #
+    # @return [String] URN ID
+    #
+    def urn_id
+      @pub["urn"].split(":").last
+    end
+
+    #
+    # Parse document type.
+    #
+    # @return [String] document type
+    #
+    def doctype
+      case @pub["stdType"]
+      when "IS" then "international-standard"
+      when "TR" then "technical-report"
+      when "TS" then "technical-specification"
+      when "PAS" then "publicly-available-specification"
+      when "SRD" then "system-reference-delivrabble"
+      else @pub["stdType"].downcase
+      end
+    end
+
+    #
+    # Parse relation.
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relation
+    #
+    def relation # rubocop:disable Metrics/MethodLength
+      try = 0
+      begin
+        uri = URI "#{DOMAIN}/webstore/webstore.nsf/AjaxRequestXML?" \
+                  "Openagent&url=#{urn_id}"
+        resp = Net::HTTP.get_response uri
+        doc = Nokogiri::XML resp.body
+        create_relations doc
+      rescue StandardError => e
+        try += 1
+        try < 3 ? retry : raise(e)
+      end
+    end
+
+    #
+    # Create relations.
+    #
+    # @param [Nokogiri::XML::Document] doc XML document
+    #
+    # @return [Array<Hash>] relations
+    #
+    def create_relations(doc) # rubocop:disable Metrics/MethodLength
+      doc.xpath('//ROW[STATUS[.!="PREPARING" and .!="PUBLISHED"]]')
+        .map do |r|
+        r_type = r.at("STATUS").text.downcase
+        type = case r_type
+               when "revised", "replaced" then "updates"
+               when "withdrawn" then "obsoletes"
+               else r_type
+               end
+        ref = r.at("FULL_NAME").text
+        fref = RelatonBib::FormattedRef.new content: ref, format: "text/plain"
+        docid = RelatonBib::DocumentIdentifier.new(id: ref, type: "IEC", primary: true)
+        bibitem = IecBibliographicItem.new(formattedref: fref, docid: [docid])
+        RelatonBib::DocumentRelation.new type: type, bibitem: bibitem
+      end
+    end
+  end
+end

data/lib/relaton_iec/hit.rb

@@ -3,12 +3,20 @@
 module RelatonIec
   # Hit.
   class Hit < RelatonBib::Hit
+    GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-iec/main/"
+
     attr_writer :fetch
 
     # Parse page.
     # @return [RelatonIec::IecBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page @hit
+      @fetch ||= begin
+        url = "#{GHURL}#{hit[:file]}"
+        resp = Net::HTTP.get URI(url)
+        hash = YAML.safe_load resp
+        hash["fetched"] = Date.today.to_s
+        IecBibliographicItem.from_hash hash
+      end
     end
 
     def part