relaton 3.0.0.pre.alpha.17 → 3.0.0.pre.alpha.18

data/lib/relaton/iec/README.adoc

@@ -0,0 +1,280 @@
+= Relaton::Iec: retrieve IEC Standards for bibliographic use using the BibliographicItem model
+
+Relaton:Iec is a Ruby gem that implements the https://github.com/metanorma/metanorma-model-iso#iso-bibliographic-item[IsoBibliographicItem model].
+
+You can use it to retrieve metadata of IEC Standards from https://www.iec.ch, and access such metadata through the `Relaton::Iec::ItemData` object.
+
+== Usage
+
+=== Search for a standard using keywords
+
+`Relaton::Iec::Bibliography.search(ref, year)` method returns hits collection. Each hit can be used to fetch a document.
+
+- `ref` - a reference to search document
+- `year` - filter by year (optional)
+
+[source,ruby]
+----
+require 'relaton/iec'
+=> true
+
+hit_collection = Relaton::Iec::Bibliography.search("IEC 60050")
+=> <Relaton::Iec::HitCollection:0x00000000005650 @ref=IEC 60050 @fetched=false>
+
+hit_collection.first
+=> <Relaton::Iec::Hit:0x00000000006260 @reference="IEC 60050" @fetched="false" @docidentifier="IEC 60050-05:1935">
+
+item = hit_collection[2].item
+=> #<Relaton::Iec::ItemData:0x000000012cbc72a8
+...
+
+item.docidentifier[0].type
+=> "IEC"
+
+item.docidentifier[0].primary
+=> true
+
+item.docidentifier[0].content
+=> "IEC 60050-07:1956"
+
+item.docidentifier[1].type
+=> "URN"
+
+item.docidentifier[1].content
+=> "urn:iec:std:iec:60050-07:1956-01:::"
+----
+
+=== Fetch document by keywords
+
+`Relaton::Iec::Bibliography.get(ref, year, opts)` method returns the document.
+
+- `ref` - a reference to search document
+- `year` - filter by year (optional)
+- `opts` - hash of options (optional). Supported options are `:all_parts` (boolean), `:keep_year` (boolean).
+
+[source,ruby]
+----
+item = Relaton::Iec::Bibliography.get("IEC 60050-112:2010")
+[relaton-iec] INFO: (IEC 60050-112:2010) Fetching from Relaton repsitory ...
+[relaton-iec] INFO: (IEC 60050-112:2010) Found: `IEC 60050-112:2010`
+=> #<Relaton::Iec::ItemData:0x000000012e5c6410
+...
+
+item = Relaton::Iec::Bibliography.get("IEC 60050-112", "2010", all_parts: true)
+[relaton-iec] INFO: (IEC 60050-112:2010) Fetching from Relaton repsitory ...
+[relaton-iec] INFO: (IEC 60050-112:2010) Found: `IEC 60050 (all parts)`
+=> #<Relaton::Iec::ItemData:0x000000012e1ebb88
+...
+
+item.docidentifier.first.content
+=> "IEC 60050 (all parts)"
+
+item = Relaton::Iec::Bibliography.get("IEC 60050-112", "2010")
+[relaton-iec] INFO: (IEC 60050-112:2010) Fetching from Relaton repsitory ...
+[relaton-iec] INFO: (IEC 60050-112:2010) Found: `IEC 60050-112:2010`
+=> #<Relaton::Iec::ItemData:0x000000012f0c6b80
+
+item.docidentifier.first.content
+=> "IEC 60050-112:2010"
+----
+
+=== Search for ISO/IEC Directives
+
+The following references are allowed to fetch:
+
+- `IEC 61360-4 DB` - IEC/SC 3D - Common Data Dictionary (CDD - V2.0015.0001)
+- `ISO/IEC DIR 1 IEC SUP` - Procedures for the technical work - Procedures specific to IEC
+- `ISO/IEC DIR 2 IEC` - Principles and rules for the structure and drafting of ISO and IEC documents
+- 'ISO/IEC DIR IEC SUP' - Procedures specific to IEC
+
+=== XML serialization
+
+Possible options:
+
+- *bibdata* - If true then wrap item with _bibdata_ element and add _ext_ element.
+- *note* - Array of hashes `{ text: "Note", type: "note" }`. These notes will be added to XML.
+
+[source,ruby]
+----
+item.to_xml
+=> "<bibitem id="IEC600501122010" type="standard" schema-version="v1.5.6">
+      <fetched>2026-02-02</fetched>
+      <title language="en" script="Latn" type="title-main">International Electrotechnical Vocabulary (IEV)</title>
+      <title language="en" script="Latn" type="title-part">Part 112: Quantities and units</title>
+      <title language="en" script="Latn" type="main">International Electrotechnical Vocabulary (IEV) - Part 112: Quantities and units</title>
+      <title language="fr" script="Latn" type="title-main">Vocabulaire Electrotechnique International (IEV)</title>
+      <title language="fr" script="Latn" type="title-part">Partie 112: Grandeurs et unités</title>
+      <title language="fr" script="Latn" type="main">Vocabulaire Electrotechnique International (IEV) - Partie 112: Grandeurs et unités</title>
+      <title language="es" script="Latn" type="title-intro">Versión Oficial En español</title>
+      <title language="es" script="Latn" type="title-main">Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
+      <title language="es" script="Latn" type="main">Versión Oficial En español - Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
+      <uri type="src">https://webstore.iec.ch/publication/162</uri>
+      <uri type="obp">https://webstore.iec.ch/preview/info_iec60050-112{ed1.0}b.pdf</uri>
+      <docidentifier type="IEC" primary="true">IEC 60050-112:2010</docidentifier>
+      <docidentifier type="URN">urn:iec:std:iec:60050-112:2010-01:::</docidentifier>
+      ...
+    </bibitem>"
+
+item.to_xml bibdata: true
+=> "<bibdata type="standard" schema-version="v1.5.6">
+      <fetched>2026-02-02</fetched>
+      <title language="en" script="Latn" type="title-main">International Electrotechnical Vocabulary (IEV)</title>
+      <title language="en" script="Latn" type="title-part">Part 112: Quantities and units</title>
+      <title language="en" script="Latn" type="main">International Electrotechnical Vocabulary (IEV) - Part 112: Quantities and units</title>
+      <title language="fr" script="Latn" type="title-main">Vocabulaire Electrotechnique International (IEV)</title>
+      <title language="fr" script="Latn" type="title-part">Partie 112: Grandeurs et unités</title>
+      <title language="fr" script="Latn" type="main">Vocabulaire Electrotechnique International (IEV) - Partie 112: Grandeurs et unités</title>
+      <title language="es" script="Latn" type="title-intro">Versión Oficial En español</title>
+      <title language="es" script="Latn" type="title-main">Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
+      <title language="es" script="Latn" type="main">Versión Oficial En español - Vocabulario Electrotécnico Internacional. Parte 112: Magnitudes y unidades.</title>
+      <uri type="src">https://webstore.iec.ch/publication/162</uri>
+      <uri type="obp">https://webstore.iec.ch/preview/info_iec60050-112{ed1.0}b.pdf</uri>
+      <docidentifier type="IEC" primary="true">IEC 60050-112:2010</docidentifier>
+      <docidentifier type="URN">urn:iec:std:iec:60050-112:2010-01:::</docidentifier>
+      ...
+      <ext schema-version="v1.1.2">
+        <doctype>international-standard</doctype>
+        ...
+      </ext>
+    </bibdata>"
+
+item.to_xml note: [{ content: "Note", type: "note" }]
+=> "<bibitem id="IEC600501122010" type="standard" schema-version="v1.5.6">
+      ...
+      <note type="note">Note</note>
+      ...
+    </bibitem>"
+----
+
+=== Typed source links
+
+Each IEC document has `src` type link source and optional `obp` type link source.
+
+[source,ruby]
+----
+item.source[0].type
+=> "src"
+
+item.source[0].content
+=> "https://webstore.iec.ch/publication/162"
+
+item.source[1].type
+=> "obp"
+
+item.source[1].content
+=> "https://webstore.iec.ch/preview/info_iec60050-112{ed1.0}b.pdf"
+----
+
+=== Create bibliographic item from XML or YAML
+
+[source,ruby]
+----
+Relaton::Iec::Item.from_xml File.read("spec/fixtures/bibitem.xml")
+=> #<Relaton::Iec::ItemData:0x000000012e5c9d90
+...
+
+Relaton::Iec::Item.from_yaml File.read("spec/fixtures/item.yaml")
+=> #<Relaton::Iec::ItemData:0x000000012f282460
+...
+----
+
+=== Converting reference to URN
+
+URN is a document identifier format. It has fields delimited by a colon. If any field is absent then its place is empty. All values are in lower-case.
+
+URN structure: +
+`urn:sdo_namespace:content_type_namespace:header:project_number:date:type:deliverable:language:relation:adjunct_type:adjunct_number:date[#/=]component_or_related_asset`
+
+* prefix
+- `urn` - value: urn
+- `sdo_namespace` - value: iec
+- `content_type_namespace` - value: std
+* base document information
+- `header` - possible values are: iec, iso, iec-iso, iec-ieee, iec-itu, iec-astm
+- `project_number` - number and partnumber. For example 67654, 60601-1, 61076-7-101
+- `date` - document date (optional). Examples: 2010-03, 2010, 2010-10-11 etc.
+- `type` - documant type (optional). Possible values: ts, tr, pas, guide, is, ser.
+- `deliverable` - (optional) possible values: prv, csv, exv, rlv, cmv
+- `language` - (optional) examples: en, fr, ru, en-fr, en-fr-ru etc.
+* adjunct document information (optional)
+- `relation` - "plus" for consolidations of ajancts with a base document or "/" for ajancts itself
+- `adjunct_type` - possible values: amd, cor, ish
+- `adjunct_number` - adjunct number. Examples: 1, 2, 3, etc.
+- `date` - adjanct date. Example: 2009
+* component or related asset information (optional)
+- `component_or_related_asset` - # or = followed by component id or related asset. Exammples: #fig-1, #sec-1, =forum
+
+For more information see https://github.com/relaton/relaton-iec/issues/22
+
+The method `Relaton::Iec.code_to_urn(code, lang)` converts document identifier to URN.
+
+* `code` is a document identifier
+* `lang` is a laguage code (optional). Examples: en, fr, en-fr etc.
+
+[source,ruby]
+----
+Relaton::Iec.code_to_urn "IEC 60050-102:2007/AMD1:2017"
+=> "urn:iec:std:iec:60050-102:2007:::::amd:1:2017"
+
+Relaton::Iec.code_to_urn "IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV", "en-fr"
+=> "urn:iec:std:iec:60034-1:1969::csv:en-fr:plus:amd:1:1977:plus:amd:2:1979:plus:amd:3:1980"
+----
+
+The method `Relaton::Iec.urn_to_code(urn)` converts URN to document identifier.
+
+[source,ruby]
+----
+Relaton::Iec.urn_to_code "urn:iec:std:iec:60050-102:2007:::::amd:1:2017"
+=> ["IEC 60050-102:2007/AMD1:2017", ""]
+
+Relaton::Iec.urn_to_code "urn:iec:std:iec:60034-1:1969::csv:en-fr:plus:amd:1:1977:plus:amd:2:1979:plus:amd:3:1980"
+=> ["IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV", "en-fr"]
+----
+
+=== Fetch data
+
+This gem uses the https://bitbucket.org/sdo-hapi/api-specifications/wiki/Guidelines[Harmonized API] as a data source. The following options are available:
+- `iec-harmonized-all` - fetches all the documents using the API
+- `iec-harmonized-latest` - fetches the latest documents using the API
+
+The method `Relaton::Iec::DataFetcher.fetch(source, output: "data", format: "yaml") fetches IEC documents from the dataset and saves them to the `./data` folder in YAML format.
+Arguments:
+
+- `source` - data source. Possible values: `iec-harmonized-all`, `iec-harmonized-latest` (default `iec-harmonized-latest`).
+- `output` - folder to save documents (default './data').
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+Relaton::Iec::DataFetcher.fetch
+Started at: 2022-06-23 09:36:55 +0200
+Stopped at: 2022-06-23 09:36:58 +0200
+Done in: 752 sec.
+=> nil
+----
+
+=== Logging
+
+Relaton::Iec uses the relaton-logger gem for logging. By default, it logs to STDOUT. To change the log levels and add other loggers, read the https://github.com/relaton/relaton-logger#usage[relaton-logger] documentation.
+
+== Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To update the index test fixture (used by tests), run `rake spec:update_index`. This downloads the latest `index-v2.zip` from the https://github.com/relaton/relaton-data-iec[relaton-data-iec] repository.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+== Exceptional Citations
+
+This gem retrieves bibliographic descriptions of ISO documents by doing searches on the ISO website, http://www.iso.org, and screenscraping the document that matches the queried document identifier. The following documents are not returned as search results from the ISO website, and the gem returns manually generated references to them.
+
+* `IEV`: used in the metanorma-iso gem to reference Electropedia entries generically. Is resolved to an "all parts" reference to IEC 60050, which in turn is resolved into the specific documents cited by their top-level clause.
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-iec.
+
+== License
+
+The gem is available as open source under the terms of the https://opensource.org/licenses/MIT[MIT License].

data/lib/relaton/iec/data_fetcher.rb

@@ -19,9 +19,10 @@ module Relaton
       def fetch(source = "iec-harmonised-latest") # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         @all = source == "iec-harmonised-all"
 
-        if @all
-          FileUtils.rm_rf @output
-        end
+        # A prune run keeps the tree and lets prune_unwritten remove stale
+        # files behind the 90% guard; the wipe is only the no-prune legacy
+        # path (a partial prune run must never wipe the dataset).
+        FileUtils.rm_rf @output if @all && !@prune
         FileUtils.mkdir_p @output
         rebuild_index
         fetch_all
@@ -35,7 +36,22 @@ module Relaton
       def last_change
         return @last_change if defined? @last_change
 
-        @last_change = File.read(LAST_CHANGE_FILE, encoding: "UTF-8") if File.exist? LAST_CHANGE_FILE
+        @last_change = File.read(last_change_file, encoding: "UTF-8") if File.exist? last_change_file
+      end
+
+      # Overridable source knobs: the projects API shares the OAuth app,
+      # pagination and change cursor with publications but has its own
+      # endpoint, list key and cursor file.
+      def entrypoint
+        ENTRYPOINT
+      end
+
+      def last_change_file
+        LAST_CHANGE_FILE
+      end
+
+      def list_key
+        "publication"
       end
 
       def last_change_max
@@ -49,7 +65,7 @@ module Relaton
       def save_last_change
         return if last_change_max.empty?
 
-        File.write LAST_CHANGE_FILE, last_change_max, encoding: "UTF-8"
+        File.write last_change_file, last_change_max, encoding: "UTF-8"
       end
 
       def index
@@ -120,7 +136,7 @@ module Relaton
             break
           end
           json = JSON.parse res.body
-          json["publication"].each { |pub| fetch_pub pub }
+          json[list_key].each { |pub| fetch_pub pub }
           page += 1
           next_page = res["link"]&.include? "rel=\"last\""
         end
@@ -150,7 +166,7 @@ module Relaton
       # @return [Net::HTTP::Response] response
       #
       def fetch_page(page)
-        url = "#{ENTRYPOINT}#{page}"
+        url = "#{entrypoint}#{page}"
         if !@all && last_change
           url += "&lastChangeTimestampFrom=#{last_change}"
         end
@@ -202,7 +218,7 @@ module Relaton
         end
         ts = pub["lastChangeTimestamp"]
         @last_change_max = ts if ts && last_change_max < ts
-        File.write file, serialize(bib), encoding: "UTF-8"
+        force_write file, serialize(bib)
       end
 
       def parse_pubid(content)

data/lib/relaton/iec/project_fetcher.rb

@@ -0,0 +1,48 @@
+module Relaton
+  module Iec
+    # Fetches in-development projects from the IEC Harmonized API
+    # (/harmonized/projects). The projects endpoint shares the OAuth app,
+    # pagination and lastChangeTimestampFrom cursor with publications —
+    # the credential's PROJ_PUBS name covers both — but carries the stage
+    # lifecycle and project relations inline, so there is no webstore
+    # round-trip.
+    class ProjectFetcher < DataFetcher
+      PROJECT_ENTRYPOINT = "https://api.iec.ch/harmonized/projects?size=100&sortBy=urn&page=".freeze
+      PROJECT_LAST_CHANGE_FILE = "last_change_projects.txt".freeze
+
+      def fetch(source = "iec-projects-latest")
+        @all = source.end_with?("-all")
+        # The wipe is the no-prune legacy path (see DataFetcher#fetch).
+        FileUtils.rm_rf @output if @all && !@prune
+        FileUtils.mkdir_p @output
+        fetch_all
+        save_last_change
+        report_errors
+      end
+
+      private
+
+      def entrypoint
+        PROJECT_ENTRYPOINT
+      end
+
+      def last_change_file
+        PROJECT_LAST_CHANGE_FILE
+      end
+
+      def list_key
+        "project"
+      end
+
+      def fetch_pub(project)
+        bib = ProjectParser.new(project, @errors).parse
+        did = bib.docidentifier.detect(&:primary)
+        file = unique_output_file(did.to_s)
+        @files << file
+        ts = project["lastChangeTimestamp"]
+        @last_change_max = ts if ts && last_change_max < ts
+        File.write file, to_yaml(bib), encoding: "UTF-8"
+      end
+    end
+  end
+end

data/lib/relaton/iec/project_parser.rb

@@ -0,0 +1,111 @@
+module Relaton
+  module Iec
+    # Parses Harmonized API project records (in-development standards) into
+    # Relaton items. Field reference: the IEC/ISO Harmonized APIs technical
+    # documentation, section "HAPI - Projects".
+    class ProjectParser < DataParser
+      # projectRelation types have no IEC-side reference, only a URN; map
+      # them onto the bibdata relation vocabulary. Agreements carrying an
+      # externalProject (Frankfurt agreement) have no URN and are skipped.
+      RELATION_TYPES = {
+        "AMENDS" => "updates",
+        "CORRECTS" => "updates",
+        "CORRECTED_BY" => "updatedBy",
+        "REVISED_BY" => "updatedBy",
+        "REPLACED_BY" => "obsoletedBy",
+        "MERGED_INTO" => "mergedInto",
+        "SPLIT_FROM" => "derivedFrom",
+        "INTERPRETS" => "commentaryOf",
+      }.freeze
+
+      private
+
+      # The project URN namespace differs from publications (iec:proj: vs
+      # iec:std:), so the parent's URN-from-reference derivation does not
+      # apply; the record's own urn is taken verbatim.
+      def docidentifier
+        ids = [iec_docid]
+        ids << Docidentifier.new(content: @pub["urn"], type: "URN") if @pub["urn"]
+        @errors[:docidentifier] &&= ids.compact.empty?
+        ids.compact
+      end
+
+      def date
+        result = {
+          "published" => @pub["publicationDate"] || @pub["editionDate"],
+          "created" => @pub["initialProjectDate"],
+          "forecast" => @pub["forecastPublicationDate"],
+        }.filter_map do |k, v|
+          next unless v
+
+          Bib::Date.new(type: k, at: v)
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      # The stage lifecycle is the project's status: the CURRENT stage when
+      # present, else the latest by effective date. stageCode is the IEC
+      # abbreviation (CFDIS, WPUB, DELPUB...), harmonizedStageCode the
+      # ISO-style numeric stage.
+      def status
+        stage = current_stage || Array(@pub["stages"]).last
+        return unless stage
+
+        result = Bib::Status.new(
+          stage: Bib::Status::Stage.new(
+            content: stage["harmonizedStageCode"], abbreviation: stage["stageCode"],
+          ),
+          iteration: stage["iteration"]&.to_s,
+        )
+        @errors[:status] &&= result.nil?
+        result
+      end
+
+      def current_stage
+        Array(@pub["stages"]).find { |s| s["status"] == "CURRENT" }
+      end
+
+      def contributor
+        super + [owner_contributor].compact
+      end
+
+      def owner_contributor
+        return unless @pub.dig("owner", "displayName")
+
+        orgname = Bib::TypedLocalizedString.new(content: "International Electrotechnical Commission",
+                                                language: "en", script: "Latn")
+        abbr = Bib::LocalizedString.new(content: "IEC", language: "en", script: "Latn")
+        tcname = Bib::TypedLocalizedString.new(content: @pub["owner"]["displayName"],
+                                               language: "en", script: "Latn")
+        subdivision = Bib::Subdivision.new(
+          type: "technical-committee",
+          name: [tcname],
+        )
+        org = Bib::Organization.new(name: [orgname], abbreviation: abbr, subdivision: [subdivision])
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(type: "author", description: [desc])
+        Bib::Contributor.new(organization: org, role: [role])
+      end
+
+      def relation
+        rels = Array(@pub["projectRelations"]).filter_map { |r| project_relation r } +
+               Array(@pub["publications"]).filter_map { |p| urn_relation "hasSuccessor", p["urn"] }
+        @errors[:relation] &&= rels.empty?
+        rels
+      end
+
+      def project_relation(rel)
+        urn_relation RELATION_TYPES.fetch(rel["type"], "related"), rel["urn"]
+      end
+
+      def urn_relation(type, urn)
+        return unless urn
+
+        docid = Docidentifier.new(content: urn, type: "URN")
+        bibitem = ItemData.new(formattedref: Bib::Formattedref.new(content: urn), docidentifier: [docid])
+        Relation.new(type: type, bibitem: bibitem)
+      end
+    end
+  end
+end

data/lib/relaton/iec.rb

@@ -20,6 +20,11 @@ require_relative "iec/model/bibdata"
 
 module Relaton
   module Iec
+    autoload :DataFetcher, "relaton/iec/data_fetcher"
+    autoload :DataParser, "relaton/iec/data_parser"
+    autoload :ProjectFetcher, "relaton/iec/project_fetcher"
+    autoload :ProjectParser, "relaton/iec/project_parser"
+
     INDEXFILE = "index-v2".freeze
     # The machine index on the data repo's Pages site (relaton#206) —
     # manifest + per-shard rows read by HitCollection's index; the zip stays