relaton-iec 1.0.1 → 1.5.pre

data/grammars/isostandard.rng

@@ -91,6 +91,12 @@
     </define>
     <define name="sections">
       <element name="sections">
+        <zeroOrMore>
+          <choice>
+            <ref name="note"/>
+            <ref name="admonition"/>
+          </choice>
+        </zeroOrMore>
         <ref name="clause"/>
         <optional>
           <choice>
@@ -247,7 +253,7 @@
     <define name="preface">
       <element name="preface">
         <optional>
-          <ref name="preface_abstract"/>
+          <ref name="abstract"/>
         </optional>
         <ref name="foreword"/>
         <optional>
@@ -263,6 +269,8 @@
         <value>publicly-available-specification</value>
         <value>international-workshop-agreement</value>
         <value>guide</value>
+        <value>amendment</value>
+        <value>technical-corrigendum</value>
       </choice>
     </define>
     <define name="structuredidentifier">
@@ -354,6 +362,9 @@
             <data type="boolean"/>
           </attribute>
         </optional>
+        <optional>
+          <attribute name="number"/>
+        </optional>
         <optional>
           <attribute name="subsequence"/>
         </optional>
@@ -510,7 +521,7 @@
         </attribute>
       </optional>
       <oneOrMore>
-        <ref name="paragraph-with-footnote"/>
+        <ref name="BasicBlock"/>
       </oneOrMore>
     </element>
   </define>

data/lib/relaton_iec.rb

@@ -1,3 +1,7 @@
+require "relaton_iso_bib"
+require "relaton_iec/hit"
+require "nokogiri"
+require "net/http"
 require "relaton_iec/version"
 require "relaton_iec/iec_bibliography"
 require "relaton_iec/iec_bibliographic_item"

data/lib/relaton_iec/iec_bibliography.rb

@@ -22,7 +22,7 @@ module RelatonIec
       # @param opts [Hash] options; restricted to :all_parts if all-parts
       #   reference is required
       # @return [String] Relaton XML serialisation of reference
-      def get(code, year = nil, opts = {})
+      def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         if year.nil?
           /^(?<code1>[^:]+):(?<year1>[^:]+)/ =~ code
           unless code1.nil?
@@ -45,27 +45,33 @@ module RelatonIec
 
       private
 
-      def fetch_ref_err(code, year, missed_years)
+      def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
         id = year ? "#{code}:#{year}" : code
         warn "[relaton-iec] WARNING: no match found online for #{id}. "\
           "The code must be exactly like it is on the standards website."
-        warn "[relaton-iec] (There was no match for #{year}, though there were matches "\
-          "found for #{missed_years.join(', ')}.)" unless missed_years.empty?
-        if /\d-\d/ =~ code
-          warn "[relaton-iec] The provided document part may not exist, or the document "\
-            "may no longer be published in parts."
+        unless missed_years.empty?
+          warn "[relaton-iec] (There was no match for #{year}, though there "\
+            "were matches found for #{missed_years.join(', ')}.)"
+        end
+        if /\d-\d/.match? code
+          warn "[relaton-iec] The provided document part may not exist, or "\
+            "the document may no longer be published in parts."
         else
-          warn "[relaton-iec] If you wanted to cite all document parts for the reference, "\
-            "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
-            "use its document type abbreviation (TS, TR, PAS, Guide)."
+          warn "[relaton-iec] If you wanted to cite all document parts for "\
+            "the reference, use \"#{code} (all parts)\".\nIf the document is "\
+            "not a standard, use its document type abbreviation (TS, TR, PAS, "\
+            "Guide)."
         end
         nil
       end
 
-      def fetch_pages(s, n)
-        workers = RelatonBib::WorkersPool.new n
+      # @param hits [Array<RelatonIec::Hit>]
+      # @param threads [Integer]
+      # @return [Array<RelatonIec::Hit>]
+      def fetch_pages(hits, threads)
+        workers = RelatonBib::WorkersPool.new threads
         workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
-        s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+        hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
         workers.end
         workers.result.sort_by { |a| a[:i] }.map { |x| x[:hit] }
       end
@@ -83,45 +89,47 @@ module RelatonIec
       end
 
       def iev(code = "IEC 60050")
-        RelatonIsoBib::XMLParser.from_xml(<<~"END")
-<bibitem>
-  <fetched>#{Date.today}</fetched>
-  <title format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary</title>
-  <link type="src">http://www.electropedia.org</link>
-  <docidentifier>#{code}:2011</docidentifier>
-  <date type="published"><on>2011</on></date>
-  <contributor>
-    <role type="publisher"/>
-    <organization>
-      <name>International Electrotechnical Commission</name>
-      <abbreviation>IEC</abbreviation>
-      <uri>www.iec.ch</uri>
-    </organization>
-  </contributor>
-  <language>en</language> <language>fr</language>
-  <script>Latn</script>
-  <status> <stage>60</stage> </status>
-  <copyright>
-    <from>2018</from>
-    <owner>
-      <organization>
-      <name>International Electrotechnical Commission</name>
-      <abbreviation>IEC</abbreviation>
-      <uri>www.iec.ch</uri>
-      </organization>
-    </owner>
-  </copyright>
-</bibitem>
-        END
+        RelatonIsoBib::XMLParser.from_xml(<<~"XML")
+          <bibitem>
+            <fetched>#{Date.today}</fetched>
+            <title format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary</title>
+            <link type="src">http://www.electropedia.org</link>
+            <docidentifier>#{code}:2011</docidentifier>
+            <date type="published"><on>2011</on></date>
+            <contributor>
+              <role type="publisher"/>
+              <organization>
+                <name>International Electrotechnical Commission</name>
+                <abbreviation>IEC</abbreviation>
+                <uri>www.iec.ch</uri>
+              </organization>
+            </contributor>
+            <language>en</language> <language>fr</language>
+            <script>Latn</script>
+            <status> <stage>60</stage> </status>
+            <copyright>
+              <from>2018</from>
+              <owner>
+                <organization>
+                <name>International Electrotechnical Commission</name>
+                <abbreviation>IEC</abbreviation>
+                <uri>www.iec.ch</uri>
+                </organization>
+              </owner>
+            </copyright>
+          </bibitem>
+        XML
       end
 
       # Sort through the results from Isobib, fetching them three at a time,
       # and return the first result that matches the code,
-      # matches the year (if provided), and which # has a title (amendments do not).
+      # matches the year (if provided), and which
+      # has a title (amendments do not).
       # Only expects the first page of results to be populated.
       # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
-      # If no match, returns any years which caused mismatch, for error reporting
-      def isobib_results_filter(result, year)
+      # If no match, returns any years which caused mismatch, for error
+      # reporting
+      def isobib_results_filter(result, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
         missed_years = []
         result.each_slice(3) do |s| # ISO website only allows 3 connections
           fetch_pages(s, 3).each_with_index do |r, _i|
@@ -143,7 +151,8 @@ module RelatonIec
         result = isobib_search_filter(code) || return
         ret = isobib_results_filter(result, year)
         if ret[:ret]
-          warn "[relaton-iec] (\"#{code}\") found #{ret[:ret].docidentifier.first.id}"
+          warn "[relaton-iec] (\"#{code}\") found "\
+          "#{ret[:ret].docidentifier.first.id}"
           ret[:ret]
         else
           fetch_ref_err(code, year, ret[:years])

data/lib/relaton_iec/scrapper.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "relaton_iso_bib"
-require "relaton_iec/hit"
-require "nokogiri"
-require "net/http"
-
 # Capybara.request_driver :poltergeist do |app|
 #   Capybara::Poltergeist::Driver.new app, js_errors: false
 # end
@@ -12,7 +7,6 @@ require "net/http"
 
 module RelatonIec
   # Scrapper.
-  # rubocop:disable Metrics/ModuleLength
   module Scrapper
     DOMAIN = "https://webstore.iec.ch"
 
@@ -32,33 +26,24 @@ module RelatonIec
     }.freeze
 
     class << self
-      # @param text [String]
-      # @return [Array<Hash>]
-      # def get(text)
-      #   iso_workers = WorkersPool.new 4
-      #   iso_workers.worker { |hit| iso_worker(hit, iso_workers) }
-      #   algolia_workers = start_algolia_search(text, iso_workers)
-      #   iso_docs = iso_workers.result
-      #   algolia_workers.end
-      #   algolia_workers.result
-      #   iso_docs
-      # end
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
       # Parse page.
-      # @param hit [Hash]
+      # @param hit_data [Hash]
       # @return [Hash]
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def parse_page(hit_data)
         doc = get_page hit_data[:url]
 
         # Fetch edition.
-        edition = doc.at("//th[contains(., 'Edition')]/following-sibling::td/span").text
+        edition = doc.at(
+          "//th[contains(., 'Edition')]/following-sibling::td/span"
+        ).text
 
         status, relations = fetch_status_relations hit_data[:url]
 
         IecBibliographicItem.new(
           fetched: Date.today.to_s,
-          docid: [RelatonBib::DocumentIdentifier.new(id: hit_data[:code], type: "IEC")],
+          docid: fetch_docid(hit_data),
           structuredidentifier: fetch_structuredidentifier(doc),
           edition: edition,
           language: ["en"],
@@ -74,52 +59,47 @@ module RelatonIec
           copyright: fetch_copyright(hit_data[:code], doc),
           link: fetch_link(doc, hit_data[:url]),
           relation: relations,
-          place: ["Geneva"],
+          place: ["Geneva"]
         )
       end
       # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       private
 
-      # Start search workers.
-      # @param text[String]
-      # @param iec_workers [Isobib::WorkersPool]
-      # @reaturn [Isobib::WorkersPool]
-      # def start_algolia_search(text, iec_workers)
-      #   index = Algolia::Index.new 'all_en'
-      #   workers = WorkersPool.new
-      #   workers.worker do |page|
-      #     algolia_worker(index, text, page, workers, iec_workers)
-      #   end
-
-      #   # Add first page so search worker will start.
-      #   workers << 0
-      # end
-
-      # Fetch ISO documents.
       # @param hit [Hash]
-      # @param isiso_workers [Isobib::WorkersPool]
-      # def iso_worker(hit, iso_workers)
-      #   print "Parse #{iso_workers.size} of #{iso_workers.nb_hits}  \r"
-      #   parse_page hit
-      # end
+      # @return [Array<RelatonBib::DocumentIdentifier>]
+      def fetch_docid(hit)
+        rest = hit[:code].downcase.sub(%r{
+          (?<head>[^\s]+)\s
+          (?<type>is|ts|tr|pas|srd|guide|tec|wp)?(?(<type>)\s)
+          (?<pnum>[\d-]+)\s?
+          (?<_dd>:)?(?(<_dd>)(?<date>[\d-]+)\s?)
+        }x, "")
+        m = $~
+        deliv = /cmv|csv|exv|prv|rlv|ser/.match(hit[:code].downcase).to_s
+        urn = ["urn", "iec", "std", m[:head].split("/").join("-"), m[:pnum],
+               m[:date], m[:type], deliv, "en"]
+        urn += fetch_ajunct(rest)
+        [
+          RelatonBib::DocumentIdentifier.new(id: hit[:code], type: "IEC"),
+          RelatonBib::DocumentIdentifier.new(id: urn.join(":"), type: "URN"),
+        ]
+      end
 
-      # Fetch hits from algolia search service.
-      # @param index[Algolia::Index]
-      # @param text [String]
-      # @param page [Integer]
-      # @param algolia_workers [Isobib::WorkersPool]
-      # @param isiso_workers [Isobib::WorkersPool]
-      # def algolia_worker(index, text, page, algolia_workers, iso_workers)
-      #   res = index.search text, facetFilters: ['category:standard'], page: page
-      #   next_page = res['page'] + 1
-      #   algolia_workers << next_page if next_page < res['nbPages']
-      #   res['hits'].each do |hit|
-      #     iso_workers.nb_hits = res['nbHits']
-      #     iso_workers << hit
-      #   end
-      #   iso_workers.end unless next_page < res['nbPages']
-      # end
+      # @param rest [String]
+      # @return [Array<String, nil>]
+      def fetch_ajunct(rest)
+        r = rest.sub(%r{
+          (?<_pl>\+)(?(<_pl>)(?<adjunct>amd)(?<adjnum>\d+)\s?)
+          (?<_d2>:)?(?(<_d2>)(?<adjdt>[\d-]+)\s?)
+        }x, "")
+        m = $~ || {}
+        return [] unless m[:adjunct]
+
+        plus = m[:adjunct] && "plus"
+        urn = [plus, m[:adjunct], m[:adjnum], m[:adjdt]]
+        urn + fetch_ajunct(r)
+      end
 
       # Fetch abstracts.
       # @param doc [Nokigiri::HTML::Document]
@@ -134,19 +114,6 @@ module RelatonIec
         }]
       end
 
-      # Get langs.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
-      # def langs(doc)
-      #   lgs = [{ lang: 'en' }]
-      #   doc.css('ul#lang-switcher ul li a').each do |lang_link|
-      #     lang_path = lang_link.attr('href')
-      #     lang = lang_path.match(%r{^\/(fr)\/})
-      #     lgs << { lang: lang[1], path: lang_path } if lang
-      #   end
-      #   lgs
-      # end
-
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
       # Get page.
@@ -154,25 +121,20 @@ module RelatonIec
       # @return [Array<Nokogiri::HTML::Document, String>]
       def get_page(url)
         uri = URI url
-        resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
+        resp = Net::HTTP.get_response(uri)
         case resp.code
         when "301"
           path = resp["location"]
           url = DOMAIN + path
           uri = URI url
-          resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
+          resp = Net::HTTP.get_response(uri)
         when "404"
           raise RelatonBib::RequestError, "Page not found #{url}"
         end
-        # n = 0
-        # while resp.body !~ /<strong/ && n < 10
-        #   resp = Net::HTTP.get_response(uri)#.encode("UTF-8")
-        #   n += 1
-        # end
         Nokogiri::HTML(resp.body)
-      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             OpenSSL::SSL::SSLError
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, OpenSSL::SSL::SSLError
         raise RelatonBib::RequestError, "Could not access #{url}"
       end
       # rubocop:enable Metrics/AbcSize
@@ -184,12 +146,12 @@ module RelatonIec
         item_ref = doc.at("//span[@itemprop='productID']")
         unless item_ref
           return RelatonIsoBib::StructuredIdentifier.new(
-            project_number: "?", part_number: "", prefix: nil, id: "?",
+            project_number: "?", part_number: "", prefix: nil, id: "?"
           )
         end
 
         m = item_ref.text.match(
-          /(?<=\s)(?<project>\d+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/,
+          /(?<=\s)(?<project>\d+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/
         )
         RelatonIsoBib::StructuredIdentifier.new(
           project_number: m[:project],
@@ -197,7 +159,7 @@ module RelatonIec
           subpart_number: m[:subpart],
           prefix: nil,
           type: "IEC",
-          id: item_ref.text,
+          id: item_ref.text
         )
       end
 
@@ -211,15 +173,12 @@ module RelatonIec
           statuses = YAML.load_file "lib/relaton_iec/statuses.yml"
           s = wip.at("STAGE").text
           stage, substage = statuses[s]["stage"].split "."
-          # status = statuses[s]["status"]
         else
-          # status   = "Published"
           stage    = "60"
           substage = "60"
         end
         RelatonBib::DocumentStatus.new(stage: stage, substage: substage)
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch workgroup.
       # @param doc [Nokogiri::HTML::Document]
@@ -237,13 +196,15 @@ module RelatonIec
           }],
         }
       end
+      # rubocop:enable Metrics/MethodLength
 
       # Fetch relations.
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
       # rubocop:disable Metrics/MethodLength
       def fetch_relations(doc)
-        doc.xpath('//ROW[STATUS[.!="PREPARING"]][STATUS[.!="PUBLISHED"]]').map do |r|
+        doc.xpath('//ROW[STATUS[.!="PREPARING"]][STATUS[.!="PUBLISHED"]]')
+          .map do |r|
           r_type = r.at("STATUS").text.downcase
           type = case r_type
                  # when 'published' then 'obsoletes' # Valid
@@ -251,9 +212,8 @@ module RelatonIec
                  when "withdrawn" then "obsoletes"
                  else r_type
                  end
-          # url = DOMAIN + "/publication/" + r.at("PUB_ID").text
           fref = RelatonBib::FormattedRef.new(
-            content: r.at("FULL_NAME").text, format: "text/plain",
+            content: r.at("FULL_NAME").text, format: "text/plain"
           )
           bibitem = IecBibliographicItem.new(formattedref: fref)
           { type: type, bibitem: bibitem }
@@ -270,22 +230,6 @@ module RelatonIec
         status = fetch_status doc
         relations = fetch_relations doc
         [status, relations]
-        # doc.css('ul.steps li').inject([]) do |a, r|
-        #   r_type = r.css('strong').text
-        #   type = case r_type
-        #          when 'Previously', 'Will be replaced by' then 'obsoletes'
-        #          when 'Corrigenda/Amendments', 'Revised by', 'Now confirmed'
-        #            'updates'
-        #          else r_type
-        #          end
-        #   if ['Now', 'Now under review'].include? type
-        #     a
-        #   else
-        #     a + r.css('a').map do |id|
-        #       { type: type, identifier: id.text, url: id['href'] }
-        #     end
-        #   end
-        # end
       end
       # rubocop:enable Metrics/MethodLength
 
@@ -293,66 +237,18 @@ module RelatonIec
       # @param doc [Nokogiri::HTML::Document]
       # @return [String]
       def fetch_type(doc)
-        doc.at('//th[contains(., "Publication type")]/following-sibling::td/span').
-          text.downcase.tr " ", "-"
-        # type_match = title.match(%r{^(ISO|IWA|IEC)(?:(/IEC|/IEEE|/PRF|
-        #   /NP)*\s|/)(TS|TR|PAS|AWI|CD|FDIS|NP|DIS|WD|R|Guide|(?=\d+))}x)
-        # #return "international-standard" if type_match.nil?
-        # if TYPES[type_match[2]]
-        #   TYPES[type_match[2]]
-        # elsif type_match[1]
-        # elsif type_match[1] == 'ISO'
-        #   'international-standard'
-        # elsif type_match[1] == 'IWA'
-        #   'international-workshop-agreement'
-        # end
-        # # rescue => _e
-        # #   puts 'Unknown document type: ' + title
+        doc.at(
+          '//th[contains(., "Publication type")]/following-sibling::td/span'
+        ).text.downcase.tr " ", "-"
       end
 
       # Fetch titles.
       # @param hit_data [Hash]
       # @return [Array<Hash>]
       def fetch_titles(hit_data)
-        titles = hit_data[:title].split " - "
-        case titles.size
-        when 0
-          intro, main, part = nil, "", nil
-        when 1
-          intro, main, part = nil, titles[0], nil
-        when 2
-          if /^(Part|Partie) \d+:/ =~ titles[1]
-            intro, main, part = nil, titles[0], titles[1]
-          else
-            intro, main, part = titles[0], titles[1], nil
-          end
-        when 3
-          if /^(Part|Partie) \d+:/ =~ titles[1]
-            intro, main, part = nil, titles[0], titles[1..2].join(" - ")
-          else
-            intro, main, part = titles[0], titles[1], titles[2]
-          end
-        else
-          intro, main, part = titles[0], titles[1], titles[2..-1]&.join(" -- ")
-        end
-        [{
-          title_intro: intro,
-          title_main:  main,
-          title_part:  part,
-          language:    "en",
-          script:      "Latn"
-        }]
+        RelatonBib::TypedTitleString.from_string hit_data[:title], "en", "Latn"
       end
 
-      # Return ISO script code.
-      # @param lang [String]
-      # @return [String]
-      # def script(lang)
-      #   case lang
-      #   when 'en', 'fr' then 'Latn'
-      #   end
-      # end
-
       # Fetch dates
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
@@ -365,6 +261,8 @@ module RelatonIec
         dates
       end
 
+      # rubocop:disable Metrics/MethodLength
+
       def fetch_contributors(code)
         code.sub(/\s.*/, "").split("/").map do |abbrev|
           case abbrev
@@ -379,12 +277,15 @@ module RelatonIec
             role: [type: "publisher"] }
         end
       end
+      # rubocop:enable Metrics/MethodLength
 
       # Fetch ICS.
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
       def fetch_ics(doc)
-        doc.xpath('//th[contains(text(), "ICS")]/following-sibling::td/a').map do |i|
+        doc.xpath(
+          '//th[contains(text(), "ICS")]/following-sibling::td/a'
+        ).map do |i|
           code = i.text.match(/[\d\.]+/).to_s.split "."
           { field: code[0], group: code[1], subgroup: code[2] }
         end
@@ -401,9 +302,11 @@ module RelatonIec
         links
       end
 
+      # rubocop:disable Metrics/MethodLength
+
       # Fetch copyright.
       # @param title [String]
-      # @return [Hash]
+      # @return [Array<Hash>]
       def fetch_copyright(code, doc)
         abbreviation = code.match(/.*?(?=\s)/).to_s
         case abbreviation
@@ -416,9 +319,12 @@ module RelatonIec
           from = doc.xpath("//span[@itemprop='releaseDate']").text
             .match(/\d{4}/).to_s
         end
-        { owner: { name: name, abbreviation: abbreviation, url: url }, from: from }
+        [{
+          owner: [{ name: name, abbreviation: abbreviation, url: url }],
+          from: from,
+        }]
       end
+      # rubocop:enable Metrics/MethodLength
     end
   end
-  # rubocop:enable Metrics/ModuleLength
 end