relaton-iso 1.20.0 → 2.0.0.pre.alpha.1

data/grammars/relaton-iso-compile.rng

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="basicdoc.rng"/>
+  <include href="relaton-iso.rng"/>
+  <start>
+    <choice>
+      <ref name="bibitem"/>
+      <ref name="bibdata"/>
+    </choice>
+  </start>
+</grammar>

data/grammars/relaton-iso.rng

@@ -0,0 +1,214 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="biblio-standoc.rng">
+    <define name="BibDataExtensionType">
+      <optional>
+        <attribute name="schema-version"/>
+      </optional>
+      <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
+      <ref name="flavor"/>
+      <optional>
+        <ref name="horizontal"/>
+      </optional>
+      <ref name="editorialgroup"/>
+      <optional>
+        <ref name="approvalgroup"/>
+      </optional>
+      <zeroOrMore>
+        <ref name="ics"/>
+      </zeroOrMore>
+      <ref name="structuredidentifier"/>
+      <optional>
+        <ref name="stagename"/>
+      </optional>
+      <optional>
+        <ref name="updates_document_type"/>
+      </optional>
+      <optional>
+        <ref name="fast_track"/>
+      </optional>
+      <optional>
+        <ref name="price-code"/>
+      </optional>
+    </define>
+    <define name="bdate">
+      <element name="date">
+        <attribute name="type">
+          <choice>
+            <ref name="BibliographicDateType"/>
+            <text/>
+          </choice>
+        </attribute>
+        <choice>
+          <group>
+            <element name="from">
+              <ref name="ISO8601Date"/>
+            </element>
+            <optional>
+              <element name="to">
+                <ref name="ISO8601Date"/>
+              </element>
+            </optional>
+          </group>
+          <element name="on">
+            <choice>
+              <ref name="ISO8601Date"/>
+              <value>--</value>
+              <value>–</value>
+            </choice>
+          </element>
+        </choice>
+      </element>
+    </define>
+    <define name="DocumentType">
+      <choice>
+        <value>international-standard</value>
+        <value>technical-specification</value>
+        <value>technical-report</value>
+        <value>publicly-available-specification</value>
+        <value>international-workshop-agreement</value>
+        <value>guide</value>
+        <value>recommendation</value>
+        <value>amendment</value>
+        <value>technical-corrigendum</value>
+        <value>directive</value>
+        <value>committee-document</value>
+        <value>addendum</value>
+      </choice>
+    </define>
+    <define name="DocumentSubtype">
+      <choice>
+        <value>specification</value>
+        <value>method-of-test</value>
+        <value>vocabulary</value>
+        <value>code-of-practice</value>
+      </choice>
+    </define>
+    <define name="structuredidentifier">
+      <element name="structuredidentifier">
+        <optional>
+          <attribute name="type"/>
+        </optional>
+        <group>
+          <ref name="documentnumber"/>
+          <optional>
+            <ref name="tc-documentnumber"/>
+          </optional>
+        </group>
+      </element>
+    </define>
+    <define name="editorialgroup">
+      <element name="editorialgroup">
+        <ref name="ISOProjectGroup"/>
+      </element>
+    </define>
+  </include>
+  <define name="updates_document_type">
+    <element name="updates-document-type">
+      <ref name="DocumentType"/>
+    </element>
+  </define>
+  <define name="ISOProjectGroup">
+    <zeroOrMore>
+      <ref name="agency"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="technical-committee"/>
+    </oneOrMore>
+    <zeroOrMore>
+      <ref name="subcommittee"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="workgroup"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="secretariat"/>
+    </optional>
+  </define>
+  <define name="approvalgroup">
+    <element name="approvalgroup">
+      <ref name="ISOProjectGroup"/>
+    </element>
+  </define>
+  <define name="agency">
+    <element name="agency">
+      <text/>
+    </element>
+  </define>
+  <define name="horizontal">
+    <element name="horizontal">
+      <data type="boolean"/>
+    </element>
+  </define>
+  <define name="documentnumber">
+    <element name="project-number">
+      <optional>
+        <attribute name="part">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="subpart">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="amendment">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="corrigendum">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="origyr">
+          <ref name="ISO8601Date"/>
+        </attribute>
+      </optional>
+      <text/>
+    </element>
+  </define>
+  <define name="tc-documentnumber">
+    <element name="tc-document-number">
+      <data type="int"/>
+    </element>
+  </define>
+  <define name="subcommittee">
+    <element name="subcommittee">
+      <ref name="IsoWorkgroup"/>
+    </element>
+  </define>
+  <define name="workgroup">
+    <element name="workgroup">
+      <ref name="IsoWorkgroup"/>
+    </element>
+  </define>
+  <define name="secretariat">
+    <element name="secretariat">
+      <text/>
+    </element>
+  </define>
+  <define name="stagename">
+    <element name="stagename">
+      <optional>
+        <attribute name="abbreviation"/>
+      </optional>
+      <text/>
+    </element>
+  </define>
+  <define name="fast_track">
+    <element name="fast-track">
+      <data type="boolean"/>
+    </element>
+  </define>
+  <define name="price-code">
+    <element name="price-code">
+      <text/>
+    </element>
+  </define>
+</grammar>

data/lib/relaton/iso/bibliography.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+# require 'relaton_iso/iso_bibliographic_item'
+# require "relaton_iso/scrapper"
+# require "relaton_iso/hit_collection"
+# require "relaton_iec"
+
+module Relaton
+  module Iso
+    # Methods for search ISO standards.
+    module Bibliography
+      extend self
+
+      # @param text [Pubid::Iso::Identifier, String]
+      # @return [RelatonIso::HitCollection]
+      def search(pubid, opts = {})
+        pubid = ::Pubid::Iso::Identifier.parse(pubid) if pubid.is_a? String
+        HitCollection.new(pubid, opts).find
+      rescue  SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+              Net::ProtocolError, OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # @param ref [String] the ISO standard Code to look up (e..g "ISO 9000")
+      # @param year [String, NilClass] the year the standard was published
+      # @param opts [Hash] options; restricted to :all_parts if all-parts
+      # @option opts [Boolean] :all_parts if all-parts reference is required
+      # @option opts [Boolean] :keep_year if undated reference should return
+      #   actual reference with year
+      #
+      # @return [RelatonIsoBib::IsoBibliographicItem] Bibliographic item
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/AbcSize
+        code = ref.gsub("\u2013", "-")
+
+        # parse "all parts" request
+        # code.sub! " (all parts)", ""
+        # opts[:all_parts] ||= $~ && opts[:all_parts].nil?
+
+        query_pubid = ::Pubid::Iso::Identifier.parse(code)
+        query_pubid.root.year = year.to_i if year&.respond_to?(:to_i)
+        query_pubid.root.all_parts ||= opts[:all_parts]
+        Util.info "Fetching from Relaton repository ...", key: query_pubid.to_s
+
+        hits, missed_year_ids = isobib_search_filter(query_pubid, opts)
+        tip_ids = look_up_with_any_types_stages(hits, ref, opts)
+        ret = hits.fetch_doc
+        return fetch_ref_err(query_pubid, missed_year_ids, tip_ids) unless ret
+
+        response_pubid = ret.docidentifier.find(&:primary) # .sub(" (all parts)", "")
+        Util.info "Found: `#{response_pubid}`", key: query_pubid.to_s
+        get_all = (query_pubid.root.year && opts[:keep_year].nil?) || opts[:keep_year] || opts[:all_parts]
+        return ret if get_all
+
+        ret.to_most_recent_reference
+      rescue ::Pubid::Core::Errors::ParseError
+        Util.warn "Is not recognized as a standards identifier.", key: code
+        nil
+      end
+
+      # @param query_pubid [Pubid::Iso::Identifier]
+      # @param pubid [Pubid::Iso::Identifier]
+      # @param all_parts [Boolean] match with any parts when true
+      # @return [Boolean]
+      def matches_parts?(query_pubid, pubid, all_parts: false)
+        # match only with documents with part number
+        return !pubid.part.nil? if all_parts
+
+        query_pubid.part == pubid.part
+      end
+
+      #
+      # Matches base of query_pubid and pubid.
+      #
+      # @param [Pubid::Iso::Identifier] query_pubid pubid to match
+      # @param [Pubid::Iso::Identifier] pubid pubid to match
+      # @param [Boolean] any_types_stages match with any types and stages
+      #
+      # @return [<Type>] <description>
+      #
+      def matches_base?(query_pubid, pubid, any_types_stages: false) # rubocop:disable Metrics?PerceivedComplexity
+        return false unless pubid.respond_to?(:publisher)
+
+        query_pubid.publisher == pubid.publisher &&
+          query_pubid.number == pubid.number &&
+          query_pubid.copublisher == pubid.copublisher &&
+          (any_types_stages || query_pubid.stage == pubid.stage) &&
+          (any_types_stages || query_pubid.is_a?(pubid.class))
+      end
+
+      # @param hit_collection [RelatonIso::HitCollection]
+      # @param year [String]
+      # @return [Array<RelatonIso::HitCollection, Array<String>>] hits and missed year IDs
+      def filter_hits_by_year(hit_collection, year)
+        missed_year_ids = Set.new
+        return [hit_collection, missed_year_ids] if year.nil?
+
+        # filter by year
+        hit_collection.select! do |hit|
+          hit.pubid.year ||= hit.hit[:year]
+          next true if check_year(year, hit)
+
+          missed_year_ids << hit.pubid.to_s if hit.pubid.year
+          false
+        end
+
+        [hit_collection, missed_year_ids]
+      end
+
+      private
+
+      def check_year(year, hit) # rubocop:disable Metrics/AbcSize
+        (hit.pubid.base.nil? && hit.pubid.year.to_s == year.to_s) ||
+          (!hit.pubid.base.nil? && hit.pubid.base.year.to_s == year.to_s) ||
+          (!hit.pubid.base.nil? && hit.pubid.year.to_s == year.to_s)
+      end
+
+      # @param pubid [Pubid::Iso::Identifier] PubID with no results
+      def fetch_ref_err(pubid, missed_year_ids, tip_ids) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        Util.info "Not found.", key: pubid.to_s
+
+        if missed_year_ids.any?
+          ids = missed_year_ids.map { |i| "`#{i}`" }.join(", ")
+          Util.info "TIP: No match for edition year #{pubid.year}, but matches exist for #{ids}.", key: pubid.to_s
+        end
+
+        if tip_ids.any?
+          ids = tip_ids.map { |i| "`#{i}`" }.join(", ")
+          Util.info "TIP: Matches exist for #{ids}.", key: pubid.to_s
+        end
+
+        if pubid.part
+          Util.info "TIP: If it cannot be found, the document may no longer be published in parts.", key: pubid.to_s
+        else
+          Util.info "TIP: If you wish to cite all document parts for the reference, " \
+                    "use `#{pubid.to_s(format: :ref_undated)} (all parts)`.", key: pubid.to_s
+        end
+
+        nil
+      end
+
+      def look_up_with_any_types_stages(hits, ref, opts)
+        return [] if hits.any? || !ref.match?(/^ISO[\/\s][A-Z]/)
+
+        ref_no_type_stage = ref.sub(/^ISO[\/\s][A-Z]+/, "ISO")
+        pubid = ::Pubid::Iso::Identifier.parse(ref_no_type_stage)
+        resp, = isobib_search_filter(pubid, opts, any_types_stages: true)
+        resp.map &:pubid
+      end
+
+      #
+      # Search for hits. If no found then trying missed stages.
+      #
+      # @param query_pubid [Pubid::Iso::Identifier] reference without correction
+      # @param opts [Hash]
+      # @param any_types_stages [Boolean] match with any stages
+      #
+      # @return [Array<RelatonIso::HitCollection, Array<String>>] hits and missed years
+      #
+      def isobib_search_filter(query_pubid, opts, any_types_stages: false)
+        hit_collection = search(query_pubid, opts)
+
+        # filter only matching hits
+        filter_hits hit_collection, query_pubid, any_types_stages
+      end
+
+      #
+      # Filter hits by query_pubid.
+      #
+      # @param hit_collection [RelatonIso::HitCollection]
+      # @param query_pubid [Pubid::Iso::Identifier]
+      # @param all_parts [Boolean]
+      # @param any_types_stages [Boolean]
+      #
+      # @return [Array<RelatonIso::HitCollection, Array<String>>] hits and missed year IDs
+      #
+      def filter_hits(hit_collection, query_pubid, any_types_stages) # rubocop:disable Metrics/AbcSize
+        # filter out
+        excludings = build_excludings(query_pubid.root.all_parts, any_types_stages)
+        no_year_ref = hit_collection.ref_pubid_no_year.exclude(*excludings)
+        hit_collection.select! do |i|
+          pubid_match?(i.pubid, query_pubid, excludings, no_year_ref) &&
+            !(query_pubid.root.all_parts && i.pubid.part.nil?)
+        end
+
+        filter_hits_by_year(hit_collection, query_pubid.root.year)
+      end
+
+      def build_excludings(all_parts, any_types_stages)
+        excludings = %i[year edition all_parts]
+        excludings += %i[type stage iteration] if any_types_stages
+        excludings << :part if all_parts
+        excludings
+      end
+
+      def pubid_match?(pubid, query_pubid, excludings, no_year_ref)
+        if pubid.is_a? String then pubid == query_pubid.to_s
+        else
+          pubid = pubid.dup
+          pubid.base = pubid.base.exclude(:year, :edition) if pubid.base
+          pubid.exclude(*excludings) == no_year_ref
+        end
+      end
+    end
+  end
+end

data/lib/relaton/iso/data_fetcher.rb

@@ -0,0 +1,227 @@
+require_relative "../iso"
+require_relative "queue"
+require_relative "scraper"
+
+module Relaton
+  module Iso
+    # Fetch all the documents from ISO website.
+    class DataFetcher < Core::DataFetcher
+      def gh_issue_channel
+        ["relaton/relaton-iso", "Error fetching ISO documents"]
+      end
+
+      #
+      # The queue is used to store the ICS page paths beeing fetching in the current run.
+      #
+      # @return [Queue] queue
+      #
+      def queue
+        @queue ||= ::Queue.new
+      end
+
+      def mutex
+        @mutex ||= Mutex.new
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create :iso, file: "#{HitCollection::INDEXFILE}.yaml"
+      end
+
+      #
+      # ISO has too many docs. GHA can't get them all in one run.
+      # So, we need to split the process into several runs.
+      # The iso_queue is used to store the doc paths that have not been fetched.
+      #
+      # @return [Relaton::Iso::Queue] queue
+      #
+      def iso_queue
+        @iso_queue ||= Relaton::Iso::Queue.new
+      end
+
+      #
+      # Go through all ICS and fetch all documents.
+      #
+      # @return [void]
+      #
+      def fetch # rubocop:disable Metrics/AbcSize
+        Util.info "Scrapping ICS pages..."
+        fetch_ics
+        Util.info "(#{Time.now}) Scrapping documents..."
+        fetch_docs
+        iso_queue.save
+        # index.sort! { |a, b| compare_docids a, b }
+        index.save
+        repot_errors
+      end
+
+      private
+
+      #
+      # Fetch ICS page recursively and store all the links to documents in the iso_queue.
+      #
+      # @param [String] path path to ICS page
+      #
+      def fetch_ics
+        threads = Array.new(3) { thread { |path| fetch_ics_page(path) } }
+        fetch_ics_page "/standards-catalogue/browse-by-ics.html"
+        sleep(1) until queue.empty?
+        threads.size.times { queue << :END }
+        threads.each(&:join)
+      end
+
+      def fetch_ics_page(path)
+        resp = get_redirection path
+        unless resp
+          Util.error "Failed fetching ICS page #{url(path)}"
+          return
+        end
+
+        page = Nokogiri::HTML(resp.body)
+        parse_doc_links page
+        parse_ics_links page
+      end
+
+      def parse_doc_links(page)
+        doc_links = page.xpath "//td[@data-title='Standard and/or project']/div/div/a"
+        @errors[:doc_links] &&= doc_links.empty?
+        doc_links.each { |item| iso_queue.add_first item[:href].split("?").first }
+      end
+
+      def parse_ics_links(page)
+        ics_links = page.xpath("//td[@data-title='ICS']/a")
+        @errors[:ics_links] &&= ics_links.empty?
+        ics_links.each { |item| queue << item[:href] }
+      end
+
+      def url(path)
+        Scraper::DOMAIN + path
+      end
+
+      #
+      # Get the page from the given path. If the page is redirected, get the
+      # page from the new path.
+      #
+      # @param [String] path path to the page
+      #
+      # @return [Net::HTTPOK, nil] HTTP response
+      #
+      def get_redirection(path) # rubocop:disable Metrics/MethodLength
+        try = 0
+        uri = URI url(path)
+        begin
+          get_response uri
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED => e
+          try += 1
+          retry if check_try try, uri
+
+          Util.warn "Failed fetching #{uri}, #{e.message}"
+        end
+      end
+
+      def get_response(uri)
+        resp = Net::HTTP.get_response(uri)
+        resp.code == "302" ? get_redirection(resp["location"]) : resp
+      end
+
+      def check_try(try, uri)
+        if try < 3
+          Util.warn "Timeout fetching #{uri}, retrying..."
+          sleep 1
+          true
+        end
+      end
+
+      def fetch_docs
+        threads = Array.new(3) { thread { |path| fetch_doc(path) } }
+        iso_queue[0..10_000].each { |docpath| queue << docpath }
+        threads.size.times { queue << :END }
+        threads.each(&:join)
+      end
+
+      #
+      # Fetch document from ISO website.
+      #
+      # @param [String] docpath document page path
+      #
+      # @return [void]
+      #
+      def fetch_doc(docpath)
+        doc = Scraper.parse_page docpath, errors: @errors
+        mutex.synchronize { save_doc doc, docpath }
+      rescue StandardError => e
+        Util.warn "Fail fetching document: #{url(docpath)}\n#{e.message}\n#{e.backtrace}"
+      end
+
+      # def compare_docids(id1, id2)
+      #   Pubid::Iso::Identifier.create(**id1).to_s <=> Pubid::Iso::Identifier.create(**id2).to_s
+      # end
+
+      #
+      # save document to file.
+      #
+      # @param [RelatonIsoBib::IsoBibliographicItem] doc document
+      #
+      # @return [void]
+      #
+      def save_doc(doc, docpath) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        docid = doc.docidentifier.detect(&:primary)
+        file = output_file docid.content.to_s
+        if File.exist?(file)
+          rewrite_with_same_or_newer doc, docid, file, docpath
+        else
+          write_file file, doc, docid
+        end
+        iso_queue.move_last docpath
+      end
+
+      def rewrite_with_same_or_newer(doc, docid, file, docpath)
+        bib = Item.from_yaml File.read(file, encoding: "UTF-8")
+        if edition_greater?(doc, bib) || replace_substage98?(doc, bib)
+          write_file file, doc, docid
+        elsif @files.include?(file) && !edition_greater?(bib, doc)
+          Util.warn "Duplicate file `#{file}` for `#{docid.content}` from #{url(docpath)}"
+        end
+      end
+
+      def edition_greater?(doc, bib)
+        doc.edition && bib.edition && doc.edition.content.to_i > bib.edition.content.to_i
+      end
+
+      def replace_substage98?(doc, bib) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        doc.edition&.content == bib.edition&.content &&
+          (doc.status&.substage&.content != "98" || bib.status&.substage&.content == "98")
+      end
+
+      def write_file(file, doc, docid)
+        @files << file
+        index.add_or_update docid.content.to_h, file
+        File.write file, serialize(doc), encoding: "UTF-8"
+      end
+
+      def to_yaml(doc)
+        Item.to_yaml doc
+      end
+
+      def to_xml(doc)
+        Bibdata.to_xml doc
+      end
+
+      #
+      # Create thread worker
+      #
+      # @return [Thread] thread
+      #
+      def thread
+        Thread.new do
+          while (path = queue.pop) != :END
+            yield path
+          end
+        end
+      end
+    end
+  end
+end