relaton-ecma 1.13.0 → 1.14.1

data/grammars/biblio.rng

@@ -33,9 +33,10 @@
       <param name="pattern">([\+\-]?\d{4})((-?)((0[1-9]|1[0-2])((-?)([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6]))))?</param>
     </data>
   </define>
-  <start>
-    <ref name="bibitem"/>
-  </start>
+  <!-- start = bibitem -->
+  <define name="BibData">
+    <ref name="BibliographicItem"/>
+  </define>
   <define name="status">
     <element name="status">
       <ref name="stage"/>
@@ -73,8 +74,14 @@
       <text/>
     </element>
   </define>
-  <define name="script">
+  <define name="locale">
     <a:documentation>ISO-639</a:documentation>
+    <element name="locale">
+      <text/>
+    </element>
+  </define>
+  <define name="script">
+    <a:documentation>ISO-3166</a:documentation>
     <element name="script">
       <text/>
     </element>
@@ -93,6 +100,9 @@
       <!-- multiple languages and scripts possible: comma delimit them if so -->
       <attribute name="language"/>
     </optional>
+    <optional>
+      <attribute name="locale"/>
+    </optional>
     <optional>
       <attribute name="script"/>
     </optional>
@@ -136,6 +146,9 @@
       <!-- multiple languages and scripts possible: comma delimit them if so -->
       <attribute name="language"/>
     </optional>
+    <optional>
+      <attribute name="locale"/>
+    </optional>
     <optional>
       <attribute name="script"/>
     </optional>
@@ -158,27 +171,30 @@
   </define>
   <define name="contributor">
     <element name="contributor">
-      <zeroOrMore>
+      <oneOrMore>
         <ref name="role"/>
-      </zeroOrMore>
+      </oneOrMore>
       <ref name="ContributorInfo"/>
     </element>
   </define>
   <define name="role">
     <element name="role">
-      <optional>
-        <attribute name="type">
-          <choice>
-            <value>author</value>
-            <value>performer</value>
-            <value>publisher</value>
-            <value>editor</value>
-            <value>adapter</value>
-            <value>translator</value>
-            <value>distributor</value>
-          </choice>
-        </attribute>
-      </optional>
+      <attribute name="type">
+        <choice>
+          <value>author</value>
+          <value>performer</value>
+          <value>publisher</value>
+          <value>editor</value>
+          <value>adapter</value>
+          <value>translator</value>
+          <value>distributor</value>
+          <value>realizer</value>
+          <value>owner</value>
+          <value>authorizer</value>
+          <value>enabler</value>
+          <value>subject</value>
+        </choice>
+      </attribute>
       <zeroOrMore>
         <ref name="roledescription"/>
       </zeroOrMore>
@@ -200,6 +216,9 @@
       <optional>
         <ref name="fullname"/>
       </optional>
+      <zeroOrMore>
+        <ref name="credential"/>
+      </zeroOrMore>
       <zeroOrMore>
         <ref name="affiliation"/>
       </zeroOrMore>
@@ -216,6 +235,11 @@
       <ref name="FullNameType"/>
     </element>
   </define>
+  <define name="credential">
+    <element name="credential">
+      <text/>
+    </element>
+  </define>
   <define name="FullNameType">
     <choice>
       <group>
@@ -289,7 +313,9 @@
       <zeroOrMore>
         <ref name="affiliationdescription"/>
       </zeroOrMore>
-      <ref name="organization"/>
+      <optional>
+        <ref name="organization"/>
+      </optional>
     </element>
   </define>
   <define name="affiliationname">
@@ -500,6 +526,17 @@
   </define>
   <define name="localityStack">
     <element name="localityStack">
+      <optional>
+        <attribute name="connective">
+          <choice>
+            <value>and</value>
+            <value>or</value>
+            <value>from</value>
+            <value>to</value>
+            <value/>
+          </choice>
+        </attribute>
+      </optional>
       <zeroOrMore>
         <ref name="locality"/>
       </zeroOrMore>
@@ -512,6 +549,17 @@
   </define>
   <define name="sourceLocalityStack">
     <element name="sourceLocalityStack">
+      <optional>
+        <attribute name="connective">
+          <choice>
+            <value>and</value>
+            <value>or</value>
+            <value>from</value>
+            <value>to</value>
+            <value/>
+          </choice>
+        </attribute>
+      </optional>
       <zeroOrMore>
         <ref name="sourceLocality"/>
       </zeroOrMore>
@@ -614,6 +662,9 @@
         <ref name="BibItemType"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="schema-version"/>
+    </optional>
     <optional>
       <ref name="fetched"/>
     </optional>
@@ -650,6 +701,9 @@
     <zeroOrMore>
       <ref name="language"/>
     </zeroOrMore>
+    <zeroOrMore>
+      <ref name="locale"/>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="script"/>
     </zeroOrMore>
@@ -705,6 +759,9 @@
         <ref name="BibItemType"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="schema-version"/>
+    </optional>
     <optional>
       <ref name="fetched"/>
     </optional>
@@ -741,6 +798,9 @@
     <zeroOrMore>
       <ref name="language"/>
     </zeroOrMore>
+    <zeroOrMore>
+      <ref name="locale"/>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="script"/>
     </zeroOrMore>
@@ -854,6 +914,15 @@
     <optional>
       <attribute name="type"/>
     </optional>
+    <optional>
+      <attribute name="language"/>
+    </optional>
+    <optional>
+      <attribute name="locale"/>
+    </optional>
+    <optional>
+      <attribute name="script"/>
+    </optional>
     <data type="anyURI"/>
   </define>
   <define name="DateType">
@@ -882,6 +951,7 @@
       <value>vote-started</value>
       <value>vote-ended</value>
       <value>announced</value>
+      <value>stable-until</value>
     </choice>
   </define>
   <define name="bdate">
@@ -930,6 +1000,9 @@
       <optional>
         <attribute name="language"/>
       </optional>
+      <optional>
+        <attribute name="locale"/>
+      </optional>
       <optional>
         <attribute name="script"/>
       </optional>
@@ -1253,7 +1326,7 @@
       <value>commentaryOf</value>
       <value>hasCommentary</value>
       <value>related</value>
-      <value>complements</value>
+      <value>hasComplement</value>
       <value>complementOf</value>
       <value>obsoletes</value>
       <value>obsoletedBy</value>

data/grammars/relaton-ecma-compile.rng

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="basicdoc.rng"/>
+  <include href="relaton-ecma.rng"/>
+  <start>
+    <choice>
+      <ref name="bibitem"/>
+      <ref name="bibdata"/>
+    </choice>
+  </start>
+</grammar>

data/grammars/relaton-ecma.rng

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="biblio-standoc.rng"/>
+</grammar>

data/lib/relaton_ecma/bibliographic_item.rb

@@ -0,0 +1,12 @@
+module RelatonEcma
+  class BibliographicItem < RelatonBib::BibliographicItem
+    #
+    # Fetch flavor ext schema version
+    #
+    # @return [String] schema version
+    #
+    def ext_schema
+      @ext_schema ||= schema_versions["relaton-model-ecma"]
+    end
+  end
+end

data/lib/relaton_ecma/data_fetcher.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "English"
+require "mechanize"
+require "relaton_ecma"
+
+module RelatonEcma
+  class DataFetcher
+    URL = "https://www.ecma-international.org/publications-and-standards/"
+
+    # @param [String] :output directory to output documents
+    # @param [String] :format output format (xml, yaml, bibxml)
+    def initialize(output: "data", format: "yaml")
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      @files = []
+      @index = Relaton::Index.find_or_create :ECMA
+      @agent = Mechanize.new
+      @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+    end
+
+    # @param bib [RelatonItu::ItuBibliographicItem]
+    def write_file(bib) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s]}, "_")
+      id += "-#{bib.edition.content.gsub('.', '_')}" if bib.edition
+      extent = bib.extent.detect { |e| e.type == "volume" }
+      id += "-#{extent.reference_from}" if extent
+      file = "#{@output}/#{id}.#{@ext}"
+      if @files.include? file
+        warn "Duplicate file #{file}"
+      else
+        @files << file
+        File.write file, render_doc(bib), encoding: "UTF-8"
+        @index.add_or_update index_id(bib), file
+      end
+    end
+
+    def index_id(bib)
+      { id: bib.docidentifier[0].id }.tap do |i|
+        i[:ed] = bib.edition.content if bib.edition
+        extent = bib.extent.detect { |e| e.type == "volume" }
+        i[:vol] = extent.reference_from if extent
+      end
+    end
+
+    def render_doc(bib)
+      case @format
+      when "yaml" then bib.to_hash.to_yaml
+      when "xml" then bib.to_xml bibdata: true
+      when "bibxml" then bib.to_bibxml
+      end
+    end
+
+    # @param hit [Nokogiri::XML::Element]
+    def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      DataParser.new(hit).parse.each { |item| write_file item }
+    end
+
+    # @param type [String]
+    def html_index(type) # rubocop:disable Metrics/MethodLength
+      result = @agent.get "#{URL}#{type}/"
+      # @last_call_time = Time.now
+      result.xpath(
+        "//li/span[1]/a",
+        "//div[contains(@class, 'entry-content-wrapper')][.//a[.='Download']]",
+      ).each do |hit|
+        # workers << hit
+        parse_page(hit)
+      rescue StandardError => e
+        warn e.message
+        warn e.backtrace
+      end
+    end
+
+    #
+    # Fetch data from Ecma website.
+    #
+    # @return [void]
+    #
+    def fetch
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      FileUtils.mkdir_p @output
+
+      html_index "standards"
+      html_index "technical-reports"
+      html_index "mementos"
+      @index.save
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+  end
+end

data/lib/relaton_ecma/data_parser.rb

@@ -0,0 +1,215 @@
+module RelatonEcma
+  class DataParser
+    MATTRS = %i[docid title date link].freeze
+    ATTRS = MATTRS + %i[abstract relation edition].freeze
+
+    #
+    # Initialize parser
+    #
+    # @param [Nokogiri::XML::Element] hit document hit
+    #
+    def initialize(hit)
+      @hit = hit
+      @bib = {
+        type: "standard", language: ["en"], script: ["Latn"], place: ["Geneva"], doctype: "document"
+      }
+      @agent = Mechanize.new
+    end
+
+    def parse # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      if @hit[:href]
+        @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+        @doc = get_page @hit[:href]
+        ATTRS.each { |a| @bib[a] = send "fetch_#{a}" }
+      else
+        MATTRS.each { |a| @bib[a] = send "fetch_mem_#{a}" }
+      end
+      @bib[:contributor] = contributor
+      items = [BibliographicItem.new(**@bib)]
+      items + parse_editions
+    end
+
+    #
+    # Get page with retries
+    #
+    # @param [String] url url to fetch
+    #
+    # @return [Mechanize::Page] document
+    #
+    def get_page(url)
+      3.times do |n|
+        sleep n
+        doc = @agent.get url
+        return doc
+      rescue StandardError => e
+        warn e.message
+      end
+    end
+
+    #
+    # Parse editions
+    #
+    # @param [Mechanize::Page] doc document
+    # @param [Hash] bib bibliographic item the last edition
+    #
+    # @return [void]
+    #
+    def parse_editions # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      return [] unless @doc
+
+      docid = @bib[:docid]
+      @doc.xpath('//div[@id="main"]/div[1]/div/main/article/div/div/standard/div/ul/li').map do |hit|
+        id, ed, @bib[:date], vol = edition_id_parts hit.at("./span", "./a").text
+        @bib[:link] = edition_link(hit) + edition_translation_link(ed)
+        next if ed.nil? || ed.empty?
+
+        @bib[:docid] = id.nil? || id.empty? ? docid : fetch_docid(id)
+        @bib[:edition] = RelatonBib::Edition.new(content: ed)
+        @bib[:extent] = vol && [RelatonBib::Locality.new("volume", vol)]
+        BibliographicItem.new(**@bib)
+      end.compact
+    end
+
+    def edition_link(hit)
+      { "src" => hit.at("./a"), "pdf" => hit.at("./span/a") }.map do |type, a|
+        RelatonBib::TypedUri.new(type: type, content: a[:href]) if a
+      end.compact
+    end
+
+    #
+    # Parse edition and date
+    #
+    # @param [String] text identifier text
+    #
+    # @return [Array<String,nil,Array<RelatonBib::BibliographicDate>>] edition and date
+    #
+    def edition_id_parts(text) # rubocop:disable Metrics/MethodLength
+      %r{^
+        (?<id>\w+(?:[\d-]+|\sTR/\d+)),?\s
+        (?:Volume\s(?<vol>[\d.]+),?\s)?
+        (?<ed>[\d.]+)(?:st|nd|rd|th)?\sedition
+        (?:[,.]\s(?<dt>\w+\s\d+))?
+      }x =~ text
+      date = [dt].compact.map do |d|
+        on = Date.strptime(d, "%B %Y").strftime("%Y-%m")
+        RelatonBib::BibliographicDate.new(type: "published", on: on)
+      end
+      [id, ed, date, vol]
+    end
+
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(id = nil)
+      id ||= @hit.text
+      [RelatonBib::DocumentIdentifier.new(type: "ECMA", id: id, primary: true)]
+    end
+
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link # rubocop:disable Metrics/AbcSize
+      link = []
+      link << RelatonBib::TypedUri.new(type: "src", content: @hit[:href]) if @hit[:href]
+      ref = @doc.at('//div[@class="ecma-item-content-wrapper"]/span/a',
+                    '//div[@class="ecma-item-content-wrapper"]/a')
+      link << RelatonBib::TypedUri.new(type: "pdf", content: ref[:href]) if ref
+      link + edition_translation_link(@bib[:edition]&.content)
+    end
+
+    def fetch_mem_link
+      @hit.xpath("./div/section/div/p/a").map do |a|
+        RelatonBib::TypedUri.new(type: "pdf", content: a[:href])
+      end
+    end
+
+    def edition_translation_link(edition)
+      translation_link.select { |l| l[:ed] == edition }.map { |l| l[:link] }
+    end
+
+    def translation_link
+      return [] unless @doc
+
+      @translation_link ||= @doc.xpath("//main/article/div/div/standard/div[2]/ul/li").map do |l|
+        a = l.at("span/a")
+        id = l.at("span").text
+        %r{\w+[\d-]+,\s(?<lang>\w+)\sversion,\s(?<ed>[\d.]+)(?:st|nd|rd|th)\sedition} =~ id
+        case lang
+        when "Japanese"
+          { ed: ed, link: RelatonBib::TypedUri.new(type: "pdf", language: "ja", script: "Jpan", content: a[:href]) }
+        end
+      end.compact
+    end
+
+    # @return [Array<Hash>]
+    def fetch_title
+      @doc.xpath('//p[@class="ecma-item-short-description"]').map do |t|
+        { content: t.text.strip, language: "en", script: "Latn" }
+      end
+    end
+
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract
+      content = @doc.xpath('//div[@class="ecma-item-content"]/p').map do |a|
+        a.text.strip.squeeze(" ").gsub(/\r\n/, "")
+      end.join "\n"
+      return [] if content.empty?
+
+      [RelatonBib::FormattedString.new(content: content, language: "en", script: "Latn")]
+    end
+
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date
+      @doc.xpath('//p[@class="ecma-item-edition"]').map do |d|
+        date = d.text.split(", ").last
+        RelatonBib::BibliographicDate.new type: "published", on: date
+      end
+    end
+
+    # @return [Array<Hash>]
+    def fetch_relation # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+      @doc.xpath("//ul[@class='ecma-item-archives']/li").map do |rel|
+        ref, ed, date, vol = edition_id_parts rel.at("span").text
+        next if ed.nil? || ed.empty?
+
+        fref = RelatonBib::FormattedRef.new content: ref, language: "en", script: "Latn"
+        docid = RelatonBib::DocumentIdentifier.new(type: "ECMA", id: ref, primary: true)
+        link = rel.xpath("span/a").map { |l| RelatonBib::TypedUri.new type: "pdf", content: l[:href] }
+        edition = RelatonBib::Edition.new content: ed
+        extent = vol && [RelatonBib::Locality.new("volume", vol)]
+        bibitem = BibliographicItem.new(
+          docid: [docid], formattedref: fref, date: date, edition: edition,
+          link: link, extent: extent
+        )
+        { type: "updates", bibitem: bibitem }
+      end.compact
+    end
+
+    #
+    # @return [RelatonBib::Edition, nil]
+    #
+    def fetch_edition
+      cnt = @doc.at('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
+      RelatonBib::Edition.new(content: cnt) if cnt && !cnt.empty?
+    end
+
+    def contributor
+      org = RelatonBib::Organization.new name: "Ecma International"
+      [{ entity: org, role: [{ type: "publisher" }] }]
+    end
+
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_mem_docid
+      code = "ECMA MEM/#{@hit.at('div[1]//p').text}"
+      fetch_docid code
+    end
+
+    def fetch_mem_date
+      date = @hit.at("div[2]//p").text
+      on = Date.strptime(date, "%B %Y").strftime "%Y-%m"
+      [RelatonBib::BibliographicDate.new(type: "published", on: on)]
+    end
+
+    def fetch_mem_title
+      year = @hit.at("div[1]//p").text
+      content = "\"Memento #{year}\" for year #{year}"
+      [{ content: content, language: "en", script: "Latn" }]
+    end
+  end
+end

data/lib/relaton_ecma/ecma_bibliography.rb

@@ -3,28 +3,74 @@
 module RelatonEcma
   # IETF bibliography module
   module EcmaBibliography
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ecma/master/"
+
     class << self
-      # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
-      # @return [RelatonBib::BibliographicEcma]
-      def search(code)
-        Scrapper.scrape_page code
+      #
+      # Search for a reference on the IETF website.
+      #
+      # @param ref [String] the ECMA standard reference to look up (e..g "ECMA-6")
+      #
+      # @return [Array<Hash>]
+      #
+      def search(ref)
+        refparts = parse_ref ref
+        return false unless refparts
+
+        index = Relaton::Index.find_or_create :ECMA, url: "#{ENDPOINT}index.zip"
+        index.search { |row| match_ref refparts, row }
+      end
+
+      def parse_ref(ref)
+        %r{^
+          (?<id>ECMA(?:[\d-]+|\s\w+/\d+))
+          (?:\sed(?<ed>[\d.]+))?
+          (?:\svol(?<vol>\d+))?
+        }x.match ref
+      end
+
+      def match_ref(refparts, row)
+        row[:id][:id] == refparts[:id] &&
+          (refparts[:ed].nil? || row[:id][:ed] == refparts[:ed]) &&
+          (refparts[:vol].nil? || row[:id][:vol] == refparts[:vol])
       end
 
       # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
       # @param year [String] not used
       # @param opts [Hash] not used
-      # @return [RelatonBib::BibliographicItem] Relaton of reference
+      # @return [RelatonEcma::BibliographicItem] Relaton of reference
       def get(code, _year = nil, _opts = {})
         warn "[relaton-ecma] (\"#{code}\") fetching..."
-        result = search code
+        result = fetch_doc(code)
         if result
           warn "[relaton-ecma] (\"#{code}\") found #{result.docidentifier.first.id}"
+          # item
         else
-          warn "[relaton-ecma] WARNING no match found online for #{code}. "\
+          warn "[relaton-ecma] WARNING no match found online for #{code}. " \
                "The code must be exactly like it is on the standards website."
         end
         result
       end
+
+      def compare_edition_volume(aaa, bbb)
+        comp = bbb[:id][:ed] <=> aaa[:id][:ed]
+        comp.zero? ? aaa[:id][:vol] <=> bbb[:id][:vol] : comp
+      end
+
+      def fetch_doc(code) # rubocop:disable Metrics/AbcSize
+        row = search(code).min { |a, b| compare_edition_volume a, b }
+        return unless row
+
+        url = "#{ENDPOINT}#{row[:file]}"
+        doc = OpenURI.open_uri url
+        hash = YAML.safe_load doc
+        hash["fetched"] = Date.today.to_s
+        BibliographicItem.from_hash hash
+      rescue OpenURI::HTTPError => e
+        return if e.io.status.first == "404"
+
+        raise RelatonBib::RequestError, "No document found for #{code} reference. #{e.message}"
+      end
     end
   end
 end

data/lib/relaton_ecma/hash_converter.rb

@@ -0,0 +1,11 @@
+module RelatonEcma
+  class HashConverter < RelatonBib::HashConverter
+    class << self
+      # @param item_hash [Hash]
+      # @return [RelatonEcma::BibliographicItem]
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+    end
+  end
+end