relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/ecma/standard_parser.rb

@@ -0,0 +1,134 @@
+module Relaton
+  module Ecma
+    class StandardParser
+      include ParserCommon
+
+      ATTRS = %i[docidentifier title date source abstract relation edition ext].freeze
+
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Mechanize::Page] doc fetched document page
+      # @param [Hash] errors error tracking hash
+      def initialize(hit:, doc:, errors: {})
+        @hit = hit
+        @doc = doc
+        @errors = errors
+      end
+
+      # @return [Hash] bibliographic item attributes
+      def to_bib_hash
+        bib = default_bib_hash
+        ATTRS.each { |a| bib[a] = send "fetch_#{a}" }
+        bib
+      end
+
+      # @return [Array] precomputed translation sources
+      def translation_source
+        @translation_source ||= parse_translation_source
+      end
+
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def fetch_docidentifier
+        result = super(@hit.text)
+        @errors[:standard_docidentifier] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title
+        result = @doc.xpath('//p[@class="ecma-item-short-description"]').map do |t|
+          Bib::Title.new(content: t.text.strip, language: "en", script: "Latn")
+        end
+        @errors[:standard_title] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
+      def fetch_abstract
+        content = @doc.xpath('//div[@class="ecma-item-content"]/p').map do |a|
+          a.text.strip.squeeze(" ").gsub("\r\n", "")
+        end.join "\n"
+        return [] if content.empty?
+
+        result = [Bib::Abstract.new(content: content, language: "en", script: "Latn")]
+        @errors[:standard_abstract] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date
+        result = @doc.xpath('//p[@class="ecma-item-edition"]').map do |d|
+          date = d.text.split(", ").last
+          Bib::Date.new type: "published", at: date
+        end
+        @errors[:standard_date] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source # rubocop:disable Metrics/AbcSize
+        source = []
+        source << Bib::Uri.new(type: "src", content: @hit[:href]) if @hit[:href]
+        ref = @doc.at('//div[@class="ecma-item-content-wrapper"]/span/a',
+                      '//div[@class="ecma-item-content-wrapper"]/a')
+        source << Bib::Uri.new(type: "pdf", content: ref[:href]) if ref
+        result = source + edition_translation_source(fetch_edition_content)
+        @errors[:standard_source] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Relation>]
+      def fetch_relation # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+        edition_parser = EditionParser.new(doc: @doc, bib: {}, errors: @errors)
+        result = @doc.xpath("//ul[@class='ecma-item-archives']/li").filter_map do |rel|
+          ref, ed, date, vol = edition_parser.edition_id_parts rel.at("span").text
+          next if ed.nil? || ed.empty?
+
+          docid = Bib::Docidentifier.new(type: "ECMA", content: ref, primary: true)
+          source = rel.xpath("span/a").map { |l| Bib::Uri.new type: "pdf", content: l[:href] }
+          edition = Bib::Edition.new content: ed
+          extent = edition_parser.create_extent(vol)
+          @errors[:standard_relation_extent] &&= extent.nil?
+          bibitem = ItemData.new(
+            docidentifier: [docid], formattedref: Bib::Formattedref.new(content: ref), date: date, edition: edition,
+            source: source, extent: extent
+          )
+          Bib::Relation.new(type: "updates", bibitem: bibitem)
+        end
+        @errors[:standard_relation] &&= result.empty?
+        result
+      end
+
+      # @return [Relaton::Bib::Edition, nil]
+      def fetch_edition
+        cnt = fetch_edition_content
+        result = Bib::Edition.new(content: cnt) if cnt && !cnt.empty?
+        @errors[:standard_edition] &&= result.nil?
+        result
+      end
+
+      private
+
+      def fetch_edition_content
+        @doc.at('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
+      end
+
+      def edition_translation_source(edition)
+        translation_source.select { |s| s[:ed] == edition }.map { |s| s[:source] }
+      end
+
+      def parse_translation_source
+        return [] unless @doc
+
+        @doc.xpath("//h2[.='Translations']/following-sibling::ul/li").map do |l|
+          a = l.at("span/a")
+          id = l.at("span").text
+          %r{\w+[\d-]+,\s(?<lang>\w+)\sversion,\s(?<ed>[\d.]+)(?:st|nd|rd|th)\sedition} =~ id
+          case lang
+          when "Japanese"
+            { ed: ed, source: Bib::Uri.new(type: "pdf", language: "ja", script: "Jpan", content: a[:href]) }
+          end
+        end.compact
+      end
+    end
+  end
+end

data/lib/relaton/ecma/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ecma
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-ecma".freeze
+    end
+  end
+end

data/lib/relaton/ecma.rb

@@ -0,0 +1,24 @@
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "ecma/util"
+require_relative "ecma/item_data"
+require_relative "ecma/item"
+require_relative "ecma/bibitem"
+require_relative "ecma/bibdata"
+require_relative "ecma/bibliography"
+
+module Relaton
+  module Ecma
+    INDEXFILE = "index-v1"
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/etsi/bibdata.rb

@@ -0,0 +1,10 @@
+require_relative "item"
+
+module Relaton
+  module Etsi
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/etsi/bibitem.rb

@@ -0,0 +1,10 @@
+require_relative "item"
+
+module Relaton
+  module Etsi
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/etsi/bibliography.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Etsi
+    # Methods for search IANA standards.
+    module Bibliography
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-etsi/refs/heads/v2/"
+
+      # @param text [String]
+      # @return [Relaton::Etsi::ItemData, nil]
+      def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        index = Relaton::Index.find_or_create :etsi, url: "#{SOURCE}index-v1.zip", file: INDEX_FILE
+        row = index.search(text).min_by { |r| r[:id] }
+        return unless row
+
+        url = "#{SOURCE}#{row[:file]}"
+        resp = Net::HTTP.get_response URI(url)
+        return unless resp.code == "200"
+
+        Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # @param ref [String] the ETSI standard Code to look up
+      # @param year [String, nil] year
+      # @param opts [Hash] options
+      # @return [Relaton::Etsi::ItemData, nil]
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(ref)
+        unless result
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        result
+      end
+
+      extend Bibliography
+    end
+  end
+end

data/lib/relaton/etsi/data_fetcher.rb

@@ -0,0 +1,128 @@
+require "date"
+require "json"
+require "mechanize"
+require "relaton/core"
+require "relaton/index"
+require_relative "../etsi"
+require_relative "data_parser"
+
+module Relaton
+  module Etsi
+    class DataFetcher < Core::DataFetcher
+      PAGE_SIZE = 50
+
+      SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
+        "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=0&onApproval=1&published=1&" \
+        "withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
+        "harmonized=0&keyword=&TB=&stdType=&frequency=&mandate=&collection=&sort=1&x=%<timestamp>s".freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create :etsi, file: INDEX_FILE
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      #
+      # Fetch all ETSI documents from the ETSI website.
+      #
+      # @param [Object] _source unused, required by superclass interface
+      #
+      def fetch(_source = nil)
+        first_page = fetch_page(1)
+        process_records(first_page)
+        fetch_remaining_pages(first_page)
+        index.save
+        report_errors
+      end
+
+      def fetch_remaining_pages(first_page)
+        total = first_page.first ? first_page.first["total_count"].to_i : 0
+        total_pages = (total / PAGE_SIZE.to_f).ceil
+        (2..total_pages).each do |page|
+          records = fetch_page(page)
+          break if records.empty?
+
+          process_records(records)
+        end
+      end
+
+      def fetch_page(page)
+        date = Time.now.to_date + 1
+        timestamp = (Time.now.to_f * 1000).to_i
+        url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
+        JSON.parse(fetch_with_retry(url))
+      end
+
+      def process_records(records)
+        records.each do |record|
+          save DataParser.new(normalize(record), @errors).parse
+        end
+      end
+
+      def normalize(record)
+        {
+          "ETSI deliverable" => record["ETSI_DELIVERABLE"],
+          "title" => record["TITLE"],
+          "Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
+          "PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
+          "Status" => derive_status(record),
+          "Keywords" => record["Keywords"].to_s,
+          "Technical body" => record["TB"],
+          "Scope" => record["Scope"],
+        }
+      end
+
+      def derive_status(record)
+        return "Withdrawn" if record["ACTION_TYPE"] == "WD"
+
+        code = record["STATUS_CODE"].to_i
+        return "On Approval" if code < 12
+        return "Historical" if code == 13
+
+        "Published"
+      end
+
+      NETWORK_ERRORS = [
+        Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
+        SocketError, Errno::ECONNRESET
+      ].freeze
+
+      def fetch_with_retry(url, retries: 3, delay: 2) # rubocop:disable Metrics/MethodLength
+        attempt = 0
+        begin
+          Mechanize.new.get(url).body
+        rescue *NETWORK_ERRORS => e
+          attempt += 1
+          if attempt <= retries
+            Util.info "Fetch failed (#{e.message}), " \
+                      "retrying (#{attempt}/#{retries})..."
+            sleep delay * attempt
+            retry
+          end
+          raise
+        end
+      end
+
+      def save(bib)
+        id = bib.docidentifier.first.content
+        file = output_file id
+        File.write file, serialize(bib), encoding: "UTF-8"
+        index.add_or_update id, file
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_xml(bib)
+        bib.to_xml bibdata: true
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/etsi/data_parser.rb

@@ -0,0 +1,208 @@
+require_relative "pubid"
+module Relaton
+  module Etsi
+    class DataParser
+      ATTRS = %i[
+        title docnumber source date docidentifier version status contributor
+        keyword ext abstract language script
+      ].freeze
+
+      def initialize(row, errors = {})
+        @row = row
+        @errors = errors
+      end
+
+      def parse
+        args = ATTRS.to_h { |attr| [attr, send(attr)] }
+        ItemData.new(type: "standard", **args)
+      end
+
+      def pubid
+        return @pubid if defined?(@pubid)
+
+        unless @row["ETSI deliverable"]
+          @errors[:pubid] &&= true
+          @pubid = nil
+          return @pubid
+        end
+
+        @errors[:pubid] &&= false
+        @pubid = PubId.parse(@row["ETSI deliverable"])
+      end
+
+      def title
+        result = if @row["title"]
+                   [Bib::Title.new(
+                     content: @row["title"], language: "en", script: "Latn",
+                   )]
+                 else
+                   []
+                 end
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def docnumber
+        @row["ETSI deliverable"]
+      end
+
+      def source
+        urls = []
+        if @row["Details link"]
+          urls << Bib::Uri.new(content: @row["Details link"], type: "src")
+        end
+        if @row["PDF link"]
+          urls << Bib::Uri.new(content: @row["PDF link"], type: "pdf")
+        end
+        @errors[:source] &&= urls.empty?
+        urls
+      end
+
+      def date
+        result = if pubid&.date
+                   [Bib::Date.new(type: "published", at: pubid.date)]
+                 else
+                   []
+                 end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      def docidentifier
+        result = if @row["ETSI deliverable"]
+                   [Bib::Docidentifier.new(
+                     content: @row["ETSI deliverable"],
+                     type: "ETSI",
+                     primary: true,
+                   )]
+                 else
+                   []
+                 end
+        @errors[:docidentifier] &&= result.empty?
+        result
+      end
+
+      def version
+        return [] unless pubid&.version
+
+        [Bib::Version.new(draft: pubid.version)]
+      end
+
+      def status
+        unless @row["Status"]
+          @errors[:status] &&= true
+          return
+        end
+
+        stage = @row["Status"]
+        if stage == "On Approval"
+          stage = "#{pubid&.type} approval"
+        end
+        @errors[:status] &&= false
+        Status.new(stage: stage)
+      end
+
+      def contributor
+        contribs = [publisher_contributor]
+        contribs << committee_contributor if @row["Technical body"]
+        contribs
+      end
+
+      def publisher_contributor
+        org = Bib::Organization.new(
+          name: [etsi_org_name], abbreviation: etsi_abbreviation,
+        )
+        role = Bib::Contributor::Role.new(type: "publisher")
+        Bib::Contributor.new(organization: org, role: [role])
+      end
+
+      def etsi_org_name
+        Bib::TypedLocalizedString.new(
+          content: "European Telecommunications Standards Institute",
+          language: "en",
+          script: "Latn",
+        )
+      end
+
+      def etsi_abbreviation
+        Bib::LocalizedString.new(
+          content: "ETSI",
+          language: "en",
+          script: "Latn",
+        )
+      end
+
+      def committee_contributor
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(
+          type: "author", description: [desc],
+        )
+        Bib::Contributor.new(organization: committee_org, role: [role])
+      end
+
+      def committee_org
+        Bib::Organization.new(
+          name: [etsi_name],
+          abbreviation: etsi_abbreviation,
+          subdivision: [technical_committee_subdivision],
+        )
+      end
+
+      def etsi_name
+        Bib::TypedLocalizedString.new(
+          content: "European Telecommunications Standards Institute",
+        )
+      end
+
+      def technical_committee_subdivision
+        Bib::Subdivision.new(
+          name: [Bib::TypedLocalizedString.new(
+            content: @row["Technical body"],
+          )],
+          type: "technical-committee",
+        )
+      end
+
+      def keyword
+        unless @row["Keywords"]
+          @errors[:keyword] &&= true
+          return []
+        end
+
+        result = @row["Keywords"].split(",").map do |kw|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: kw.strip, language: "en", script: "Latn"))
+        end
+        @errors[:keyword] &&= result.empty?
+        result
+      end
+
+      def ext
+        Ext.new(doctype: doctype, flavor: "etsi")
+      end
+
+      def doctype
+        return unless pubid
+
+        Doctype.create_from_abbreviation pubid.type
+      end
+
+      def abstract
+        result = if @row["Scope"]
+                   [Bib::Abstract.new(content: @row["Scope"], language: "en", script: "Latn")]
+                 else
+                   []
+                 end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      def language
+        ["en"]
+      end
+
+      def script
+        ["Latn"]
+      end
+    end
+  end
+end

data/lib/relaton/etsi/doctype.rb

@@ -0,0 +1,30 @@
+module Relaton
+  module Etsi
+    class Doctype < Bib::Doctype
+      DOCTYPES = {
+        "EN" => "European Standard",
+        "ES" => "ETSI Standard",
+        "EG" => "ETSI Guide",
+        "TS" => "Technical Specification",
+        "GS" => "Group Specification",
+        "GR" => "Group Report",
+        "TR" => "Technical Report",
+        "ETR" => "ETSI Technical Report",
+        "GTS" => "GSM Technical Specification",
+        "SR" => "Special Report",
+        "TCRTR" => "Technical Committee Reference Technical Report",
+        "TBR" => "Technical Basis for Regulation",
+        "ETS" => "European Telecommunication Standard",
+        "I-ETS" => "Interim European Telecommunication Standard",
+        "NET" => "Norme Européenne de Télécommunication",
+      }.freeze
+
+      attribute :abbreviation, :string, values: DOCTYPES.keys
+      attribute :content, :string, values: DOCTYPES.values
+
+      def self.create_from_abbreviation(abbreviation)
+        new(content: DOCTYPES[abbreviation], abbreviation: abbreviation)
+      end
+    end
+  end
+end

data/lib/relaton/etsi/ext.rb

@@ -0,0 +1,31 @@
+require_relative "doctype"
+
+module Relaton
+  module Etsi
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :marker, :string, values: %w[Current Superseded]
+      attribute :frequency, :string, collection: true
+      attribute :mandate, :string, collection: true
+      attribute :custom_collection, :string, values: %W[
+        HSs\scited\sin\sOJ HSs\snot\syet\scited\sin\sOJ HSs\sRED\scited\sin\sOJ HSs\sEMC\scited\sin\sOJ
+      ]
+
+      xml do
+        map_element "marker", to: :marker
+        map_element "frequency", to: :frequency
+        map_element "mandate", to: :mandate
+        map_element "custom-collection", to: :custom_collection
+      end
+
+      key_value do
+        map_element "marker", to: :marker
+        map_element "frequency", to: :frequency
+        map_element "mandate", to: :mandate
+        map_element "custom_collection", to: :custom_collection
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-etsi"]
+    end
+  end
+end

data/lib/relaton/etsi/item.rb

@@ -0,0 +1,15 @@
+require "relaton/bib"
+require_relative "item_data"
+require_relative "ext"
+require_relative "status"
+
+module Relaton
+  module Etsi
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :status, Status
+    end
+  end
+end

data/lib/relaton/etsi/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Etsi
+    class ItemData < Bib::ItemData
+    end
+  end
+end