relaton-etsi 1.17.0

data/grammars/relaton-etsi-compile.rng

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="basicdoc.rng"/>
+  <include href="relaton-etsi.rng"/>
+  <start>
+    <choice>
+      <ref name="bibitem"/>
+      <ref name="bibdata"/>
+    </choice>
+  </start>
+</grammar>

data/grammars/relaton-etsi.rng

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="biblio-standoc.rng">
+    <define name="BibDataExtensionType">
+      <optional>
+        <attribute name="schema-version"/>
+      </optional>
+      <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
+      <ref name="editorialgroup"/>
+      <ref name="marker"/>
+      <zeroOrMore>
+        <ref name="frequency"/>
+      </zeroOrMore>
+      <zeroOrMore>
+        <ref name="mandate"/>
+      </zeroOrMore>
+      <optional>
+        <ref name="custom-collection"/>
+      </optional>
+    </define>
+    <define name="status">
+      <element name="status">
+        <ref name="stage"/>
+      </element>
+    </define>
+    <define name="stage">
+      <element name="stage">
+        <ref name="StageType"/>
+      </element>
+    </define>
+    <define name="doctype">
+      <element name="doctype">
+        <optional>
+          <attribute name="abbreviation">
+            <ref name="DocumentTypeAbbreviation"/>
+          </attribute>
+        </optional>
+        <ref name="DocumentType"/>
+      </element>
+    </define>
+    <define name="DocumentType">
+      <choice>
+        <value>European Standard</value>
+        <value>ETSI Standard</value>
+        <value>ETSI Guide</value>
+        <value>Technical Specification</value>
+        <value>Group Specification</value>
+        <value>Group Report</value>
+        <value>Technical Report</value>
+        <value>ETSI Technical Report</value>
+        <value>GSM Technical Specification</value>
+        <value>Special Report</value>
+        <value>Technical Committee Reference Technical Report</value>
+        <value>Technical Basis for Regulation</value>
+        <value>European Telecommunication Standard</value>
+        <value>Interim European Telecommunication Standard</value>
+        <value>Norme Européenne de Télécommunication</value>
+      </choice>
+    </define>
+  </include>
+  <define name="StageType">
+    <choice>
+      <value>EN approval</value>
+      <value>SG approval</value>
+      <value>ES approval</value>
+      <value>Published</value>
+      <value>Withdrawn</value>
+      <value>Historical</value>
+    </choice>
+  </define>
+  <define name="DocumentTypeAbbreviation">
+    <choice>
+      <value>EN</value>
+      <value>ES</value>
+      <value>EG</value>
+      <value>TS</value>
+      <value>GS</value>
+      <value>GR</value>
+      <value>TR</value>
+      <value>ETR</value>
+      <value>GTS</value>
+      <value>SR</value>
+      <value>TCRTR</value>
+      <value>TBR</value>
+      <value>ETS</value>
+      <value>I-ETS</value>
+      <value>NET</value>
+    </choice>
+  </define>
+  <define name="marker">
+    <element name="marker">
+      <choice>
+        <value>Current</value>
+        <value>Superseded</value>
+      </choice>
+    </element>
+  </define>
+  <define name="frequency">
+    <element name="frequency">
+      <text/>
+    </element>
+  </define>
+  <define name="mandate">
+    <element name="mandate">
+      <text/>
+    </element>
+  </define>
+  <define name="custom-collection">
+    <element name="custom-collection">
+      <choice>
+        <value>HSs cited in OJ</value>
+        <value>HSs not yet cited in OJ</value>
+        <value>HSs RED cited in OJ</value>
+        <value>HSs EMC cited in OJ</value>
+      </choice>
+    </element>
+  </define>
+</grammar>

data/lib/relaton_etsi/bibliographic_item.rb

@@ -0,0 +1,75 @@
+module RelatonEtsi
+  class BibliographicItem < RelatonBib::BibliographicItem
+    MAKER = %w[Current Superseded].freeze
+    CUSTOM_COLLECTION = ["HSs cited in OJ", "HSs not yet cited in OJ", "HSs RED cited in OJ", "HSs EMC cited in OJ"].freeze
+
+    attr_reader :marker, :frequency, :mandate, :custom_collection
+
+    #
+    # Constructor.
+    #
+    # @param [RelatonEtsi::DocumentType] doctype document type
+    # @param [String] marker current or superseded
+    # @param [Array<String>] frequency list of frequencies
+    # @param [Array<String>] mandate list of mandates
+    # @param [String, nil] custom_collection custom collection
+    #
+    def initialize(**args)
+      @marker = args.delete(:marker)
+      @frequency = args.delete(:frequency) || []
+      @mandate = args.delete(:mandate) || []
+      @custom_collection = args.delete(:custom_collection)
+      super(**args)
+    end
+
+    def self.from_hash(hash)
+      item_hash = HashConverter.hash_to_bib(hash)
+      new(**item_hash)
+    end
+
+    #
+    # Fetch flavour schema version
+    #
+    # @return [String] flavour schema versio
+    #
+    def ext_schema
+      @ext_schema ||= schema_versions["relaton-model-etsi"]
+    end
+
+    def to_xml(**opts) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      super(**opts) do |b|
+        if opts[:bibdata] && (doctype || subdoctype || editorialgroup || marker || frequency.any? || mandate.any? || custom_collection)
+          ext = b.ext do
+            doctype&.to_xml b
+            b.subdoctype subdoctype if subdoctype
+            editorialgroup&.to_xml b
+            b.marker marker if marker
+            frequency.each { |f| b.frequency f }
+            mandate.each { |m| b.mandate m }
+            b.send("custom-collection", custom_collection) if custom_collection
+          end
+          ext["schema-version"] = ext_schema unless opts[:embedded]
+        end
+      end
+    end
+
+    def to_hash # rubocop:disable Metrics/AbcSize
+      hash = super
+      hash["marker"] = marker if marker
+      hash["frequency"] = frequency if frequency.any?
+      hash["mandate"] = mandate if mandate.any?
+      hash["custom_collection"] = custom_collection if custom_collection
+      hash
+    end
+
+    def to_asciibib(prefix = "") # rubocop:disable Metrics/AbcSize
+      out = super
+      pref = prefix.empty? ? prefix : "#{prefix}."
+      out += "#{pref}marker:: #{marker}\n" if marker
+      out += frequency.map { |f| "#{pref}frequency:: #{f}\n" }.join
+      out += mandate.map { |m| "#{pref}mandate:: #{m}\n" }.join
+      out += "#{pref}custom_collection:: #{custom_collection}" if custom_collection
+      out
+    end
+  end
+end

data/lib/relaton_etsi/bibliography.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module RelatonEtsi
+  # Methods for search IANA standards.
+  module Bibliography
+    SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-etsi/main/"
+    INDEX_FILE = "index-v1.yaml"
+
+    # @param text [String]
+    # @return [RelatonBib::BibliographicItem]
+    def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      index = Relaton::Index.find_or_create :etsi, url: "#{SOURCE}index-v1.zip", file: INDEX_FILE
+      row = index.search(text).min_by { |r| r[:id] }
+      return unless row
+
+      url = "#{SOURCE}#{row[:file]}"
+      resp = Net::HTTP.get_response URI(url)
+      return unless resp.code == "200"
+
+      hash = YAML.safe_load resp.body
+      bib_hash = HashConverter.hash_to_bib(hash)
+      bib_hash[:fetched] = Date.today.to_s
+      BibliographicItem.new(**bib_hash)
+    rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+           EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+           Net::ProtocolError, Errno::ETIMEDOUT => e
+      raise RelatonBib::RequestError, e.message
+    end
+
+    # @param ref [String] the ETSI standard Code to look up
+    # @param year [String, nil] year
+    # @param opts [Hash] options
+    # @return [RelatonEtsi::BibliographicItem]
+    def get(ref, _year = nil, _opts = {})
+      Util.warn "(#{ref}) Fetching from Relaton repository ..."
+      result = search(ref)
+      unless result
+        Util.warn "(#{ref}) Not found."
+        return
+      end
+
+      Util.warn "(#{ref}) Found: `#{result.docidentifier[0].id}`"
+      result
+    end
+
+    extend Bibliography
+  end
+end

data/lib/relaton_etsi/config.rb

@@ -0,0 +1,10 @@
+module RelatonEtsi
+  module Config
+    include RelatonBib::Config
+  end
+  extend Config
+
+  class Configuration < RelatonBib::Configuration
+    PROGNAME = "relaton-etsi".freeze
+  end
+end

data/lib/relaton_etsi/data_fetcher.rb

@@ -0,0 +1,56 @@
+module RelatonEtsi
+  class DataFetcher
+    #
+    # Initialize data fetcher.
+    #
+    # @param [String] output output directory
+    # @param [String] format output format (xml, bibxml, yaml)
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def index1
+      @index1 ||= Relaton::Index.find_or_create :etsi, file: Bibliography::INDEX_FILE
+    end
+
+    def fetch
+      url = "https://www.etsi.org/?option=com_standardssearch&view=data&format=csv&includeScope=1&page=1&search=&" \
+            "title=1&etsiNumber=1&content=1&version=0&onApproval=1&published=1&withdrawn=1&historical=1&isCurrent=1&" \
+            "superseded=1&startDate=1988-01-15&endDate=2023-10-31&harmonized=0&keyword=&TB=&stdType=&frequency=&" \
+            "mandate=&collection=&sort=1&x=1698720135146"
+      csv = OpenURI.open_uri(url) { |f| f.readlines.join }
+      CSV.parse(csv, headers: true, col_sep: ';', skip_lines: /sep=;/).each do |row|
+        save DataParser.new(row).parse
+      end
+      index1.save
+    end
+
+    def save(bib)
+      filename = bib.docidentifier.first.id.gsub(/\//, "-").gsub(/\s|\./, "_").gsub(/\(|\)/, "")
+      file = File.join @output, "#{filename}.#{@ext}"
+      File.write file, content(bib), encoding: "UTF-8"
+      index1.add_or_update bib.docidentifier.first.id, file
+    end
+
+    def content(bib)
+      case @format
+      when "xml" then bib.to_xml bibdata: true
+      when "yaml" then bib.to_hash.to_yaml
+      else bib.send "to_#{@format}"
+      end
+    end
+  end
+end

data/lib/relaton_etsi/data_parser.rb

@@ -0,0 +1,88 @@
+module RelatonEtsi
+  class DataParser
+    ATTRS = %i[id title docnumber link date docid version status keyword editorialgroup
+               doctype abstract language script].freeze
+
+    def initialize(row)
+      @row = row
+    end
+
+    def parse
+      args = ATTRS.each_with_object({}) do |attr, hash|
+        hash[attr] = send(attr)
+      end
+      BibliographicItem.new(**args)
+    end
+
+    def pubid
+      @pubid ||= PubId.parse(@row["ETSI deliverable"])
+    end
+
+    def id
+      @row["ETSI deliverable"].gsub(/[\s\(\)]/, "")
+    end
+
+    def title
+      [RelatonBib::TypedTitleString.new(content: @row["title"], language: "en", script: "Latn")]
+    end
+
+    def docnumber
+      @row["ETSI deliverable"]
+    end
+
+    def link
+      urls = []
+      urls << RelatonBib::TypedUri.new(content: @row["Details link"], type: "src")
+      urls << RelatonBib::TypedUri.new(content: @row["PDF link"], type: "pdf")
+    end
+
+    def date
+      return [] unless pubid.date
+
+      [RelatonBib::BibliographicDate.new(type: "published", on: pubid.date)]
+    end
+
+    def docid
+      [RelatonBib::DocumentIdentifier.new(id: @row["ETSI deliverable"], type: "ETSI", primary: true)]
+    end
+
+    def version
+      return [] unless pubid.version
+
+      [RelatonBib::BibliographicItem::Version.new(nil, pubid.version)]
+    end
+
+    def status
+      status = @row["Status"] == "On Approval" ? "#{pubid.type} approval" : @row["Status"]
+      RelatonBib::DocumentStatus.new(stage: status)
+    end
+
+    def keyword
+      @row["Keywords"].split(",")
+    end
+
+    def editorialgroup
+      wg = RelatonBib::WorkGroup.new name: @row["Technical body"]
+      tc = RelatonBib::TechnicalCommittee.new wg
+      RelatonBib::EditorialGroup.new [tc]
+    end
+
+    def doctype
+      DocumentType.create_from_abbreviation pubid.type
+    end
+
+    def abstract
+      return [] unless @row["Scope"]
+
+      [RelatonBib::FormattedString.new(content: @row["Scope"], language: "en", script: "Latn")]
+    end
+
+    def language
+      ["en"]
+    end
+
+    def script
+      ["Latn"]
+    end
+  end
+end

data/lib/relaton_etsi/document_type.rb

@@ -0,0 +1,46 @@
+module RelatonEtsi
+  class DocumentType < RelatonBib::DocumentType
+    DOCTYPES = {
+      "EN" => "European Standard",
+      "ES" => "ETSI Standard",
+      "EG" => "ETSI Guide",
+      "TS" => "Technical Specification",
+      "GS" => "Group Specification",
+      "GR" => "Group Report",
+      "TR" => "Technical Report",
+      "ETR" => "ETSI Technical Report",
+      "GTS" => "GSM Technical Specification",
+      "SR" => "Special Report",
+      "TCRTR" => "Technical Committee Reference Technical Report",
+      "TBR" => "Technical Basis for Regulation",
+      "ETS" => "European Telecommunication Standard",
+      "I-ETS" => "Interim European Telecommunication Standard",
+      "NET" => "Norme Européenne de Télécommunication",
+    }.freeze
+
+    def initialize(type:, abbreviation: nil)
+      check_type type
+      check_abbreviation abbreviation
+      abbreviation ||= DOCTYPES.key(type)
+      super
+    end
+
+    def self.create_from_abbreviation(abbreviation)
+      new type: DOCTYPES[abbreviation], abbreviation: abbreviation
+    end
+
+    private
+
+    def check_type(type)
+      return if DOCTYPES.value? type
+
+      Util.warn "WARNING: invalid doctype: `#{type}`"
+    end
+
+    def check_abbreviation(abbreviation)
+      return if abbreviation.nil? || DOCTYPES.key?(abbreviation)
+
+      Util.warn "WARNING: invalid doctype abbreviation: `#{abbreviation}`"
+    end
+  end
+end

data/lib/relaton_etsi/hash_converter.rb

@@ -0,0 +1,14 @@
+module RelatonEtsi
+  module HashConverter
+    include RelatonBib::HashConverter
+    extend self
+
+    private
+
+    # @param item_hash [Hash]
+    # @return [RelatonEtsi::BibliographicItem]
+    def bib_item(item_hash)
+      BibliographicItem.new(**item_hash)
+    end
+  end
+end

data/lib/relaton_etsi/processor.rb

@@ -0,0 +1,61 @@
+require "relaton/processor"
+
+module RelatonEtsi
+  class Processor < Relaton::Processor
+    attr_reader :idtype
+
+    def initialize # rubocop:disable Lint/MissingSuper
+      @short = :relaton_etsi
+      @prefix = "ETSI"
+      @defaultprefix = %r{^ETSI\s}
+      @idtype = "ETSI"
+      @datasets = %w[etsi-csv]
+    end
+
+    # @param code [String]
+    # @param date [String, nil] year
+    # @param opts [Hash]
+    # @return [RelatonEtsi::BibliographicItem]
+    def get(code, date, opts)
+      ::RelatonEtsi::Bibliography.get(code, date, opts)
+    end
+
+    #
+    # Fetch all the documents from http://xml2rfc.tools.ietf.org/public/rfc/bibxml-3gpp-new/
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
+    # @param xml [String]
+    # @return [RelatonEtsi::BibliographicItem]
+    def from_xml(xml)
+      ::RelatonEtsi::XMLParser.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [RelatonEtsi::BibliographicItem]
+    def hash_to_bib(hash)
+      item_hash = ::RelatonEtsi::HashConverter.hash_to_bib(hash)
+      ::RelatonEtsi::BibliographicItem.new(**item_hash)
+    end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def grammar_hash
+      @grammar_hash ||= ::RelatonEtsi.grammar_hash
+    end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:etsi, url: true, file: Bibliography::INDEX_FILE).remove_file
+    end
+  end
+end

data/lib/relaton_etsi/pubid.rb

@@ -0,0 +1,35 @@
+module RelatonEtsi
+  class PubId
+    class Parser
+      def initialize(id)
+        @strscan = StringScanner.new id
+      end
+
+      def parse
+        @strscan.scan(/^ETSI\s+/)
+        type = @strscan.scan(/\S+/)
+        @strscan.scan(/\s+/)
+        docnumber = @strscan.scan_until(/(?=\s(V\d+\.\d+\.\d+)|ed\.\d+)/)
+        version = @strscan.scan(/\d+\.\d+\.\d+/) if @strscan.scan(/\sV(?=\d+\.\d+\.\d+)/)
+        edition = @strscan.scan(/\d+/) if @strscan.scan(/\sed\.(?=\d+)/)
+        date = @strscan.scan(/\d{4}-\d{2}/) if @strscan.scan(/\s\(/)
+
+        { type: type, docnumber: docnumber, version: version, edition: edition, date: date }
+      end
+    end
+
+    attr_accessor :type, :docnumber, :version, :edition, :date
+
+    def initialize(type:, docnumber:, version:, edition:, date:)
+      @type = type
+      @docnumber = docnumber
+      @version = version
+      @edition = edition
+      @date = date
+    end
+
+    def self.parse(id)
+      new(**Parser.new(id).parse)
+    end
+  end
+end

data/lib/relaton_etsi/util.rb

@@ -0,0 +1,9 @@
+module RelatonEtsi
+  module Util
+    extend RelatonBib::Util
+
+    def self.logger
+      RelatonEtsi.configuration.logger
+    end
+  end
+end

data/lib/relaton_etsi/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RelatonEtsi
+  VERSION = "1.17.0"
+end

data/lib/relaton_etsi/xml_parser.rb

@@ -0,0 +1,44 @@
+module RelatonEtsi
+  class XMLParser < RelatonBib::XMLParser
+    class << self
+      private
+
+      # @param intem [Nokogiri::XML::Document]
+      # @return [Hash]
+      def item_data(item) # rubocop:disable Metrics/AbcSize
+        data = super
+        ext = item.at "./ext"
+        return data unless ext
+
+        data[:marker] = ext.at("./marker")&.text
+        data[:frequency] = ext.xpath("./frequency").map(&:text)
+        data[:mandate] = ext.xpath("./mandate").map(&:text)
+        data[:custom_collection] = ext.at("./custom-collection")&.text
+        data
+      end
+
+      # @param item_hash [Hash]
+      # @return [RelatonEtsi::BibliographicItem]
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+
+      # def fetch_status(item)
+      #   status = item.at "./status"
+      #   return unless status
+
+      #   DocumentStatus.new(
+      #     stage: status.at("stage")&.text,
+      #     substage: status.at("substage")&.text,
+      #     iteration: status.at("iteration")&.text,
+      #   )
+      # end
+
+      # # @param item [Nokogiri::XML::Element]
+      # # @return [Array<RelatonBib::DocumentRelation>]
+      # def fetch_relations(item)
+      #   super item, DocumentRelation
+      # end
+    end
+  end
+end

data/lib/relaton_etsi.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "open-uri"
+require "csv"
+require "relaton/index"
+require "relaton_bib"
+require_relative "relaton_etsi/version"
+require_relative "relaton_etsi/config"
+require_relative "relaton_etsi/util"
+require_relative "relaton_etsi/pubid"
+require_relative "relaton_etsi/document_type"
+require_relative "relaton_etsi/bibliographic_item"
+require_relative "relaton_etsi/xml_parser"
+require_relative "relaton_etsi/hash_converter"
+require_relative "relaton_etsi/bibliography"
+require_relative "relaton_etsi/data_fetcher"
+require_relative "relaton_etsi/data_parser"
+
+module RelatonEtsi
+  class Error < StandardError; end
+
+  # Returns hash of gem versions used to generate data model.
+  # @return [String]
+  def grammar_hash
+    Digest::MD5.hexdigest RelatonEtsi::VERSION + RelatonBib::VERSION
+  end
+
+  extend self
+end

data/sig/relaton_etsi.rbs

@@ -0,0 +1,4 @@
+module RelatonEtsi
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end