relaton-3gpp 1.20.0 → 2.0.0.pre.alpha.1

data/grammars/relaton-3gpp.rng

@@ -21,7 +21,9 @@
       <optional>
         <ref name="docsubtype"/>
       </optional>
-      <ref name="editorialgroup"/>
+      <optional>
+        <ref name="editorialgroup"/>
+      </optional>
       <zeroOrMore>
         <ref name="ics"/>
       </zeroOrMore>

data/lib/relaton/3gpp/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module ThreeGpp
+    # This class represents a bibliographic item as a bibdata.
+    class Bibdata < Item
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/3gpp/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module ThreeGpp
+    # This class represents a bibliographic item as a bibitem.
+    class Bibitem < Item
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/3gpp/bibliography.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Relaton
+  module ThreeGpp
+    # Methods for search IANA standards.
+    module Bibliography
+      # SOURCE = "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-3gpp-new/"
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-3gpp/refs/heads/data-v2/"
+      INDEX_FILE = "index-v1.yaml"
+
+      # @param text [String]
+      # @return [RelatonBib::BibliographicItem]
+      def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        index = Relaton::Index.find_or_create "3GPP", url: "#{SOURCE}index-v1.zip", file: INDEX_FILE
+        row = index.search(text.sub(/^3GPP\s/, "")).min_by { |r| r[:id] }
+        return unless row
+
+        # file = text.sub(/^3GPP\s/, "").gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+        url = "#{SOURCE}#{row[:file]}"
+        resp = Net::HTTP.get_response URI(url)
+        return unless resp.code == "200"
+
+        item = Item.from_yaml(resp.body)
+        item.fetched = Date.today.to_s
+        item
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+            EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+            Net::ProtocolError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # @param ref [String] the W3C standard Code to look up
+      # @param year [String, NilClass] not used
+      # @param opts [Hash] options
+      # @return [RelatonBib::BibliographicItem]
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(ref)
+        unless result
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        result
+      end
+
+      extend Bibliography
+    end
+  end
+end

data/lib/relaton/3gpp/data_fetcher.rb

@@ -0,0 +1,225 @@
+require "fileutils"
+require "net/ftp"
+require_relative "../3gpp"
+require_relative "parser"
+
+module Relaton
+  module ThreeGpp
+    class DataFetcher < Core::DataFetcher
+      CURRENT = "current.yaml".freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create "3gpp", file: Bibliography::INDEX_FILE
+      end
+
+      #
+      # Parse documents
+      #
+      # @param [String] source source of documents, status-smg-3gpp for updare or status-smg-3gpp-force for renewal
+      #
+      def fetch(source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        renewal = source == "status-smg-3GPP-force"
+        file = get_file renewal
+        return unless file && File.exist?(file) && File.size(file) > 20_000_000
+
+        if renewal
+          FileUtils.rm_f Dir.glob(File.join(@output, "/*")) # if renewal && dbs["2001-04-25_schedule"].any?
+          index.remove_all # if renewal
+        end
+        CSV.open(file, "r:bom|utf-8", headers: true).each do |row|
+          save_doc Parser.parse(row)
+        end
+        File.write CURRENT, @current.to_yaml, encoding: "UTF-8"
+        index.save
+      end
+
+      #
+      # Get file from FTP. If file does not exist or changed, return nil
+      #
+      # @param [Boolean] renewal force to update all documents
+      #
+      # @return [String, nil] file name
+      #
+      def get_file(renewal) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        @current = YAML.load_file CURRENT if File.exist? CURRENT
+        @current ||= {}
+        n = 0
+        begin
+          ftp = Net::FTP.new("www.3gpp.org")
+          ftp.resume = true
+          ftp.login
+          ftp.chdir "/Information/Databases/"
+          file_path = ftp.list("*.csv").first
+          return unless file_path
+
+          d, t, _, file = file_path.split
+          dt = DateTime.strptime("#{d} #{t}", "%m-%d-%y %I:%M%p")
+          if !renewal && file == @current["file"] && !@current["date"].empty? && dt == DateTime.parse(@current["date"])
+            return
+          end
+
+          tmp_file = File.join Dir.tmpdir, "3gpp.csv"
+          ftp.get(file, tmp_file)
+        rescue Net::ReadTimeout => e
+          n += 1
+          retry if n < 5
+          raise e
+        end
+        @current["file"] = file
+        @current["date"] = dt.to_s
+        tmp_file
+      end
+
+      #
+      # Save document to file
+      #
+      # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+      #
+      def save_doc(bib) # rubocop:disable Metrics/MethodLength
+        return unless bib
+
+        bib1 = bib
+        file = output_file(bib1.docnumber)
+        if @files.include? file
+          bib1 = merge_duplication bib1, file
+          Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if bib1.nil?
+        else
+          @files << file
+          index.add_or_update bib1.docnumber, file
+        end
+        File.write file, serialize(bib1), encoding: "UTF-8" unless bib1.nil?
+      end
+
+      #
+      # Merge duplication
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [String] file file name of existing bibliographic item
+      #
+      # @return [Relaton3gpp::BibliographicItem, nil] merged bibliographic item or nil if no merge has been done
+      #
+      def merge_duplication(bib, file)
+        hash = YAML.load_file file
+        existed = Item.from_hash hash
+        changed = update_source bib, existed
+        bib1, bib2, chng = transposed_relation bib, existed
+        changed ||= chng
+        chng = add_contributor(bib1, bib2)
+        changed ||= chng
+        bib1 if changed
+      end
+
+      #
+      # Update link in case one of bibliographic items has no link
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib1
+      # @param [Relaton3gpp::BibliographicItem] bib2
+      #
+      # @return [Boolean] true if link has been updated
+      #
+      def update_source(bib1, bib2)
+        if bib1.source.any? && bib2.source.empty?
+          bib2.source = bib1.source
+          true
+        elsif bib1.source.empty? && bib2.source.any?
+          bib1.source = bib2.source
+          true
+        else false
+        end
+      end
+
+      #
+      # If one of bibliographic items has date gereater than anotherm=, make it relation
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
+      #
+      # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
+      #   related bibliographic item, true if relation has been added
+      #
+      def transposed_relation(bib, existed) # rubocop:disable Metrics/CyclomaticComplexity
+        return [bib, existed, false] if bib.date.none? && existed.date.none? ||
+          bib.date.any? && existed.date.none?
+        return [existed, bib, true] if bib.date.none? && existed.date.any?
+
+        check_transposed_date bib, existed
+      end
+
+      #
+      # Check if date of one bibliographic item is transposed to another
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
+      #
+      # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
+      #   related bibliographic item, true if relation has been added
+      #
+      def check_transposed_date(bib, existed)
+        if bib.date[0].at < existed.date[0].at
+          add_transposed_relation bib, existed
+          [bib, existed, true]
+        elsif bib.date[0].at > existed.date[0].at
+          add_transposed_relation existed, bib
+          [existed, bib, true]
+        else [bib, existed, false]
+        end
+      end
+
+      #
+      # Add transposed relation
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib1 the main bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] bib2 the transposed bibliographic item
+      #
+      # @return [Relaton3gpp::BibliographicItem]
+      #
+      def add_transposed_relation(bib1, bib2)
+        bib2.relation.each { |r| bib1.relation << r }
+        bib2.instance_variable_set :@relation, []
+        desc = Bib::LocalizedMarkedUpString.new content: "equivalent"
+        rel = Bib::Relation.new(type: "adoptedAs", bibitem: bib2, description: desc)
+        bib1.relation << rel
+      end
+
+      def add_contributor(bib1, bib2) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        changed = false
+
+        bib2.contributor.each do |bc|
+          next unless bc.person
+
+          existed = bib1.contributor.find { |ic| ic.person&.name == bc.person.name }
+          if existed
+            chng = add_affiliation existed, bc.person.affiliation
+            changed ||= chng
+          else
+            bib1.contributor << bc
+            changed = true
+          end
+        end
+
+        changed
+      end
+
+      def add_affiliation(contrib, affiliation)
+        changed = false
+
+        affiliation.each do |a|
+          unless contrib.person.affiliation.include? a
+            contrib.person.affiliation << a
+            changed = true
+          end
+        end
+
+        changed
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml(bib)
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml(bib)
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module ThreeGpp
+    class Doctype < Relaton::Bib::Doctype
+      TYPES = %w[TR TS].freeze
+
+      attribute :content, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/3gpp/ext.rb

@@ -0,0 +1,34 @@
+require_relative "doctype"
+require_relative "release"
+
+module Relaton
+  module ThreeGpp
+    class Ext < Lutaml::Model::Serializable
+      attribute :schema_version, method: :get_schema_version
+      attribute :doctype, Doctype
+      attribute :subdoctype, :string, values: %w[spec release]
+      attribute :editorialgroup, Relaton::Bib::EditorialGroup
+      attribute :ics, Relaton::Bib::ICS, collection: true
+      attribute :radiotechnology, :string, values: %w[2G 3G LTE 5G]
+      attribute :common_ims_spec, :boolean
+      attribute :internal, :boolean
+      attribute :release, Release
+
+      def get_schema_version
+        Relaton.schema_versions["relaton-model-3gpp"]
+      end
+
+      xml do
+        map_attribute "schema-version", to: :schema_version
+        map_element "doctype", to: :doctype
+        map_element "subdoctype", to: :subdoctype
+        map_element "editorialgroup", to: :editorialgroup
+        map_element "ics", to: :ics
+        map_element "radiotechnology", to: :radiotechnology
+        map_element "common-ims-spec", to: :common_ims_spec
+        map_element "internal", to: :internal
+        map_element "release", to: :release
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/item.rb

@@ -0,0 +1,12 @@
+require_relative "item_data"
+require_relative "ext"
+
+module Relaton
+  module ThreeGpp
+    class Item < Relaton::Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/3gpp/item_data.rb

@@ -0,0 +1,23 @@
+module Relaton
+  module ThreeGpp
+    class ItemData < Relaton::Bib::ItemData
+      def to_xml(bibdata: false, **opts)
+        add_notes opts[:note] do
+          bibdata ? Bibdata.to_xml(self) : Bibitem.to_xml(self)
+        end
+      end
+
+      def to_yaml(**opts)
+        add_notes opts[:note] do
+          Item.to_yaml(self)
+        end
+      end
+
+      def to_json(**opts)
+        add_notes opts[:note] do
+          Item.to_json(self)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/parser.rb

@@ -0,0 +1,304 @@
+require "csv"
+
+module Relaton
+  module ThreeGpp
+    class Parser
+      # DOCTYPES = { "TS" => "Technical Specification", "TR" => "Technical Report" }.freeze
+
+      #
+      # Document parser initalization
+      #
+      # @param [CSV::Row] row CSV row
+      #
+      def initialize(row)
+        @row = row
+      end
+
+      #
+      # Initialize document parser and run it
+      #
+      # @param [CSV:Row] row CSV row
+      #
+      # @return [RelatonBib:BibliographicItem, nil] bibliographic item
+      #
+      def self.parse(row)
+        new(row).parse
+      end
+
+      #
+      # Parse document
+      #
+      # @return [Relaton3gpp:BibliographicItem, nil] bibliographic item
+      #
+      def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        ItemData.new(
+          id: parse_id,
+          type: "standard",
+          language: ["en"],
+          script: ["Latn"],
+          title: parse_title,
+          source: parse_source,
+          # abstract: parse_abstract,
+          docidentifier: parse_docid,
+          docnumber: number,
+          date: parse_date,
+          version: parse_version,
+          contributor: parse_contributor,
+          place: parse_place,
+          ext: parse_ext,
+          # biblionote: parse_note,
+          # docstatus: parse_status,
+          # common_ims_spec: @spec[:ComIMS] == "1",
+          # internal: @spec[:"For publication"] == "0",
+        )
+      end
+
+      def parse_id
+        number.gsub(/[\s:\/]/, "-").squeeze("-").upcase
+      end
+
+      #
+      # Parse title
+      #
+      # @return [RelatonBib::TypedTitleStringCollection] title
+      #
+      def parse_title
+        [Bib::Title.new(content: @row["Title"], type: "main")]
+        # RelatonBib::TypedTitleStringCollection.new [t]
+      end
+
+      #
+      # Parse link
+      #
+      # @return [Array<RelatonBib::TypedUri>] link
+      #
+      def parse_source
+        return [] unless @row["Link"]
+
+        [Bib::Uri.new(type: "src", content: @row["Link"])]
+      end
+
+      #
+      # Parse abstract
+      #
+      # @return [Array<RelatonBib::FormattedString>]
+      #
+      # def parse_abstract
+      #   return [] unless @spec[:description]
+
+      #   [RelatonBib::FormattedString.new(content: @spec[:description])]
+      # end
+
+      #
+      # Parse docidentifier
+      #
+      # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
+      #
+      def parse_docid
+        [Bib::Docidentifier.new(type: "3GPP", content: "3GPP #{number}", primary: true)]
+      end
+
+      #
+      # Generate number
+      #
+      # @return [String] number
+      #
+      def number
+        num = "#{doctype_abbr} #{@row[0]}"
+        num += ":#{release}" if release
+        "#{num}/#{version}"
+      end
+
+      def version
+        @row["Version"]
+      end
+
+      def parse_version
+        [Bib::Version.new(draft: version)]
+      end
+
+      def doctype_abbr
+        @row["Is TS"] == "1" ? "TS" : "TR"
+      end
+
+      def release
+        @release ||=  case @row["WPM Code 2G"]
+                      when /Release_(\d+)/ then "REL-#{$1}"
+                      when /PH(\d+)/ then "Ph#{$1}"
+                      else @row["Release"]
+                      end
+      end
+
+      #
+      # Version
+      #
+      # @return [String] version
+      #
+      # def version
+      #   "#{@row[:MAJOR_VERSION_NB]}.#{@row[:TECHNICAL_VERSION_NB]}.#{@row[:EDITORIAL_VERSION_NB]}"
+      # end
+
+      #
+      # Parse date
+      #
+      # @return [Array<RelatonBib::BibliographicDate>] date
+      #
+      def parse_date
+        dates = []
+        if @row["Date"]
+          on = Date.parse(@row["Date"]).to_s
+          dates << Bib::Date.new(type: "published", at: on)
+        end
+        dates
+      end
+
+      #
+      # Parse note
+      #
+      # @return [RelatonBib::BiblioNoteCollection] notes
+      #
+      # def parse_note
+      #   n = []
+      #   if @specrel && @specrel[:remarks] && @specrel[:remarks] != "."
+      #     n << RelatonBib::BiblioNote.new(type: "remark", content: @specrel[:remarks])
+      #   end
+      #   if @row[:comment] && @row[:comment] != "."
+      #     n << RelatonBib::BiblioNote.new(type: "comment", content: @row[:comment])
+      #   end
+      #   RelatonBib::BiblioNoteCollection.new n
+      # end
+
+      #
+      # Prase status
+      #
+      # @return [RelatnoBib::DocumentStatus, nil] status
+      #
+      # def parse_status
+      #   if @specrel && @specrel[:withdrawn] == "1"
+      #     RelatonBib::DocumentStatus.new stage: "withdrawn"
+      #   elsif @spec[:"For publication"] == "1"
+      #     RelatonBib::DocumentStatus.new stage: "published"
+      #   end
+      # end
+
+      #
+      # Create contributors
+      #
+      # @return [Array<RelatonBib::ContributionInfo>] contributor
+      #
+      def parse_contributor # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        name = Bib::TypedLocalizedString.new(content: "3rd Generation Partnership Project")
+        abbrev = Bib::LocalizedString.new content: "3GPP"
+        org = Bib::Organization.new(name: [name], abbreviation: abbrev, address: [address])
+        contribs = [Bib::Contributor.new(organization: org, role: contributor_role)]
+        return contribs unless @row["Last Name"] && @row["Last Name"] != "Vacant"
+
+        role = Bib::Contributor::Role.new type: "author"
+        contribs << Bib::Contributor.new(person: person, role: [role])
+      end
+
+      def address
+        Bib::Address.new(
+          street: ["c/o ETSI 650, route des Lucioles", "3GPP Mobile Competence Centre"],
+          postcode: "06921", city: "Sophia Antipolis Cedex", country: "France"
+        )
+      end
+
+      def contributor_role
+        [Bib::Contributor::Role.new(type: "author"), Bib::Contributor::Role.new(type: "publisher")]
+      end
+
+      def person
+        surname = Bib::LocalizedString.new content: @row["Last Name"], language: "en", script: "Latn"
+        forename = Bib::FullNameType::Forename.new content: @row["First Name"], language: "en", script: "Latn"
+        name = Bib::FullName.new(surname: surname, forename: [forename])
+        Bib::Person.new(name: name, affiliation: affiliation)
+      end
+
+      def affiliation
+        return [] if @row["Organisation"].nil? || @row["Organisation"].empty?
+
+        name = Bib::TypedLocalizedString.new(content: @row["Organisation"])
+        org = Bib::Organization.new(name: [name])
+        [Bib::Affiliation.new(organization: org)]
+      end
+
+      def parse_place
+        [Bib::Place.new(formatted_place: "Sophia Antipolis Cedex, France")]
+      end
+
+      def parse_ext
+        Ext.new(
+          doctype: parse_doctype,
+          editorialgroup: parse_editorialgroup,
+          radiotechnology: parse_radiotechnology,
+          release: parse_release,
+        )
+      end
+
+      def parse_doctype
+        # type = DOCTYPES[doctype_abbr]
+        # Doctype.new(abbreviation: doctype_abbr, content: type)
+        Doctype.new(content: doctype_abbr) # 3GPP grammar alloves only TS and TR content
+      end
+
+      #
+      # Parse editorialgroup
+      #
+      # @return [RelatonBib::EditorialGroup] editorialgroups
+      #
+      def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        eg = []
+        prime = @row["Responsible Primary"]
+        eg << create_workgroup(prime, "prime") unless prime.nil? || prime.empty?
+
+        @row["Responsible Secondary"].strip.split(", ").each do |wg|
+          eg << create_workgroup(wg, "other")
+        end
+        Bib::EditorialGroup.new technical_committee: eg
+      end
+
+      def create_workgroup(name, type)
+        Bib::WorkGroup.new(content: name, type: type)
+      end
+
+      #
+      # Parse radio technology
+      #
+      # @return [String] radio technology
+      #
+      def parse_radiotechnology
+        case @row["WPM Code 3G"]
+        when /5G/ then "5G"
+        when /4G/ then "LTE"
+        when /3G/ then "3G"
+        else @row["WPM Code 2G"] && "2G"
+        end
+      end
+
+      #
+      # Parse release
+      #
+      # @return [Relaton3gpp::Release, nil] release
+      #
+      def parse_release # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        project_start = Date.parse(@row["Project Start"]) if @row["Project Start"]
+        project_end = Date.parse(@row["Project End"]) if @row["Project End"]
+        Release.new(
+          # version2g: @rel[:version_2g],
+          # version3g: @rel[:version_3g],
+          # defunct: @rel[:defunct] == "1",
+          wpm_code_2g: @row["WPM Code 2G"],
+          wpm_code_3g: @row["WPM Code 3G"],
+          # freeze_meeting: @rel[:"freeze meeting"],
+          freeze_stage1_meeting: @row["Stage 1 Freeze"],
+          freeze_stage2_meeting: @row["Stage 2 Freeze"],
+          freeze_stage3_meeting: @row["Stage 3 Freeze"],
+          close_meeting: @row["Close Meeting"],
+          project_start: project_start,
+          project_end: project_end,
+        )
+      end
+    end
+  end
+end