relaton-3gpp 1.20.0 → 2.0.0.pre.alpha.1

data/lib/relaton_3gpp/data_fetcher.rb

@@ -1,283 +0,0 @@
-module Relaton3gpp
-  class DataFetcher
-    CURRENT = "current.yaml".freeze
-    #
-    # Data fetcher initializer
-    #
-    # @param [String] output directory to save files
-    # @param [String] format format of output files (xml, yaml, bibxml)
-    #
-    def initialize(output, format)
-      require "fileutils"
-      require "net/ftp"
-      require "csv"
-
-      @output = output
-      @format = format
-      @ext = format.sub(/^bib/, "")
-      @files = []
-    end
-
-    def index
-      @index ||= Relaton::Index.find_or_create "3gpp", file: "index-v1.yaml"
-    end
-
-    #
-    # Initialize fetcher and run fetch
-    #
-    # @param [Strin] source source name
-    # @param [Strin] output directory to save files, default: "data"
-    # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
-    #
-    def self.fetch(source, output: "data", format: "yaml")
-      t1 = Time.now
-      puts "Started at: #{t1}"
-      FileUtils.mkdir_p output
-      new(output, format).fetch(source == "status-smg-3GPP-force")
-      t2 = Time.now
-      puts "Stopped at: #{t2}"
-      puts "Done in: #{(t2 - t1).round} sec."
-    end
-
-    #
-    # Parse documents
-    #
-    # @param [Boolean] renewal force to update all documents
-    #
-    def fetch(renewal) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      file = get_file renewal
-      return unless file && File.exist?(file) && File.size(file) > 20_000_000
-
-      if renewal
-        FileUtils.rm_f File.join(@output, "/*") # if renewal && dbs["2001-04-25_schedule"].any?
-        index.remove_all # if renewal
-      end
-      CSV.open(file, "r:bom|utf-8", headers: true).each do |row|
-        save_doc Parser.parse(row)
-      end
-      File.write CURRENT, @current.to_yaml, encoding: "UTF-8"
-      index.save
-    end
-
-    #
-    # Get file from FTP. If file does not exist or changed, return nil
-    #
-    # @param [Boolean] renewal force to update all documents
-    #
-    # @return [String, nil] file name
-    #
-    def get_file(renewal) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      @current = YAML.load_file CURRENT if File.exist? CURRENT
-      @current ||= {}
-      n = 0
-      begin
-        ftp = Net::FTP.new("www.3gpp.org")
-        ftp.resume = true
-        ftp.login
-        ftp.chdir "/Information/Databases/"
-        file_path = ftp.list("*.csv").first
-        return unless file_path
-
-        d, t, _, file = file_path.split
-        dt = DateTime.strptime("#{d} #{t}", "%m-%d-%y %I:%M%p")
-        if !renewal && file == @current["file"] && !@current["date"].empty? && dt == DateTime.parse(@current["date"])
-          return
-        end
-
-        tmp_file = File.join Dir.tmpdir, "3gpp.csv"
-        ftp.get(file, tmp_file)
-      rescue Net::ReadTimeout => e
-        n += 1
-        retry if n < 5
-        raise e
-      end
-      @current["file"] = file
-      @current["date"] = dt.to_s
-      tmp_file
-    end
-
-    #
-    # Fetch document
-    #
-    # @param [Hash] row row from mdb
-    # @param [Array<Hash>] specs specs
-    # @param [Array<Hash>] specrels specrels
-    # @param [Array<Hash>] releases releases
-    # @param [Array<Hash>] tstatus tstatus
-    #
-    # @return [Relaton3gpp::BibliographicItem, nil] bibliographic item
-    #
-    # def fetch_doc(row, specs, specrels, releases, tstatus)
-    #   doc = Parser.parse row, specs, specrels, releases, tstatus
-    #   save_doc doc
-    # rescue StandardError => e
-    #   warn "Error: #{e.message}"
-    #   warn "PubID: #{row[:spec]}:#{row[:release]}/#{row[:MAJOR_VERSION_NB]}."\
-    #        "#{row[:TECHNICAL_VERSION_NB]}.#{row[:EDITORIAL_VERSION_NB]}"
-    #   warn e.backtrace[0..5].join("\n")
-    # end
-
-    #
-    # Save document to file
-    #
-    # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
-    #
-    def save_doc(bib) # rubocop:disable Metrics/MethodLength
-      return unless bib
-
-      bib1 = bib
-      file = file_name(bib1)
-      if @files.include? file
-        bib1 = merge_duplication bib1, file
-        Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if bib1.nil?
-      else
-        @files << file
-        index.add_or_update bib1.docnumber, file
-      end
-      File.write file, serialise(bib1), encoding: "UTF-8" unless bib1.nil?
-    end
-
-    #
-    # Merge duplication
-    #
-    # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
-    # @param [String] file file name of existing bibliographic item
-    #
-    # @return [Relaton3gpp::BibliographicItem, nil] merged bibliographic item or nil if no merge has been done
-    #
-    def merge_duplication(bib, file)
-      hash = YAML.load_file file
-      existed = BibliographicItem.from_hash hash
-      changed = update_link bib, existed
-      bib1, bib2, chng = transposed_relation bib, existed
-      changed ||= chng
-      chng = add_contributor(bib1, bib2)
-      changed ||= chng
-      bib1 if changed
-    end
-
-    #
-    # Update link in case one of bibliographic items has no link
-    #
-    # @param [Relaton3gpp::BibliographicItem] bib1
-    # @param [Relaton3gpp::BibliographicItem] bib2
-    #
-    # @return [Boolean] true if link has been updated
-    #
-    def update_link(bib1, bib2)
-      if bib1.link.any? && bib2.link.empty?
-        bib2.instance_variable_set(:@link, bib1.link)
-        true
-      elsif bib1.link.empty? && bib2.link.any?
-        bib1.instance_variable_set(:@link, bib2.link)
-        true
-      else false
-      end
-    end
-
-    #
-    # If one of bibliographic items has date gereater than anotherm=, make it relation
-    #
-    # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
-    # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
-    #
-    # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
-    #   related bibliographic item, true if relation has been added
-    #
-    def transposed_relation(bib, existed) # rubocop:disable Metrics/CyclomaticComplexity
-      return [bib, existed, false] if bib.date.none? && existed.date.none? ||
-        bib.date.any? && existed.date.none?
-      return [existed, bib, true] if bib.date.none? && existed.date.any?
-
-      check_transposed_date bib, existed
-    end
-
-    #
-    # Check if date of one bibliographic item is transposed to another
-    #
-    # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
-    # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
-    #
-    # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
-    #   related bibliographic item, true if relation has been added
-    #
-    def check_transposed_date(bib, existed)
-      if bib.date[0].on < existed.date[0].on
-        add_transposed_relation bib, existed
-        [bib, existed, true]
-      elsif bib.date[0].on > existed.date[0].on
-        add_transposed_relation existed, bib
-        [existed, bib, true]
-      else [bib, existed, false]
-      end
-    end
-
-    #
-    # Add transposed relation
-    #
-    # @param [Relaton3gpp::BibliographicItem] bib1 the main bibliographic item
-    # @param [Relaton3gpp::BibliographicItem] bib2 the transposed bibliographic item
-    #
-    # @return [Relaton3gpp::BibliographicItem]
-    #
-    def add_transposed_relation(bib1, bib2)
-      bib2.relation.each { |r| bib1.relation << r }
-      bib2.instance_variable_set :@relation, RelatonBib::DocRelationCollection.new([])
-      dec = RelatonBib::FormattedString.new content: "equivalent"
-      rel = RelatonBib::DocumentRelation.new(type: "adoptedAs", bibitem: bib2, description: dec)
-      bib1.relation << rel
-    end
-
-    def add_contributor(bib1, bib2) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
-      changed = false
-
-      bib2.contributor.each do |bc|
-        next if bc.entity.is_a? RelatonBib::Organization
-
-        existed = bib1.contributor.find { |ic| ic.entity.name == bc.entity.name }
-        if existed
-          chng = add_affiliation existed, bc.entity.affiliation
-          changed ||= chng
-        else
-          bib1.contributor << bc
-          changed = true
-        end
-      end
-
-      changed
-    end
-
-    def add_affiliation(contrib, affiliation)
-      changed = false
-
-      affiliation.each do |a|
-        unless contrib.entity.affiliation.include? a
-          contrib.entity.affiliation << a
-          changed = true
-        end
-      end
-
-      changed
-    end
-
-    def serialise(bib)
-      case @format
-      when "xml" then bib.to_xml(bibdata: true)
-      when "yaml" then bib.to_hash.to_yaml
-      else bib.send("to_#{@format}")
-      end
-    end
-
-    #
-    # Generate file name
-    #
-    # @param [RelatonW3c::W3cBibliographicItem] bib bibliographic item
-    #
-    # @return [String] file name
-    #
-    def file_name(bib)
-      name = bib.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
-      File.join @output, "#{name}.#{@ext}"
-    end
-  end
-end

data/lib/relaton_3gpp/document_type.rb

@@ -1,18 +0,0 @@
-module Relaton3gpp
-  class DocumentType < RelatonBib::DocumentType
-    DOCTYPES = %w[TS TR].freeze
-
-    def initialize(type:, abbreviation: nil)
-      check type
-      super
-    end
-
-    # @param type [String]
-    def check(type)
-      unless DOCTYPES.include? type
-        Util.warn "Unknown doctype: `#{type}`"
-        Util.warn "Possible doctypes: `#{DOCTYPES.join '`, `'}`"
-      end
-    end
-  end
-end

data/lib/relaton_3gpp/hash_converter.rb

@@ -1,31 +0,0 @@
-module Relaton3gpp
-  module HashConverter
-    include RelatonBib::HashConverter
-    extend self
-
-    def hash_to_bib(args)
-      hash = super
-      hash[:radiotechnology] = hash[:ext][:radiotechnology] if hash.dig(:ext, :radiotechnology)
-      hash[:common_ims_spec] = hash[:ext][:"common-ims-spec"] if hash.dig(:ext, :"common-ims-spec")
-      release_hash_to_bib(hash)
-      hash
-    end
-
-    def release_hash_to_bib(hash)
-      release = hash.dig(:ext, :release) || hash[:release] # @TODO remove hash[:release] after release is moved to ext
-      return unless release
-
-      hash[:release] = Release.new(**release)
-    end
-
-    # @param item_hash [Hash]
-    # @return [Relaton3gpp::BibliographicItem]
-    def bib_item(item_hash)
-      BibliographicItem.new(**item_hash)
-    end
-
-    def create_doctype(**type)
-      DocumentType.new(**type)
-    end
-  end
-end

data/lib/relaton_3gpp/parser.rb

@@ -1,273 +0,0 @@
-module Relaton3gpp
-  class Parser
-    DOCTYPES = { "TS" => "Technical Specification", "TR" => "Technical Report"}.freeze
-
-    #
-    # Document parser initalization
-    #
-    # @param [CSV::Row] row CSV row
-    #
-    def initialize(row)
-      @row = row
-    end
-
-    #
-    # Initialize document parser and run it
-    #
-    # @param [CSV:Row] row CSV row
-    #
-    # @return [RelatonBib:BibliographicItem, nil] bibliographic item
-    #
-    def self.parse(row)
-      new(row).parse
-    end
-
-    #
-    # Parse document
-    #
-    # @return [Relaton3gpp:BibliographicItem, nil] bibliographic item
-    #
-    def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      Relaton3gpp::BibliographicItem.new(
-        type: "standard",
-        language: ["en"],
-        script: ["Latn"],
-        title: parse_title,
-        link: parse_link,
-        # abstract: parse_abstract,
-        docid: parse_docid,
-        docnumber: number,
-        date: parse_date,
-        doctype: parse_doctype,
-        editorialgroup: parse_editorialgroup,
-        version: parse_version,
-        # biblionote: parse_note,
-        # docstatus: parse_status,
-        radiotechnology: parse_radiotechnology,
-        # common_ims_spec: @spec[:ComIMS] == "1",
-        # internal: @spec[:"For publication"] == "0",
-        release: parse_release,
-        contributor: parse_contributor,
-        place: ["Sophia Antipolis Cedex, France"],
-      )
-    end
-
-    #
-    # Parse title
-    #
-    # @return [RelatonBib::TypedTitleStringCollection] title
-    #
-    def parse_title
-      t = RelatonBib::TypedTitleString.new content: @row["Title"], type: "main"
-      RelatonBib::TypedTitleStringCollection.new [t]
-    end
-
-    #
-    # Parse link
-    #
-    # @return [Array<RelatonBib::TypedUri>] link
-    #
-    def parse_link
-      return [] unless @row["Link"]
-
-      [RelatonBib::TypedUri.new(type: "src", content: @row["Link"])]
-    end
-
-    #
-    # Parse abstract
-    #
-    # @return [Array<RelatonBib::FormattedString>]
-    #
-    # def parse_abstract
-    #   return [] unless @spec[:description]
-
-    #   [RelatonBib::FormattedString.new(content: @spec[:description])]
-    # end
-
-    #
-    # Parse docidentifier
-    #
-    # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
-    #
-    def parse_docid
-      [RelatonBib::DocumentIdentifier.new(type: "3GPP", id: "3GPP #{number}", primary: true)]
-    end
-
-    #
-    # Generate number
-    #
-    # @return [String] number
-    #
-    def number
-      num = "#{doctype_abbr} #{@row[0]}"
-      num += ":#{release}" if release
-      "#{num}/#{version}"
-    end
-
-    def version
-      @row["Version"]
-    end
-
-    def parse_version
-      [RelatonBib::BibliographicItem::Version.new(nil, version)]
-    end
-
-    def doctype_abbr
-      @row["Is TS"] == "1" ? "TS" : "TR"
-    end
-
-    def release
-      @release ||=  case @row["WPM Code 2G"]
-                    when /Release_(\d+)/ then "REL-#{$1}"
-                    when /PH(\d+)/ then "Ph#{$1}"
-                    else @row["Release"]
-                    end
-    end
-
-    #
-    # Version
-    #
-    # @return [String] version
-    #
-    # def version
-    #   "#{@row[:MAJOR_VERSION_NB]}.#{@row[:TECHNICAL_VERSION_NB]}.#{@row[:EDITORIAL_VERSION_NB]}"
-    # end
-
-    #
-    # Parse date
-    #
-    # @return [Array<RelatonBib::BibliographicDate>] date
-    #
-    def parse_date
-      dates = []
-      if @row["Date"]
-        on = Date.parse(@row["Date"]).to_s
-        dates << RelatonBib::BibliographicDate.new(type: "published", on: on)
-      end
-      dates
-    end
-
-    def parse_doctype
-      # type = DOCTYPES[doctype_abbr]
-      DocumentType.new(type: doctype_abbr)
-    end
-
-    #
-    # Parse editorialgroup
-    #
-    # @return [RelatonBib::EditorialGroup] editorialgroups
-    #
-    def parse_editorialgroup # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      eg = []
-      prime = @row["Responsible Primary"]
-      eg << create_workgroup(prime, "prime") if prime
-
-      @row["Responsible Secondary"].strip.split(", ").each do |wg|
-        eg << create_workgroup(wg, "other")
-      end
-      RelatonBib::EditorialGroup.new eg
-    end
-
-    def create_workgroup(name, type)
-      wgf = RelatonBib::WorkGroup.new(name: name, type: type)
-      RelatonBib::TechnicalCommittee.new(wgf)
-    end
-
-    #
-    # Parse note
-    #
-    # @return [RelatonBib::BiblioNoteCollection] notes
-    #
-    # def parse_note
-    #   n = []
-    #   if @specrel && @specrel[:remarks] && @specrel[:remarks] != "."
-    #     n << RelatonBib::BiblioNote.new(type: "remark", content: @specrel[:remarks])
-    #   end
-    #   if @row[:comment] && @row[:comment] != "."
-    #     n << RelatonBib::BiblioNote.new(type: "comment", content: @row[:comment])
-    #   end
-    #   RelatonBib::BiblioNoteCollection.new n
-    # end
-
-    #
-    # Prase status
-    #
-    # @return [RelatnoBib::DocumentStatus, nil] status
-    #
-    # def parse_status
-    #   if @specrel && @specrel[:withdrawn] == "1"
-    #     RelatonBib::DocumentStatus.new stage: "withdrawn"
-    #   elsif @spec[:"For publication"] == "1"
-    #     RelatonBib::DocumentStatus.new stage: "published"
-    #   end
-    # end
-
-    #
-    # Parse radio technology
-    #
-    # @return [String] radio technology
-    #
-    def parse_radiotechnology
-      case @row["WPM Code 3G"]
-      when /5G/ then "5G"
-      when /4G/ then "LTE"
-      when /3G/ then "3G"
-      else @row["WPM Code 2G"] && "2G"
-      end
-    end
-
-    #
-    # Parse release
-    #
-    # @return [Relaton3gpp::Release, nil] release
-    #
-    def parse_release # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      project_start = Date.parse(@row["Project Start"]).to_s if @row["Project Start"]
-      project_end = Date.parse(@row["Project End"]).to_s if @row["Project End"]
-      Release.new(
-        # version2g: @rel[:version_2g],
-        # version3g: @rel[:version_3g],
-        # defunct: @rel[:defunct] == "1",
-        wpm_code_2g: @row["WPM Code 2G"],
-        wpm_code_3g: @row["WPM Code 3G"],
-        # freeze_meeting: @rel[:"freeze meeting"],
-        freeze_stage1_meeting: @row["Stage 1 Freeze"],
-        freeze_stage2_meeting: @row["Stage 2 Freeze"],
-        freeze_stage3_meeting: @row["Stage 3 Freeze"],
-        close_meeting: @row["Close Meeting"],
-        project_start: project_start,
-        project_end: project_end,
-      )
-    end
-
-    #
-    # Create contributors
-    #
-    # @return [Array<RelatonBib::ContributionInfo>] contributor
-    #
-    def parse_contributor # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      address = RelatonBib::Address.new(
-        street: ["c/o ETSI 650, route des Lucioles", "3GPP Mobile Competence Centre"],
-        postcode: "06921", city: "Sophia Antipolis Cedex", country: "France"
-      )
-      org = RelatonBib::Organization.new(
-        name: "3rd Generation Partnership Project", abbreviation: "3GPP", contact: [address],
-      )
-      roles = [{ type: "author" }, { type: "publisher" }]
-      contribs = [RelatonBib::ContributionInfo.new(entity: org, role: roles)]
-      return contribs unless @row["Last Name"] && @row["Last Name"] != "Vacant"
-
-      aff = []
-      if @row["Organisation"]
-        org = RelatonBib::Organization.new(name: @row["Organisation"])
-        aff << RelatonBib::Affiliation.new(organization: org)
-      end
-      surname = RelatonBib::LocalizedString.new @row["Last Name"], "en", "Latn"
-      forename = RelatonBib::Forename.new content: @row["First Name"], language: ["en"], script: ["Latn"]
-      name = RelatonBib::FullName.new(surname: surname, forename: [forename])
-      person = RelatonBib::Person.new(name: name, affiliation: aff)
-      role = { type: "author" }
-      contribs << RelatonBib::ContributionInfo.new(entity: person, role: [role])
-    end
-  end
-end

data/lib/relaton_3gpp/processor.rb

@@ -1,61 +0,0 @@
-require "relaton/processor"
-
-module Relaton3gpp
-  class Processor < Relaton::Processor
-    attr_reader :idtype
-
-    def initialize # rubocop:disable Lint/MissingSuper
-      @short = :relaton_3gpp
-      @prefix = "3GPP"
-      @defaultprefix = %r{^3GPP\s}
-      @idtype = "3GPP"
-      @datasets = %w[status-smg-3GPP status-smg-3GPP-force]
-    end
-
-    # @param code [String]
-    # @param date [String, NilClass] year
-    # @param opts [Hash]
-    # @return [RelatonBib::BibliographicItem]
-    def get(code, date, opts)
-      ::Relaton3gpp::Bibliography.get(code, date, opts)
-    end
-
-    #
-    # Fetch all the documents from http://xml2rfc.tools.ietf.org/public/rfc/bibxml-3gpp-new/
-    #
-    # @param [String] source source name
-    # @param [Hash] opts
-    # @option opts [String] :output directory to output documents
-    # @option opts [String] :format
-    #
-    def fetch_data(source, opts)
-      DataFetcher.fetch(source, **opts)
-    end
-
-    # @param xml [String]
-    # @return [RelatonBib::BibliographicItem]
-    def from_xml(xml)
-      ::Relaton3gpp::XMLParser.from_xml xml
-    end
-
-    # @param hash [Hash]
-    # @return [RelatonBib::BibliographicItem]
-    def hash_to_bib(hash)
-      item_hash = ::Relaton3gpp::HashConverter.hash_to_bib(hash)
-      ::Relaton3gpp::BibliographicItem.new(**item_hash)
-    end
-
-    # Returns hash of XML grammar
-    # @return [String]
-    def grammar_hash
-      @grammar_hash ||= ::Relaton3gpp.grammar_hash
-    end
-
-    #
-    # Remove index file
-    #
-    def remove_index_file
-      Relaton::Index.find_or_create("3GPP", url: true, file: Bibliography::INDEX_FILE).remove_file
-    end
-  end
-end