relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/3gpp/data_fetcher.rb

@@ -0,0 +1,303 @@
+require "fileutils"
+require "net/http"
+require "uri"
+require_relative "../3gpp"
+require_relative "parser"
+
+module Relaton
+  module ThreeGpp
+    class DataFetcher < Core::DataFetcher
+      CURRENT = "current.yaml".freeze
+      CSV_URL = "https://www.3gpp.org/ftp/Information/Databases/3GPPBibliography.csv".freeze
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      # The index is pubid-backed: `pubid_class:` makes Relaton::Index sort the
+      # rows by `id.root.number` on save and serialize each id to its
+      # `_type: pubid:3gpp:*` hash. Without it a v1-shaped file would be
+      # written under a v2 name, with no error and no narrowing — FileIO#save
+      # only calls `to_hash` when the value is an instance of `pubid_class`.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          "3gpp", file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Tgpp::Identifier
+        )
+      end
+
+      #
+      # Parse documents
+      #
+      # @param [String] source source of documents, status-smg-3gpp for updare
+      #   or status-smg-3gpp-force for renewal
+      #
+      def fetch(source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        renewal = source == "status-smg-3GPP-force"
+        file = get_file renewal
+        return unless file && File.exist?(file) && File.size(file) > 20_000_000
+
+        if renewal
+          FileUtils.rm_f Dir.glob(File.join(@output, "/*"))
+          index.remove_all # if renewal
+        end
+        CSV.open(file, "r:bom|utf-8", headers: true, col_sep: ";").each do |row|
+          save_doc Parser.parse(row, @errors)
+        end
+        File.write CURRENT, @current.to_yaml, encoding: "UTF-8"
+        index.save
+        report_errors
+      end
+
+      #
+      # Get file via HTTPS. If file has not changed, return nil
+      #
+      # @param [Boolean] renewal force to update all documents
+      #
+      # @return [String, nil] file name
+      #
+      def get_file(renewal) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        @current = YAML.load_file CURRENT if File.exist? CURRENT
+        @current ||= {}
+        uri = URI(CSV_URL)
+        n = 0
+        begin
+          last_modified = head_last_modified(uri)
+          return unless last_modified
+
+          dt = DateTime.parse(last_modified)
+          if !renewal && !@current["date"].to_s.empty? &&
+              dt == DateTime.parse(@current["date"])
+            return
+          end
+
+          tmp_file = File.join Dir.tmpdir, "3gpp.csv"
+          download(uri, tmp_file)
+        rescue Net::OpenTimeout, Net::ReadTimeout, SocketError => e
+          n += 1
+          retry if n < 5
+          raise e
+        end
+        @current["date"] = dt.to_s
+        tmp_file
+      end
+
+      def head_last_modified(uri)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                            open_timeout: 30, read_timeout: 30) do |http|
+          resp = http.request(Net::HTTP::Head.new(uri.request_uri))
+          raise "HTTP #{resp.code} from #{uri}" unless resp.is_a?(Net::HTTPSuccess)
+
+          resp["last-modified"]
+        end
+      end
+
+      def download(uri, tmp_file)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                            open_timeout: 30, read_timeout: 120) do |http|
+          http.request(Net::HTTP::Get.new(uri.request_uri)) do |resp|
+            raise "HTTP #{resp.code} from #{uri}" unless resp.is_a?(Net::HTTPSuccess)
+
+            File.open(tmp_file, "wb") { |f| resp.read_body { |chunk| f.write(chunk) } }
+          end
+        end
+      end
+
+      #
+      # Save document to file
+      #
+      # @param [RelatonW3c::W3cBibliographicItem, nil] bib bibliographic item
+      #
+      def save_doc(bib) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        return unless bib
+
+        bib1 = bib
+        # A DISTINCT docnumber that sanitizes to the same filename must not be
+        # merged into the other document — give it a path of its own. Only a
+        # genuine duplicate (same docnumber, same path) still merges.
+        file = unique_output_file(bib1.docnumber)
+        # A reserved path only ever belongs to one docnumber, so a hit here is
+        # the same document again and still merges. Checked FIRST: a
+        # disambiguated path stays != output_file forever, so gating the merge
+        # on that comparison would drop the merge enrichment on every repeat of
+        # a disambiguated docnumber.
+        if @files.include? file
+          bib1 = merge_duplication bib1, file
+          Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if bib1.nil?
+        else
+          if file != output_file(bib1.docnumber)
+            Util.warn "File #{output_file bib1.docnumber} already exists. " \
+                      "Document: #{bib.docnumber}. Writing #{file} instead."
+          end
+          @files << file
+          index_primary bib1.docidentifier&.detect(&:primary), file
+        end
+        File.write file, serialize(bib1), encoding: "UTF-8" unless bib1.nil?
+      end
+
+      #
+      # Add a document's primary id to the index.
+      #
+      # The pubid is already parsed on `Docidentifier`, so index it directly
+      # rather than re-parsing `docnumber`. `pubid.to_s` (no publisher token)
+      # reproduces the `docnumber` string this used to store byte-for-byte, so
+      # the index key does not move; `docnumber` still names the output file.
+      #
+      # An id pubid cannot rebuild is recorded in `@errors` — the inherited
+      # `report_errors` logs a String value as the message — and the row is
+      # skipped rather than indexed unparsed: `Relaton::Index` rejects the
+      # WHOLE index if a single row fails to deserialize, and its sort calls
+      # `.root.number` on every id. The data file is still written, so the
+      # document is unindexed, never lost. (The W3C precedent.)
+      #
+      # @param [Docidentifier, nil] docid the document's primary identifier
+      # @param [String] file path the document was written to
+      #
+      def index_primary(docid, file)
+        if docid&.pubid
+          index.add_or_update docid.pubid, file
+          return
+        end
+
+        id = docid&.content || file
+        @errors[id.to_s] = "Unparseable primary id `#{id}` was not indexed (#{file})"
+      end
+
+      #
+      # Merge duplication
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [String] file file name of existing bibliographic item
+      #
+      # @return [Relaton3gpp::BibliographicItem, nil] merged bibliographic item or nil if no merge has been done
+      #
+      def merge_duplication(bib, file)
+        hash = YAML.load_file file
+        existed = Item.from_hash hash
+        changed = update_source? bib, existed
+        bib1, bib2, chng = transposed_relation bib, existed
+        changed ||= chng
+        chng = add_contributor(bib1, bib2)
+        changed ||= chng
+        bib1 if changed
+      end
+
+      #
+      # Update link in case one of bibliographic items has no link
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib1
+      # @param [Relaton3gpp::BibliographicItem] bib2
+      #
+      # @return [Boolean] true if link has been updated
+      #
+      def update_source?(bib1, bib2)
+        if bib1.source.any? && bib2.source.empty?
+          bib2.source = bib1.source
+          true
+        elsif bib1.source.empty? && bib2.source.any?
+          bib1.source = bib2.source
+          true
+        else false
+        end
+      end
+
+      #
+      # If one of bibliographic items has date gereater than anotherm=, make it relation
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
+      #
+      # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
+      #   related bibliographic item, true if relation has been added
+      #
+      def transposed_relation(bib, existed) # rubocop:disable Metrics/CyclomaticComplexity
+        if (bib.date.none? && existed.date.none?) || (bib.date.any? && existed.date.none?)
+          return [bib, existed, false]
+        end
+        return [existed, bib, true] if bib.date.none? && existed.date.any?
+
+        check_transposed_date bib, existed
+      end
+
+      #
+      # Check if date of one bibliographic item is transposed to another
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib new bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] existed existing bibliographic item
+      #
+      # @return [Array<Relaton3gpp::BibliographicItem, Boolean>] main bibliographic item,
+      #   related bibliographic item, true if relation has been added
+      #
+      def check_transposed_date(bib, existed)
+        if bib.date[0].at < existed.date[0].at
+          add_transposed_relation bib, existed
+          [bib, existed, true]
+        elsif bib.date[0].at > existed.date[0].at
+          add_transposed_relation existed, bib
+          [existed, bib, true]
+        else [bib, existed, false]
+        end
+      end
+
+      #
+      # Add transposed relation
+      #
+      # @param [Relaton3gpp::BibliographicItem] bib1 the main bibliographic item
+      # @param [Relaton3gpp::BibliographicItem] bib2 the transposed bibliographic item
+      #
+      # @return [Relaton3gpp::BibliographicItem]
+      #
+      def add_transposed_relation(bib1, bib2)
+        bib2.relation.each { |r| bib1.relation << r }
+        bib2.relation.clear
+        desc = Bib::LocalizedMarkedUpString.new content: "equivalent"
+        rel = Bib::Relation.new(type: "adoptedAs", bibitem: bib2, description: desc)
+        bib1.relation << rel
+      end
+
+      def add_contributor(bib1, bib2) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        changed = false
+
+        bib2.contributor.each do |bc|
+          next unless bc.person
+
+          existed = bib1.contributor.find { |ic| ic.person&.name == bc.person.name }
+          if existed
+            chng = add_affiliation existed, bc.person.affiliation
+            changed ||= chng
+          else
+            bib1.contributor << bc
+            changed = true
+          end
+        end
+
+        changed
+      end
+
+      def add_affiliation(contrib, affiliation)
+        changed = false
+
+        affiliation.each do |a|
+          unless contrib.person.affiliation.include? a
+            contrib.person.affiliation << a
+            changed = true
+          end
+        end
+
+        changed
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/docidentifier.rb

@@ -0,0 +1,114 @@
+module Relaton
+  module ThreeGpp
+    # Structured 3GPP document identifier. `content` stays the canonical string
+    # (e.g. "3GPP TS 29.198-04-1:REL-5/5.0.0") and the parsed
+    # `Pubid::Tgpp::Identifier` is kept alongside it in `@pubid`.
+    #
+    # Without this subclass 3GPP items inherit `Bib::Docidentifier`, whose
+    # `remove_part!` / `remove_date!` / `to_all_parts!` all raise
+    # NotImplementedError — so `ItemData#to_all_parts` and
+    # `#to_most_recent_reference`, which broadcast to every docidentifier,
+    # raised on every 3GPP item.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! can write the re-rendered
+      # string back WITHOUT re-parsing (a re-parse would rebuild @pubid from
+      # the string and discard the in-place mutations just made).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= render(pubid) if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          # Rescue LoadError so a missing pubid gem degrades to a plain string,
+          # and StandardError so non-3GPP or unparseable content just leaves
+          # @pubid nil rather than raising. Deserialization reaches this setter
+          # without the flavor entry file necessarily having been loaded, hence
+          # the local require.
+          require "pubid"
+          ::Pubid::Tgpp::Identifier.parse(value.to_s)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # `Pubid::Tgpp::Identifier` models number/suffix/parts/release/version.
+      # It has no date, and `suffix` is folded into the document code, so:
+      #
+      #   * #remove_part! clears `parts` — a real, separable component here
+      #     (unlike IALA's, which is a no-op): "TS 29.198-04-1" -> "TS 29.198".
+      #   * #remove_date! clears `release` AND `version`. 3GPP carries no date;
+      #     those two are its version discriminators — the same pair
+      #     `Bibliography#ignored` treats as omittable — so clearing both gives
+      #     the version-agnostic ("most recent") reference "TS 23.207".
+      #   * #to_all_parts! does both, then wraps `@pubid` in pubid's `AllParts`.
+      #     `content` stays the plain stripped id — the Tgpp renderer has no
+      #     "(all parts)" marker, and `content` is cached from the pre-wrap
+      #     pubid on purpose. `#pubid` itself, read directly, now answers
+      #     `all_parts? == true` and renders WITH pubid's generic marker
+      #     (`#to_s`), since it's the wrapper.
+      #
+      # All three no-op safely when @pubid is nil (parse failed, or the pubid
+      # gem is unavailable), so ItemData#to_all_parts and
+      # #to_most_recent_reference never raise on 3GPP items.
+
+      def remove_part!
+        clear_attrs! :parts
+      end
+
+      def remove_date!
+        clear_attrs! :release, :version
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy — `remove_part!`
+        # / `remove_date!` mutate `@pubid` in place, so a bare reference here
+        # would lose the original part/version to that mutation too.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attrs!(*attrs)
+        return unless @pubid
+
+        cleared = attrs.select { |attr| @pubid.respond_to?("#{attr}=") }
+        return if cleared.empty?
+
+        # `parts` is a collection: empty it rather than nil it, so #code keeps
+        # rendering through `parts.map` instead of tripping over nil.
+        cleared.each do |attr|
+          @pubid.public_send("#{attr}=", attr == :parts ? [] : nil)
+        end
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(render(@pubid)) if @pubid
+      end
+
+      # `Pubid::Tgpp::Identifier#to_s` defaults to OMITTING the "3GPP " token —
+      # deliberately, so it reproduces the index id. The stored docidentifier
+      # content carries the prefix ("3GPP TS 23.207:REL-4/4.0.0"), so every
+      # render here must opt back in or a mutation would silently strip it.
+      def render(pubid)
+        pubid.to_s(with_publisher: true)
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module ThreeGpp
+    class Doctype < Relaton::Bib::Doctype
+      TYPES = %w[TR TS].freeze
+
+      attribute :content, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/3gpp/ext.rb

@@ -0,0 +1,31 @@
+require_relative "doctype"
+require_relative "release"
+
+module Relaton
+  module ThreeGpp
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :release, Release
+      attribute :subdoctype, :string, values: %w[spec release]
+      attribute :radiotechnology, :string, values: %w[2G 3G LTE 5G]
+      attribute :common_ims_spec, :boolean
+      attribute :internal, :boolean
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-3gpp"]
+
+      xml do
+        map_element "radiotechnology", to: :radiotechnology
+        map_element "common-ims-spec", to: :common_ims_spec
+        map_element "internal", to: :internal
+        map_element "release", to: :release
+      end
+
+      key_value do
+        map_element "radiotechnology", to: :radiotechnology
+        map_element "common_ims_spec", to: :common_ims_spec
+        map_element "internal", to: :internal
+        map_element "release", to: :release
+      end
+    end
+  end
+end

data/lib/relaton/3gpp/item.rb

@@ -0,0 +1,18 @@
+require_relative "item_data"
+require_relative "ext"
+require_relative "docidentifier"
+
+module Relaton
+  module ThreeGpp
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      # Shadow the Bib::Docidentifier declaration in Bib::ItemShared so items
+      # carry the pubid-backed 3GPP identifier. Bibitem and Bibdata subclass
+      # Item, so this one declaration covers all three.
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end

data/lib/relaton/3gpp/item_data.rb

@@ -0,0 +1,15 @@
+module Relaton
+  module ThreeGpp
+    class ItemData < Bib::ItemData
+      def create_id(without_date: false)
+        return id if id && !id.empty?
+
+        docid = find_primary_docid
+        return unless docid
+
+        pubid = without_date ? docid.content.sub(/:\d{4}$/, "") : docid.content
+        self.id = pubid.to_s.sub(/\A3GPP\s+/, "").gsub(/\W+/, "")
+      end
+    end
+  end
+end