relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/iana/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iana
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/iana/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iana
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/iana/bibliography.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Iana
+    # Methods for search IANA standards.
+    module Bibliography
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
+
+      # @param text [String]
+      # @return [Relaton::Iana::ItemData, nil]
+      def search(text) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        index = Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
+        id = text.sub(/^IANA\s/, "")
+        row = index.search(id).min_by { |i| i[:id] }
+        return unless row
+
+        url = "#{SOURCE}#{row[:file]}"
+        resp = Net::HTTP.get_response URI(url)
+        return unless resp.code == "200"
+
+        Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # @param ref [String] the W3C standard Code to look up
+      # @param year [String, NilClass] not used
+      # @param opts [Hash] options
+      # @return [Relaton::Iana::ItemData, nil]
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(ref)
+        unless result
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        result
+      end
+
+      extend self
+    end
+  end
+end

data/lib/relaton/iana/data_fetcher.rb

@@ -0,0 +1,71 @@
+require "relaton/core"
+require_relative "../iana"
+require_relative "parser"
+
+module Relaton
+  module Iana
+    class DataFetcher < Core::DataFetcher
+      #
+      # Parse documents
+      #
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(_source = nil)
+        Dir["iana-registries/**/*.xml"].each do |file|
+          content = File.read file, encoding: "UTF-8"
+          parse(content) if content.include? "<registry"
+        rescue StandardError => e
+          Util.error "Error: #{e.message}. File: #{file}"
+        end
+        index.save
+        report_errors
+      end
+
+      private
+
+      def index
+        @index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml"
+      end
+
+      def parse(content)
+        xml = Nokogiri::XML(content)
+        registry = xml.at("/xmlns:registry")
+        doc = Parser.parse registry, nil, @errors
+        save_doc doc
+        registry.xpath("./xmlns:registry").each { |r| save_doc Parser.parse(r, doc, @errors) }
+      end
+
+      #
+      # Save document to file
+      #
+      # @param [RelatonIana::IanaBibliographicItem, nil] bib bibliographic item
+      #
+      def save_doc(bib) # rubocop:disable Metrics/MethodLength
+        return unless bib
+
+        file = output_file(bib.docnumber)
+        if @files.include? file
+          Util.warn "File #{file} already exists. Document: #{bib.docnumber}"
+        else
+          @files << file
+        end
+        index.add_or_update bib.docnumber, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/iana/item.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Iana
+    class Item < Bib::Item
+      model ItemData
+    end
+  end
+end

data/lib/relaton/iana/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Iana
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/iana/parser.rb

@@ -0,0 +1,146 @@
+module Relaton
+  module Iana
+    class Parser
+      #
+      # Document parser initalization
+      #
+      # @param [Nokogiri::XML::Element] xml
+      #
+      def initialize(xml, rootdoc, errors = {})
+        @xml = xml
+        @rootdoc = rootdoc
+        @errors = errors
+      end
+
+      #
+      # Initialize document parser and run it
+      #
+      # @param [Nokogiri::XML::Element] xml
+      #
+      # @return [Relaton::Iana::ItemData, nil] bibliographic item
+      #
+      def self.parse(xml, rootdoc = nil, errors = {})
+        new(xml, rootdoc, errors).parse
+      end
+
+      #
+      # Parse document
+      #
+      # @return [Relaton::Iana::ItemData, nil] bibliographic item
+      #
+      def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        return unless @xml
+
+        Relaton::Iana::ItemData.new(
+          type: "standard",
+          language: ["en"],
+          script: ["Latn"],
+          title: parse_title,
+          source: parse_source,
+          docidentifier: parse_docid,
+          docnumber: docnumber,
+          date: parse_date,
+          contributor: contributor,
+        )
+      end
+
+      #
+      # Parse title
+      #
+      # @return [Array<Relaton::Bib::Title>] title
+      #
+      def parse_title
+        content = @xml.at("./xmlns:title")&.text || @xml[:id]
+        result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse link
+      #
+      # @return [Array<Relaton::Bib::Uri>] link
+      #
+      def parse_source
+        result = if @rootdoc then @rootdoc.source
+                 elsif @xml[:id]
+                   uri = URI.join "https://www.iana.org/assignments/", @xml[:id]
+                   [Bib::Uri.new(type: "src", content: uri.to_s)]
+                 else
+                   []
+                 end
+        @errors[:source] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse docidentifier
+      #
+      # @return [Arra<RelatonBib::DocumentIdentifier>] docidentifier
+      #
+      def parse_docid
+        result = [Bib::Docidentifier.new(type: "IANA", content: pub_id, primary: true)]
+        @errors[:docid] &&= result.empty?
+        result
+      end
+
+      #
+      # Create anchor
+      #
+      # @return [String] anchor
+      #
+      def anchor
+        docnumber.upcase.gsub("/", "__")
+      end
+
+      #
+      # Generate PubID
+      #
+      # @return [String] PubID
+      #
+      def pub_id
+        "IANA #{docnumber}"
+      end
+
+      #
+      # Create docnumber
+      #
+      # @return [String] docnumber
+      #
+      def docnumber
+        dn = ""
+        dn += "#{@rootdoc.docnumber}/" if @rootdoc
+        id = @xml["id"].to_s
+        @errors[:docnumber] &&= id.empty?
+        dn + id
+      end
+
+      #
+      # Parse date
+      #
+      # @return [Array<Relaton::Bib::Date>] date
+      #
+      def parse_date
+        d = @xml.xpath("./xmlns:created|./xmlns:published|./xmlns:updated").map do |dt|
+          Bib::Date.new(type: dt.name, at: dt.text)
+        end
+        result = d.none? && @rootdoc ? @rootdoc.date : d
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Create contributor
+      #
+      # @return [Array<Relaton::Bib::Contributor>] contributor
+      #
+      def contributor
+        orgname = Bib::TypedLocalizedString.new(content: "Internet Assigned Numbers Authority")
+        abbrev = Bib::LocalizedString.new(content: "IANA")
+        org = Bib::Organization.new(name: [orgname], abbreviation: abbrev)
+        role = Bib::Contributor::Role.new(type: "publisher")
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+    end
+  end
+end

data/lib/relaton/iana/processor.rb

@@ -0,0 +1,68 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Iana
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_iana
+        @prefix = "IANA"
+        @defaultprefix = %r{^IANA\s}
+        @idtype = "IANA"
+        @datasets = %w[iana-registries]
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Iana::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../iana"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from https://github.com/ietf-ribose/iana-registries
+      #
+      # @param [String] source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format output format, "xml" or "yaml"
+      #
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Iana::ItemData]
+      def from_xml(xml)
+        require_relative "../iana"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [RelatonIana::ItemData]
+      def from_yaml(yaml)
+        require_relative "../iana"
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../iana"
+        @grammar_hash ||= ::Relaton::Iana.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../iana"
+        Relaton::Index.find_or_create(:iana, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/iana/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iana
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-iana".freeze
+    end
+  end
+end

data/lib/relaton/iana.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "iana/util"
+require_relative "iana/item_data"
+require_relative "iana/item"
+require_relative "iana/bibitem"
+require_relative "iana/bibdata"
+require_relative "iana/bibliography"
+# require_relative "relaton_iana/parser"
+# require_relative "relaton_iana/data_fetcher"
+
+module Relaton
+  module Iana
+    INDEXFILE = "index-v1"
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/iec/bibliography.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+require_relative "hit_collection"
+require "date"
+
+module Relaton
+  module Iec
+    # Class methods for search IEC standards.
+    class Bibliography
+      extend Core::ArrayWrapper
+
+      DOCTYPES = %w[TS TR PAS SRD TEC STTR WP Guide OD CS CA].freeze
+
+      class << self
+        ##
+        # Search for standards entries.
+        #
+        # @param pubid [Pubid::Iec::Identifier]
+        # @param exclude [Array<Symbol>] keys to exclude from comparison
+        # @return [Relaton::Iec::HitCollection]
+        def search(pubid, exclude: [:year])
+          HitCollection.new(pubid).search(exclude: exclude)
+        rescue SocketError, OpenSSL::SSL::SSLError => e
+          raise Relaton::RequestError, e.message
+        end
+
+        # @param code [String] the IEC standard code to look up (e.g. "IEC 8000")
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options; restricted to :all_parts if all-parts
+        #   reference is required
+        # @return [Relaton::Iec::ItemData, nil]
+        def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+          opts[:all_parts] ||= code.match?(/\s\(all parts\)/)
+          ref = code.sub(/\s\(all parts\)/, "")
+          return iev if ref.casecmp("IEV").zero?
+
+          pubid = ::Pubid::Iec::Identifier.parse ref.upcase
+          if year
+            pubid.date = ::Pubid::Components::Date.new(year: year.to_s)
+          end
+
+          ret = iecbib_get(pubid, opts)
+          return nil if ret.nil?
+
+          ret = ret.to_most_recent_reference unless pubid.date&.year || opts[:keep_year] ||
+            opts[:publication_date_before] || opts[:publication_date_after]
+          ret
+        end
+
+        private
+
+        def iev(code = "IEC 60050")
+          ItemData.new(
+            type: "standard",
+            fetched: Date.today,
+            title: [Bib::Title.new(
+              content: "International Electrotechnical Vocabulary", language: "en", script: "Latn"
+            )],
+            source: [Bib::Uri.new(content: "http://www.electropedia.org")],
+            docidentifier: [Docidentifier.new(content: "#{code}:2011", type: "IEC", primary: true)],
+            date: [Bib::Date.new(type: "published", at: "2011")],
+            contributor: [Bib::Contributor.new(
+              role: [Bib::Contributor::Role.new(type: "publisher")],
+              organization: Bib::Organization.new(
+                name: [Bib::TypedLocalizedString.new(
+                  content: "International Electrotechnical Commission", language: "en", script: "Latn"
+                )],
+                abbreviation: Bib::LocalizedString.new(content: "IEC", language: "en", script: "Latn"),
+                uri: [Bib::Uri.new(content: "www.iec.ch")]
+              )
+            )],
+            language: %w(en fr),
+            script: "Latn",
+            status: Bib::Status.new(stage: Bib::Status::Stage.new(content: "60")),
+            copyright: Bib::Copyright.new(
+              from: "2018",
+              owner: [Bib::ContributionInfo.new(
+                organization: Bib::Organization.new(
+                  name: [Bib::TypedLocalizedString.new(
+                    content: "International Electrotechnical Commission", language: "en", script: "Latn"
+                  )],
+                  abbreviation: Bib::LocalizedString.new(content: "IEC", language: "en", script: "Latn"),
+                  uri: [Bib::Uri.new(content: "www.iec.ch")]
+                )
+              )]
+            )
+          )
+        end
+
+        # @param pubid [Pubid::Iec::Identifier]
+        # @param opts [Hash]
+        # @return [Relaton::Iec::ItemData, nil]
+        def iecbib_get(pubid, opts) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          Util.info "Fetching from Relaton repository ...", key: pubid.to_s
+          # pubid 2.x splits multi-level numbers ("61326-2-6") into part +
+          # subpart, so all-parts matching must ignore subpart too.
+          exclude = opts[:all_parts] ? %i[year part subpart] : %i[year]
+          result = search(pubid, exclude: exclude) || return
+
+          if opts[:all_parts]
+            ret = result.to_all_parts(pubid.date&.year&.to_s, opts)
+            Util.info "Found: `#{ret&.docidentifier&.first&.content}`", key: pubid.to_s if ret
+            return ret
+          end
+
+          ret = find_match(result, pubid, opts)
+          return ret if ret
+
+          provide_tips(pubid, result)
+          nil
+        end
+
+        # Find exact match considering year and date filters.
+        # @param result [Relaton::Iec::HitCollection]
+        # @param pubid [Pubid::Iec::Identifier]
+        # @param opts [Hash]
+        # @return [Relaton::Iec::ItemData, nil]
+        def find_match(result, pubid, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+          pubid_year = pubid.date&.year
+          if pubid_year
+            hit = result.detect { |h| h.hit[:id].date&.year == pubid_year }
+            return fetch_and_check_date(hit, pubid, opts) if hit
+          elsif opts[:publication_date_before] || opts[:publication_date_after]
+            candidates = result.select { |h| year_in_range?(h.hit[:id].date&.year.to_i, opts) }
+            candidates = candidates.sort_by { |h| -h.hit[:id].date&.year.to_i }
+            candidates.each do |h|
+              ret = fetch_and_check_date(h, pubid, opts)
+              return ret if ret
+            end
+            return nil
+          else
+            hit = result.max_by { |h| h.hit[:id].date&.year.to_i }
+            return unless hit
+
+            ret = hit.item
+            Util.info "Found: `#{ret.docidentifier.first.content}`", key: pubid.to_s
+            return ret
+          end
+          nil
+        end
+
+        # Check if a year falls within the date filter range.
+        # @param year [Integer]
+        # @param opts [Hash]
+        # @return [Boolean]
+        def year_in_range?(year, opts)
+          return false if year.zero?
+
+          if opts[:publication_date_before]
+            return false if year > opts[:publication_date_before].year
+          end
+          if opts[:publication_date_after]
+            return false if year < opts[:publication_date_after].year
+          end
+          true
+        end
+
+        # Check if the item's published date falls within the filter range.
+        # @param item [Relaton::Iec::ItemData]
+        # @param opts [Hash]
+        # @return [Boolean]
+        def publication_date_in_range?(item, opts)
+          pub_date_entry = item.date.find { |d| d.type == "published" }
+          return true unless pub_date_entry&.at
+
+          pub_date = pub_date_entry.at.to_date
+          return true unless pub_date
+
+          if opts[:publication_date_before]
+            return false if pub_date >= opts[:publication_date_before]
+          end
+          if opts[:publication_date_after]
+            return false if pub_date < opts[:publication_date_after]
+          end
+          true
+        end
+
+        # Fetch the item for a hit and check if its publication date is in range.
+        # @param hit [Relaton::Iec::Hit]
+        # @param pubid [Pubid::Iec::Identifier]
+        # @param opts [Hash]
+        # @return [Relaton::Iec::ItemData, nil]
+        def fetch_and_check_date(hit, pubid, opts)
+          ret = hit.item
+          if publication_date_in_range?(ret, opts)
+            Util.info "Found: `#{ret.docidentifier.first.content}`", key: pubid.to_s
+            freeze_item(ret, opts)
+          end
+        end
+
+        # Freeze a document in time by filtering out relations, dates, and status
+        # that fall outside the specified date range.
+        def freeze_item(item, opts) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+          return item unless opts[:publication_date_before] || opts[:publication_date_after]
+
+          item.relation = item.relation.select { |r| relation_in_range?(r, opts) }
+
+          had_obsoleted = item.date.any? { |d| d.type == "obsoleted" }
+          item.date = item.date.select { |d| date_entry_in_range?(d, opts) }
+          lost_obsoleted = had_obsoleted && item.date.none? { |d| d.type == "obsoleted" }
+
+          if lost_obsoleted && item.status&.stage&.content == "95"
+            item.status.stage.content = "60"
+            item.status.substage&.content = "60"
+          end
+
+          item
+        end
+
+        # Check if a relation's bibitem falls within the date range.
+        def relation_in_range?(rel, opts) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+          rel.bibitem.date&.each do |d|
+            dt = (d.at || d.from)&.to_date
+            next unless dt
+
+            return false if opts[:publication_date_before] && dt >= opts[:publication_date_before]
+            return false if opts[:publication_date_after] && dt < opts[:publication_date_after]
+          end
+
+          rel.bibitem.docidentifier&.each do |did|
+            year = did.to_s[/:(\d{4})/, 1]&.to_i
+            next unless year&.positive?
+
+            return false if opts[:publication_date_before] && year >= opts[:publication_date_before].year
+            return false if opts[:publication_date_after] && year < opts[:publication_date_after].year
+          end
+
+          true
+        end
+
+        # Check if a date entry falls within the date range (always keeps published).
+        def date_entry_in_range?(date_entry, opts)
+          return true if date_entry.type == "published"
+
+          dt = (date_entry.at || date_entry.from)&.to_date
+          return true unless dt
+
+          return false if opts[:publication_date_before] && dt >= opts[:publication_date_before]
+          return false if opts[:publication_date_after] && dt < opts[:publication_date_after]
+
+          true
+        end
+
+        # Analyze why no match was found and give helpful tips.
+        # @param pubid [Pubid::Iec::Identifier]
+        # @param result [Relaton::Iec::HitCollection]
+        def provide_tips(pubid, result) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+          Util.info "Not found.", key: pubid.to_s
+
+          # Year mismatch: hits exist but not for the requested year
+          if pubid.date&.year && result.any?
+            years = result.map { |h| h.hit[:id].date&.year&.to_s }.compact.uniq.sort
+            Util.info "TIP: No match for edition year `#{pubid.date&.year}`, " \
+                      "but matches exist for `#{years.join('`, `')}`.", key: pubid.to_s
+            return
+          end
+
+          # Search broadly (exclude year + part) to check for part/type mismatches
+          broad = search(pubid, exclude: %i[year part])
+
+          # Part mismatch: no part given but parts exist
+          unless pubid.part
+            if broad.any?
+              parts = broad.map { |h| h.hit[:id].to_s }.uniq.sort
+              Util.info "TIP: If you wish to cite all document parts for " \
+                        "the reference, use `#{pubid} (all parts)`. " \
+                        "Available: `#{parts.join('`, `')}`.", key: pubid.to_s
+              return
+            end
+          end
+
+          # Doctype mismatch: search excluding type to find entries with same number but different type
+          type_broad = search(pubid, exclude: %i[year type])
+          if type_broad.any?
+            types = type_broad.map { |h| h.hit[:id].to_s }.uniq.sort
+            Util.info "TIP: No match for type, but matches exist: " \
+                      "`#{types.join('`, `')}`.", key: pubid.to_s
+          end
+        end
+      end
+    end
+  end
+end