relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/ccsds/data/parser.rb

@@ -0,0 +1,194 @@
+require_relative "../model/item"
+require_relative "fetcher"
+require_relative "iso_references"
+
+module Relaton
+  module Ccsds
+    class DataParser
+      include Core::ArrayWrapper
+
+      DOCTYPES = { B: "standard", M: "practice", G: "report", O: "specification", Y: "record" }.freeze
+      DOMAIN = "https://public.ccsds.org".freeze
+
+      AREAS = {
+        "SEA" => "Systems Engineering Area",
+        "MOIMS" => "Mission Operations and Information Management Services Area",
+        "CSS" => "Cross Support Services Area",
+        "SOIS" => "Spacecraft Onboard Interface Services Area",
+        "SLS" => "Space Link Services Area",
+        "SIS" => "Space Internetworking Services Area",
+      }.freeze
+
+      ATTRS = %i[
+        title docidentifier abstract date status source edition relation contributor ext
+      ].freeze
+
+      ID_MAPPING = {
+        "CCSDS 320.0-B-1-S Corrigendum 1" => "CCSDS 320.0-B-1-S Cor. 1", # TODO relaton/relaton-data-ccsds#5
+        "CCSDS 701.00-R-3" => "CCSDS 701.00-R-3-S", # TODO relaton/relaton-data-ccsds#8
+      }.freeze
+
+      def initialize(doc, docs, successor = nil)
+        @doc = doc
+        @docs = docs
+        @successor = successor
+      end
+
+      def parse
+        args = ATTRS.each_with_object({}) { |a, o| o[a] = send "parse_#{a}" }
+        item = ItemData.new(**args)
+        item.create_id
+        item
+      end
+
+      def parse_title
+        t = @doc[3]
+        [Bib::Title.new(content: t, language: "en", script: "Latn")]
+      end
+
+      def parse_docidentifier
+        [Bib::Docidentifier.new(content: docidentifier, type: "CCSDS", primary: true)]
+      end
+
+      def docidentifier
+        @docidentifier ||= begin
+          docid = parse_identifier(@doc[2])
+          @successor ? docid.sub(/(-S|s)(?=\s|$)/, "") : docid
+        end
+      end
+
+      def parse_identifier(link)
+        id = link.strip.match(/(?<=>).+(?=<\/a>)/).to_s
+        ID_MAPPING[id] || id
+      end
+
+      def parse_abstract
+        a = @doc[7]
+        [Bib::Abstract.new(content: a, language: "en", script: "Latn")]
+      end
+
+      def parse_doctype
+        />CCSDS\s[\d.]+-(?<type>\w+)/ =~ @doc[2]
+        Doctype.new content: DOCTYPES[type&.to_sym]
+      end
+
+      def parse_date
+        at = @doc[6]
+        [Bib::Date.new(type: "published", at: at)]
+      end
+
+      def parse_status
+        stage = Bib::Status::Stage.new content: @successor ? "withdrawn" : "published"
+        Bib::Status.new stage: stage
+      end
+
+      def parse_source
+        sources = []
+        src = create_source(@doc[2], "src")
+        sources << src if src
+
+        type = @doc[1].match(/https?:[^"]+(pdf|doc)"/)
+        return sources unless type
+
+        download = create_source(@doc[1], type[1])
+        sources.tap { |s| s << download if download }
+      end
+
+      def create_source(link, type)
+        href = parse_href(link)
+        return unless href
+
+        Bib::Uri.new(type: type, content: href)
+      end
+
+      def parse_href(link)
+        /(?<href>https?:[^"]+)/.match(link)&.[](:href)
+      end
+
+      def parse_edition
+        return unless @doc[5]
+
+        Bib::Edition.new content: @doc[5]
+      end
+
+      def parse_relation
+        @docs.each_with_object(successors + adopted) do |d, a|
+          id = parse_identifier d[2]
+          type = relation_type id
+          next unless type
+
+          a << create_relation(type, id)
+        end
+      end
+
+      def adopted
+        /(?<href>https?:[^"]+)/ =~ @doc[9]
+        array(href).each_with_object([]) do |uri, acc|
+          /\/(?<ref_id>\d+)\.html/ =~ uri
+          iso_id = IsoReferences.instance[ref_id]
+          unless iso_id
+            Util.warn "ISO reference not found for #{uri}"
+            next
+          end
+          acc << create_relation("adoptedAs", iso_id, uri, "ISO")
+        end
+      end
+
+      def successors
+        return [] unless @successor
+
+        @successors ||= begin
+          if @successor.relation.none? { |r| r.type == "successorOf" }
+            @successor.relation << create_relation("successorOf", docidentifier)
+          end
+          [create_relation("hasSuccessor", @successor.docidentifier[0].content)]
+        end
+      end
+
+      # TODO: cover this
+      def relation_type(rel_id)
+        return if rel_id == docidentifier ||
+          rel_id.match(DataFetcher::TRRGX).to_s != docidentifier.match(DataFetcher::TRRGX).to_s
+
+        if rel_id.include?(docidentifier.sub(DataFetcher::TRRGX, ""))
+          "updatedBy"
+        elsif docidentifier.include?(rel_id.sub(DataFetcher::TRRGX, ""))
+          "updates"
+        end
+      end
+
+      def create_relation(type, rel_id, uri = nil, id_type = "CCSDS")
+        id = Bib::Docidentifier.new content: rel_id, type: id_type, primary: true
+        source = array(uri).map { |u| Bib::Uri.new(type: "src", content: u) }
+        bibitem = Bib::ItemData.new docidentifier: [id], source: source, formattedref: Bib::Formattedref.new(content: rel_id)
+        Bib::Relation.new type: type, bibitem: bibitem
+      end
+
+      def parse_contributor # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        array(@doc[8]).each_with_object([]) do |wg, acc|
+          /^(?<name>[^<]+)/ =~ wg
+          next if name.nil? || name.strip.empty?
+
+          sdname = Bib::TypedLocalizedString.new content: name.strip
+          subdiv = Bib::Subdivision.new type: "technical-committee", name: [sdname]
+          orgname = Bib::TypedLocalizedString.new content: "CCSDS"
+          org = Bib::Organization.new name: [orgname], subdivision: [subdiv]
+          description = Bib::LocalizedMarkedUpString.new(content: "committee")
+          role = Bib::Contributor::Role.new type: "author", description: [description]
+          acc << Bib::Contributor.new(role: [role], organization: org)
+        end
+      end
+
+      def parse_ext
+        Ext.new flavor: "ccsds", doctype: parse_doctype, technology_area: parse_technology_area
+      end
+
+      def parse_technology_area
+        return unless @doc[8]
+
+        desc = @doc[8].split("-").first
+        AREAS[desc]
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/hit.rb

@@ -0,0 +1,24 @@
+require "mechanize"
+module Relaton
+  module Ccsds
+    class Hit < Relaton::Core::Hit
+      # attr_reader :code
+
+      # def initialize(code:, url:)
+      #   @code = code
+      #   @url = url
+      # end
+
+      def item
+        return @item if @item
+
+        resp = Mechanize.new.get(hit[:url])
+        @item = Item.from_yaml(resp.body)
+        @item.fetched = Date.today.to_s
+        @item
+      rescue Mechanize::Error => e
+        raise Relaton::RequestError, e.message
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/hit_collection.rb

@@ -0,0 +1,42 @@
+require_relative "hit"
+
+module Relaton
+  module Ccsds
+    class HitCollection < Relaton::Core::HitCollection
+      GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-ccsds/refs/heads/v2/".freeze
+
+      #
+      # Search his in index.
+      #
+      # @return [<Type>] <description>
+      #
+      def fetch
+        @array = rows.map { |row| Hit.new code: row[:id], url: "#{GHURL}#{row[:file]}" }
+        self
+      rescue SocketError, OpenURI::HTTPError, OpenSSL::SSL::SSLError, Errno::ECONNRESET => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # Pubid index (index-v2): `:id` deserializes to a Pubid::Ccsds::Identifier
+      # via pubid_class, so search narrows by number with binary search.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :ccsds, url: "#{GHURL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml", pubid_class: Pubid::Ccsds::Identifier
+        )
+      end
+
+      def pubid
+        @pubid ||= Pubid::Ccsds::Identifier.parse(ref)
+      end
+
+      def rows
+        if pubid.edition
+          index.search(pubid)
+        else
+          # search(pubid) narrows candidates by number via binary search first.
+          index.search(pubid) { |r| r[:id].exclude(:edition) == pubid }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/item_data.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Ccsds
+    class ItemData < Bib::ItemData
+      def relation=(value)
+        @relation = value || []
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ccsds
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/ccsds/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ccsds
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/ccsds/model/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Ccsds
+    class Doctype < Bib::Doctype
+      TYPES = %w[standard practice report specification record].freeze
+
+      attribute :content, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/ccsds/model/ext.rb

@@ -0,0 +1,19 @@
+require_relative "doctype"
+
+module Relaton
+  module Ccsds
+    class Ext < Lutaml::Model::Serializable
+      attribute :schema_version, :string
+      attribute :doctype, Doctype
+      attribute :structuredidentifier, Bib::StructuredIdentifier, collection: true
+      attribute :technology_area, :string, values: %w[SEA MOIMS CSS SOIS SLS SIS]
+
+      xml do
+        map_attribute "schema-version", to: :schema_version
+        map_element "doctype", to: :doctype
+        map_element "structuredidentifier", to: :structuredidentifier
+        map_element "technology-area", to: :technology_area
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/model/item.rb

@@ -0,0 +1,13 @@
+require "relaton/bib"
+require_relative "../item_data"
+require_relative "ext"
+
+module Relaton
+  module Ccsds
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/ccsds/processor.rb

@@ -0,0 +1,68 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Ccsds
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_ccsds
+        @prefix = "CCSDS"
+        @defaultprefix = %r{^CCSDS(?!\w)}
+        @idtype = "CCSDS"
+        @datasets = %w[ccsds]
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Ccsds::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../ccsds"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from a source
+      #
+      # @param [String] source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(source = "ccsds", **opts)
+        require_relative "data/fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Ccsds::ItemData]
+      def from_xml(xml)
+        require_relative "../ccsds"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Ccsds::ItemData]
+      def from_yaml(yaml)
+        require_relative "../ccsds"
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../ccsds"
+        @grammar_hash ||= ::Relaton::Ccsds.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../ccsds"
+        Relaton::Index.find_or_create(:ccsds, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/util.rb

@@ -0,0 +1,10 @@
+require "relaton/bib/util"
+
+module Relaton
+  module Ccsds
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-ccsds".freeze
+    end
+  end
+end

data/lib/relaton/ccsds.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# require "mechanize"
+# require "relaton/core"
+# require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+# require_relative "ccsds/processor"
+require_relative "ccsds/util"
+require_relative "ccsds/model/item"
+require_relative "ccsds/model/bibitem"
+require_relative "ccsds/model/bibdata"
+require_relative "ccsds/bibliography"
+
+module Relaton
+  module Ccsds
+    # The pubid-v2 (lean hash) index this gem produces and consumes. The legacy
+    # index-v1 (pubid-v1 hash, for the released gem line) is rebuilt separately
+    # by the data repo's build_index_v1.rb.
+    INDEXFILE = "index-v2"
+
+    class Error < StandardError; end
+    # Your code goes here...
+
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/cen/bibliography.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Cen
+    # Class methods for search Cenelec standards.
+    class Bibliography
+      class << self
+        # @param text [String]
+        # @return [RelatonCen::HitCollection]
+        def search(text, year = nil)
+          # /^C?EN\s(?<code>.+)/ =~ text
+          HitCollection.new(text, year).search
+        rescue Mechanize::ResponseCodeError, Net::ReadTimeout => e
+          raise Relaton::RequestError, e.message
+        end
+
+        #
+        # @param code [String] the CEN standard Code to look up
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options
+        # @option opts [Boolean] :keep_year don't upate reference
+        #
+        # @return [RelatonBib::BibliographicItem, nil]
+        #
+        def get(code, year = nil, opts = {})
+          code_parts = code_to_parts code
+          year ||= code_parts[:year] if code_parts
+
+          bib_get(code, year, opts)
+        end
+
+        #
+        # Decopmposes a CEN standard code into its parts.
+        #
+        # @param [String] code the CEN standard code to decompose
+        #
+        # @return [MatchData] the decomposition of the code
+        #
+        def code_to_parts(code)
+          %r{^
+            (?<code>[^:-]+)(?:-(?<part>\d+))?
+            (?::(?<year>\d{4}))?
+            (?:\+(?<amd>[A-Z]\d+)(?:(?<amy>\d{4}))?)?
+            (?:\/(?<ac>AC\d+:\d{4}))?
+          }x.match code
+        end
+
+        private
+
+        def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
+          # id = year ? "#{code}:#{year}" : code
+          # Util.warn "WARNING: No match found online for `#{id}`. " \
+          #           "The code must be exactly like it is on the standards website."
+          unless missed_years.empty?
+            Util.info "There was no match for `#{year}`, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`."
+          end
+          # if /\d-\d/.match? code
+          #   warn "[relaton-cen] The provided document part may not exist, or "\
+          #     "the document may no longer be published in parts."
+          # else
+          #   warn "[relaton-cen] If you wanted to cite all document parts for "\
+          #     "the reference, use \"#{code} (all parts)\".\nIf the document "\
+          #     "is not a standard, use its document type abbreviation (TS, TR, "]
+          #     "PAS, Guide)."
+          # end
+          nil
+        end
+
+        # @param code [String]
+        # @return [RelatonCen::HitCollection]
+        def search_filter(code) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+          parts = code_to_parts code
+          result = search(code)
+          result.select! do |i|
+            pts = code_to_parts i.hit[:code]
+            parts[:code] == pts[:code] &&
+              (!parts[:part] || parts[:part] == pts[:part]) &&
+              (!parts[:year] || parts[:year] == pts[:year]) &&
+              parts[:amd] == pts[:amd] && (!parts[:amy] || parts[:amy] == pts[:amy])
+          end
+        end
+
+        # Sort through the results from Isobib, fetching them three at a time,
+        # and return the first result that matches the code,
+        # matches the year (if provided), and which # has a title (amendments do not).
+        # Only expects the first page of results to be populated.
+        # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+        # If no match, returns any years which caused mismatch, for error reporting
+        def isobib_results_filter(result, year)
+          missed_years = []
+          result.each do |r|
+            /:(?<pyear>\d{4})/ =~ r.hit[:code]
+            if !year || year == pyear
+              ret = r.item
+              return { ret: ret } if ret
+            end
+
+            missed_years << pyear
+          end
+          { years: missed_years }
+        end
+
+        def bib_get(code, year, opts) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
+          ref = year.nil? || code.match?(/:\d{4}/) ? code : "#{code}:#{year}"
+          Util.info "Fetching from standards.cencenelec.eu ...", key: ref
+          result = search_filter(code) || return
+          ret = isobib_results_filter(result, year)
+          if ret[:ret]
+            bib = year || opts[:keep_year] ? ret[:ret] : ret[:ret].to_most_recent_reference
+            Util.info "Found: `#{bib.docidentifier.first&.content}`", key: ref
+            bib
+          else
+            Util.info "Not found.", key: ref
+            fetch_ref_err(code, year, ret[:years])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/cen/committees.yaml

@@ -0,0 +1,66 @@
+ASD-STAN: Aerospace
+CEN/CLC/ETSI/SMCG: CEN-CENELEC-ETSI Coordination Group on Smart Meters
+CEN/CLC/ETSI/JWG eAcc: eAccessibility
+CEN/CLC/ETSI/SEG-CG: CEN-CENELEC-ETSI Coordination Group on Smart Grids
+CEN/CLC/ETSI/SF-SSCC: CEN-CENELEC-ETSI Sector Forum on Smart and Sustainable Cities and Communities
+CEN/CLC/Guides: Group for CEN-CENELEC Guides
+CEN/CLC/JTC 1: Criteria for conformity assessment bodies
+CEN/CLC/JTC 2: Power Engineering
+CEN/CLC/JTC 3: Quality management and corresponding general aspects for medical devices
+CEN/CLC/JTC 4: Services for fire safety and security systems
+CEN/CLC/JTC 5: Space
+CEN/CLC/JTC 6: Hydrogen in energy systems
+CEN/CLC/JTC 10: Energy-related products - Material Efficiency Aspects for Ecodesign
+CEN/CLC/JTC 11: Accessibility in the built environment
+CEN/CLC/JTC 12: Design for All
+CEN/CLC/JTC 13: Cybersecurity and Data Protection
+CEN/CLC/JTC 14: Energy management and energy efficiency in the framework of energy transition
+CEN/CLC/JTC 15: Energy measurement plan for organizations
+CEN/CLC/JTC 16: CEN/CENELEC Joint Technical Committee on Active Implantable Medical Devices
+CEN/CLC/JTC 17: Gas Appliances with Combined Heat and Power
+CEN/CLC/JTC 18: Non automatic weighing instruments (NAWI)
+CEN/CLC/JTC 19: Blockchain and Distributed Ledger Technologies
+CEN/CLC/JTC 20: Hyperloop systems
+CEN/CLC/JTC 21: Artificial Intelligence
+CEN/CLC/WS 017: Development of a GALILEO enabled label
+CEN/CLC/WS 018: Assessment of the resilience of transport infrastructure to potentially disruptive events
+CEN/CLC/WS SEA-TITAN: Modular and cross-cutting Power Take-Off units for wave energy converters. Recommendations and laboratory testing
+CEN/CLC/WS SEP2: Industry Best Practices and an Industry Code of Conduct for Licensing of Standard Essential Patents in the field of 5G and Internet of Things
+CEN/CLC/WS HECTOS: CEN-CENELEC Workshop on Guidelines on evaluation systems and schemes for physical security products
+CEN/CLC/WS INACHUS: Urban search and rescue (USaR) robotic platform technical and procedural interoperability
+CEN/CLC/WS Monsoon: Predictive management of data intensive industrial processes
+CEN/CLC/WS REEMAIN: CEN/CENELEC Workshop on REEMAIN Methodology for Resource and Energy Efficiency Manufacturing
+CEN/CLC/WS SEP-IoT: Workshop on Best Practices and a Code of Conduct for Licensing Industry Standard Essential Patents in 5G and the Internet of Things (IoT), including the Industrial Internet
+CEN/CLC/WS ZONeSEC: Interoperability of security systems for the surveillance of widezones
+CEN/CLC/WS EINSTEIN: Good Practice Thermal Energy Audits (GPTEA)
+CEN/CLC/WS WiseGRID: Reference model for distribution application for microgrids
+CEN/CLC/WS EFPFInterOp: European Connected Factory Platform for Agile Manufacturing Interoperability
+CEN/CLC/WS ZDMterm: Zero Defects in Digital Manufacturing Terminology
+CEN/SS C20: Explosives and firework
+CEN/SS S08: Air quality
+CEN/SS S12: Gas analysis
+CEN/Guides: Group for CEN Guides
+CEN/SS A03: Postal services
+CEN/SS A07: Translation and Interpretation services
+CEN/SS A08: Funeral services
+CEN/SS A10: Services of Real Estate Agents
+CEN/SS A11: Security services
+CEN/SS A99: Services - Undetermined
+CEN/SS B02: Structures
+CEN/SS B09: Energy Performance of Buildings Directive (EPBD)
+CEN/SS B99: Building and construction - Undetermined
+CEN/SS C01: Food Products
+CEN/SS C10: Starch
+CEN/SS F01: Technical drawings
+CEN/SS F02: Units and symbols
+CEN/SS F05: Measuring Instruments
+CEN/SS F12: Information Processing Systems
+CEN/SS F16: Graphical symbols
+CEN/SS F17: Administrative documents
+CEN/SS F20: Quality assurance
+CEN/SS F23: Energy
+CEN/SS H10: Sewing machines
+CEN/SS H22: Smokers' lighters
+CEN/SS H99: Products for household and leisure use - Undetermined
+
+TC 459: ECISS - European Committee for Iron and Steel Standardization

data/lib/relaton/cen/hit.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Cen
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      attr_writer :item
+
+      # Parse page.
+      # @return [IsoRelatonBib::IsoBibliographicItem]
+      def item
+        @fetch ||= Scraper.parse_page self
+      end
+    end
+  end
+end