relaton-ccsds 1.14.0

data/lib/relaton_ccsds/bibliographic_item.rb

@@ -0,0 +1,49 @@
+module RelatonCcsds
+  class BibliographicItem < RelatonBib::BibliographicItem
+    DOCTYPES = %w[standard practice report specification record].freeze
+
+    attr_reader :technology_area
+
+    # @param technology_area [String, nil]
+    def initialize(**args)
+      if args[:doctype] && !DOCTYPES.include?(args[:doctype])
+        Util.warn "WARNING: invalid doctype: #{args[:doctype]}"
+      end
+      @technology_area = args.delete(:technology_area)
+      super
+    end
+
+    #
+    # Fetch flavor schema version
+    #
+    # @return [String] schema version
+    #
+    def ext_schema
+      @ext_schema ||= schema_versions["relaton-model-ccsds"]
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(**opts) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      super do |builder|
+        if opts[:bibdata] && (doctype || editorialgroup || technology_area)
+          ext = builder.ext do |b|
+            b.doctype doctype if doctype
+            editorialgroup&.to_xml b
+            b.send(:"technology-area", technology_area) if technology_area
+          end
+          ext["schema-version"] = ext_schema if !opts[:embedded] && respond_to?(:ext_schema) && ext_schema
+        end
+      end
+    end
+
+    # @return [Hash]
+    def to_hash
+      hash = super
+      if technology_area
+        hash["ext"] ||= {}
+        hash["ext"]["technology_area"] = technology_area
+      end
+      hash
+    end
+  end
+end

data/lib/relaton_ccsds/bibliography.rb

@@ -0,0 +1,37 @@
+module RelatonCcsds
+  module Bibliography
+    extend self
+
+    #
+    # Search for CCSDS standards by document reference.
+    #
+    # @param [String] ref document reference
+    #
+    # @return [RelatonCcsds::HitCollection] collection of hits
+    #
+    def search(ref)
+      RelatonCcsds::HitCollection.new(ref).fetch
+    end
+
+    #
+    # Get CCSDS standard by document reference.
+    #
+    # @param text [String]
+    # @param year [String, nil]
+    # @param opts [Hash]
+    #
+    # @return [RelatonCcsds::BibliographicItem]
+    #
+    def get(ref, _year = nil, _opts = {})
+      Util.warn "(#{ref}) fetching..."
+      hits = search ref
+      if hits.empty?
+        Util.warn "(#{ref}) not found."
+        return nil
+      end
+      doc = hits.first.doc
+      Util.warn "(#{ref}) found `#{hits.first.code}`."
+      doc
+    end
+  end
+end

data/lib/relaton_ccsds/config.rb

@@ -0,0 +1,10 @@
+module RelatonCcsds
+  module Config
+    include RelatonBib::Config
+  end
+  extend Config
+
+  class Configuration < RelatonBib::Configuration
+    PROGNAME = "relaton-ccsd".freeze
+  end
+end

data/lib/relaton_ccsds/data_fetcher.rb

@@ -0,0 +1,102 @@
+module RelatonCcsds
+  class DataFetcher
+    ACTIVE_PUBS_URL = <<~URL.freeze
+      https://public.ccsds.org/_api/web/lists/getbytitle('CCSDS%20Publications')/items?$top=1000&$select=Dcoument_x0020_Title,
+      Document_x0020_Number,Book_x0020_Type,Issue_x0020_Number,calPublishedMonth,calPublishedYear,Description0,Working_x0020_Group,
+      FileRef,ISO_x0020_Number,Patents,Extra_x0020_Link,Area,calActive,calHtmlColorCode&$filter=Book_x0020_Type%20eq%20%27Blue%20
+      Book%27%20or%20Book_x0020_Type%20eq%20%27Magenta%20Book%27%20or%20Book_x0020_Type%20eq%20%27Green%20Book%27%20or%20
+      Book_x0020_Type%20eq%20%27Orange%20Book%27%20or%20Book_x0020_Type%20eq%20%27Yellow%20Book%20-%20Reports%20and%20Records%27%20
+      or%20Book_x0020_Type%20eq%20%27Yellow%20Book%20-%20CCSDS%20Normative%20Procedures%27
+    URL
+
+    OBSOLETE_PUBS_URL = <<~URL.freeze
+      https://public.ccsds.org/_api/web/lists/getbytitle('CCSDS%20Publications')/items?$top=1000&$select=Dcoument_x0020_Title,
+      Document_x0020_Number,Book_x0020_Type,Issue_x0020_Number,calPublishedMonth,calPublishedYear,Description0,Working_x0020_Group,
+      FileRef,ISO_x0020_Number,Patents,Extra_x0020_Link,Area,calHtmlColorCode&$filter=Book_x0020_Type%20eq%20%27Silver%20Book%27
+    URL
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub "bibxml", "xml"
+      @files = []
+    end
+
+    def agent
+      return @agent if @agent
+
+      @agent = Mechanize.new
+      @agent.request_headers = { "Accept" => "application/json;odata=verbose" }
+      @agent
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create "CCSDS", file: "index-v1.yaml"
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def fetch
+      fetch_docs ACTIVE_PUBS_URL
+      fetch_docs OBSOLETE_PUBS_URL, retired: true
+      index.save
+    end
+
+    def fetch_docs(url, retired: false)
+      resp = agent.get(url)
+      json = JSON.parse resp.body
+      @array = json["d"]["results"].map do |doc|
+        parse_and_save doc, json["d"]["results"], retired
+      end
+    end
+
+    def parse_and_save(doc, results, retired)
+      bibitem = DataParser.new(doc, results).parse
+      if retired
+        predecessor = DataParser.new(doc, results, bibitem).parse
+        save_bib predecessor
+      end
+      save_bib bibitem
+    end
+
+    def save_bib(bib)
+      id = bib.docidentifier.first.id
+      file = File.join @output, "#{id.gsub(/[.\s]+/, '-')}.#{@ext}"
+      if @files.include? file
+        puts "(#{file}) file already exists. Trying to merge links ..."
+        merge_links bib, file
+      else
+        @files << file
+      end
+      File.write file, content(bib), encoding: "UTF-8"
+      index.add_or_update id, file
+    end
+
+    def merge_links(bib, file) # rubocop:disable Metrics/AbcSize
+      hash = YAML.load_file file
+      bib2 = BibliographicItem.from_hash hash
+      if bib.link[0].type == bib2.link[0].type
+        warn "(#{file}) links are the same."
+        return
+      end
+      warn "(#{file}) links are merged."
+      bib.link << bib2.link[0]
+    end
+
+    def content(bib)
+      case @format
+      when "yaml" then bib.to_hash.to_yaml
+      when "xml" then bib.to_xml(bibdata: true)
+      else bib.send "to_#{@format}"
+      end
+    end
+  end
+end

data/lib/relaton_ccsds/data_parser.rb

@@ -0,0 +1,139 @@
+module RelatonCcsds
+  class DataParser
+    DOCTYPES = { B: "standard", M: "practice", G: "report", O: "specification", Y: "record" }.freeze
+    DOMAIN = "https://public.ccsds.org".freeze
+
+    AREAS = {
+      "SEA" => "Systems Engineering Area",
+      "MOIMS" => "Mission Operations and Information Management Services Area",
+      "CSS" => "Cross Support Services Area",
+      "SOIS" => "Spacecraft Onboard Interface Services Area",
+      "SLS" => "Space Link Services Area",
+      "SIS" => "Space Internetworking Services Area",
+    }.freeze
+
+    ATTRS = %i[
+      title docid abstract doctype date docstatus link edition relation editorialgroup technology_area
+    ].freeze
+
+    ID_MAPPING = {
+      "CCSDS 320.0-B-1-S Corrigendum 1" => "CCSDS 320.0-B-1-S Cor. 1", # TODO relaton/relaton-data-ccsds#5
+      "CCSDS 701.00-R-3" => "CCSDS 701.00-R-3-S", # TODO relaton/relaton-data-ccsds#8
+    }.freeze
+
+    def initialize(doc, docs, successor = nil)
+      @doc = doc
+      @docs = docs
+      @successor = successor
+    end
+
+    def parse
+      args = ATTRS.each_with_object({}) { |a, o| o[a] = send "parse_#{a}" }
+      BibliographicItem.new(**args)
+    end
+
+    def parse_title
+      t = @doc["Dcoument_x0020_Title"]
+      [RelatonBib::TypedTitleString.new(content: t, language: "en", script: "Latn")]
+    end
+
+    def parse_docid
+      [RelatonBib::DocumentIdentifier.new(id: docidentifier, type: "CCSDS", primary: true)]
+    end
+
+    def docidentifier(id = nil)
+      id ||= @doc["Document_x0020_Number"].strip
+      docid = ID_MAPPING[id] || id
+      return docid unless @successor
+
+      docid.sub(/(-S|s)(?=\s|$)/, "")
+    end
+
+    def parse_abstract
+      a = @doc["Description0"]
+      [RelatonBib::FormattedString.new(content: a, language: "en", script: "Latn")]
+    end
+
+    def parse_doctype
+      /^CCSDS\s[\d.]+-(?<type>\w+)/ =~ @doc["Document_x0020_Number"]
+      DOCTYPES[type&.to_sym]
+    end
+
+    def parse_date
+      on = "#{@doc['calPublishedMonth']} #{@doc['calPublishedYear']}"
+      [RelatonBib::BibliographicDate.new(type: "published", on: on)]
+    end
+
+    def parse_docstatus
+      stage = @successor ? "withdrawn" : "published"
+      RelatonBib::DocumentStatus.new stage: stage
+    end
+
+    def parse_link
+      l = "#{DOMAIN}#{@doc['FileRef']}"
+      t = File.extname(@doc["FileRef"])&.sub(/^\./, "")
+      [RelatonBib::TypedUri.new(type: t, content: l)]
+    end
+
+    def parse_edition
+      @doc["Issue_x0020_Number"]
+    end
+
+    def parse_relation
+      @docs.each_with_object(successor + adopted) do |d, a|
+        id = docidentifier d["Document_x0020_Number"].strip
+        type = relaton_type id
+        next unless type
+
+        a << create_relation(type, id)
+      end
+    end
+
+    def adopted
+      return [] unless @doc["ISO_x0020_Number"]
+
+      code = @doc["ISO_x0020_Number"]["Description"].match(/(?<=\s)\d+$/)
+      id = "ISO #{code}"
+      [create_relation("adoptedAs", id)]
+    end
+
+    def successor
+      return [] unless @successor
+
+      @successor.relation << create_relation("successorOf", docidentifier)
+      [create_relation("hasSuccessor", @successor.docidentifier[0].id)]
+    end
+
+    def relaton_type(rel_id)
+      same = rel_id == docidentifier
+      if rel_id.include?(docidentifier) && !same
+        "hasEdition"
+      elsif docidentifier.include?(rel_id) && !same
+        "editionOf"
+      end
+    end
+
+    def create_relation(type, rel_id)
+      id = RelatonBib::DocumentIdentifier.new id: rel_id, type: "CCSDS", primary: true
+      fref = RelatonBib::FormattedRef.new content: rel_id
+      bibitem = RelatonBib::BibliographicItem.new docid: [id], formattedref: fref
+      RelatonBib::DocumentRelation.new type: type, bibitem: bibitem
+    end
+
+    def parse_editorialgroup
+      name = @doc.dig("Working_x0020_Group", "Description")
+      return unless name
+
+      wg = RelatonBib::WorkGroup.new name: name
+      tc = RelatonBib::TechnicalCommittee.new wg
+      RelatonBib::EditorialGroup.new([tc])
+    end
+
+    def parse_technology_area
+      desc = @doc.dig("Working_x0020_Group", "Description")
+      return unless desc
+
+      AREAS[desc.match(/^[A-Z]+(?=-)/)&.to_s]
+    end
+  end
+end

data/lib/relaton_ccsds/hash_converter.rb

@@ -0,0 +1,21 @@
+module RelatonCcsds
+  class HashConverter < RelatonBib::HashConverter
+    class << self
+      # @param args [Hash]
+      # @return [Hash]
+      def hash_to_bib(args)
+        ret = super
+        return unless ret
+
+        ret[:technology_area] = ret[:ext][:technology_area] if ret[:ext]
+        ret
+      end
+
+      # @param item_hash [Hash]
+      # @return [RelatonCie::BibliographicItem]
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+    end
+  end
+end

data/lib/relaton_ccsds/hit.rb

@@ -0,0 +1,21 @@
+module RelatonCcsds
+  class Hit
+    attr_reader :code
+
+    def initialize(code:, url:)
+      @code = code
+      @url = url
+    end
+
+    def doc
+      return @doc if @doc
+
+      resp = Mechanize.new.get(@url)
+      hash = YAML.safe_load(resp.body)
+      hash["fetched"] = Date.today.to_s
+      @doc = BibliographicItem.from_hash(hash)
+    rescue Mechanize::Error => e
+      raise RelatonBib::RequestError, e.message
+    end
+  end
+end

data/lib/relaton_ccsds/hit_collection.rb

@@ -0,0 +1,23 @@
+module RelatonCcsds
+  class HitCollection < RelatonBib::HitCollection
+    GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-ccsds/main/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
+
+    #
+    # Search his in index.
+    #
+    # @return [<Type>] <description>
+    #
+    def fetch
+      rows = index.search text
+      @array = rows.map { |row| Hit.new code: row[:id], url: "#{GHURL}#{row[:file]}" }
+      self
+    rescue SocketError, OpenURI::HTTPError, OpenSSL::SSL::SSLError, Errno::ECONNRESET => e
+      raise RelatonBib::RequestError, e.message
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :ccsds, url: "#{GHURL}index-v1.zip", file: INDEX_FILE
+    end
+  end
+end

data/lib/relaton_ccsds/processor.rb

@@ -0,0 +1,60 @@
+require "relaton/processor"
+
+module RelatonCcsds
+  class Processor < Relaton::Processor
+    attr_reader :idtype
+
+    def initialize # rubocop:disable Lint/MissingSuper
+      @short = :relaton_ccsds
+      @prefix = "CCSDS"
+      @defaultprefix = %r{^CCSDS(?!\w)}
+      @idtype = "CCSDS"
+      @datasets = %w[ccsds]
+    end
+
+    # @param code [String]
+    # @param date [String, NilClass] year
+    # @param opts [Hash]
+    # @return [RelatonCcsds::BibliographicItem]
+    def get(code, date, opts)
+      ::RelatonCcsds::Bibliography.get(code, date, opts)
+    end
+
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
+    # @param xml [String]
+    # @return [RelatonCcsds::CcBibliographicItem]
+    def from_xml(xml)
+      ::RelatonCcsds::XMLParser.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [RelatonIsoBib::CcBibliographicItem]
+    def hash_to_bib(hash)
+      ::RelatonCcsds::BibliographicItem.from_hash hash
+    end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def grammar_hash
+      @grammar_hash ||= ::RelatonCcsds.grammar_hash
+    end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:CC, url: true, file: HitCollection::INDEX_FILE).remove_file
+    end
+  end
+end

data/lib/relaton_ccsds/util.rb

@@ -0,0 +1,9 @@
+module RelatonCcsds
+  module Util
+    extend RelatonBib::Util
+
+    def self.logger
+      RelatonCcsds.configuration.logger
+    end
+  end
+end

data/lib/relaton_ccsds/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RelatonCcsds
+  VERSION = "1.14.0"
+end

data/lib/relaton_ccsds/xml_parser.rb

@@ -0,0 +1,31 @@
+module RelatonCcsds
+  class XMLParser < RelatonBib::XMLParser
+    class << self
+      private
+
+      #
+      # Parse bibitem data
+      #
+      # @param bibitem [Nokogiri::XML::Element] bibitem element
+      #
+      # @return [Hash] bibitem data
+      #
+      def item_data(doc)
+        resp = super
+        resp[:technology_area] = doc.at("./ext/technology-area")&.text
+        resp
+      end
+
+      #
+      # override RelatonBib::XMLParser#bib_item method
+      #
+      # @param item_hash [Hash]
+      #
+      # @return [RelatonCcsds::BibliographicItem]
+      #
+      def bib_item(item_hash)
+        BibliographicItem.new(**item_hash)
+      end
+    end
+  end
+end

data/lib/relaton_ccsds.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "mechanize"
+require "relaton_bib"
+require "relaton/index"
+require_relative "relaton_ccsds/version"
+require_relative "relaton_ccsds/bibliographic_item"
+require_relative "relaton_ccsds/config"
+require_relative "relaton_ccsds/util"
+require_relative "relaton_ccsds/bibliography"
+require_relative "relaton_ccsds/hit"
+require_relative "relaton_ccsds/hit_collection"
+require_relative "relaton_ccsds/data_fetcher"
+require_relative "relaton_ccsds/data_parser"
+require_relative "relaton_ccsds/hash_converter"
+require_relative "relaton_ccsds/xml_parser"
+
+module RelatonCcsds
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.grammar_hash
+    gem_path = File.expand_path "..", __dir__
+    grammars_path = File.join gem_path, "grammars", "*"
+    grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+    Digest::MD5.hexdigest grammars
+  end
+end

data/relaton_ccsds.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "lib/relaton_ccsds/version"
+
+Gem::Specification.new do |spec|
+  spec.name         = "relaton-ccsds"
+  spec.version      = RelatonCcsds::VERSION
+  spec.authors      = ["Ribose Inc."]
+  spec.email        = ["open.source@ribose.com"]
+
+  spec.summary      = "RelatonCcsds: retrive www.ccsds.org Standards"
+  spec.description  = "RelatonCcsds: retrive www.ccsds.org Standards"
+  spec.homepage     = "https://github.com/metanorma/relaton-ccsds"
+  spec.license      = "MIT"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 2.7.0"
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_dependency "mechanize", "~> 2.7"
+  spec.add_dependency "relaton-bib", "~> 1.14.13"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: relaton-ccsds
+version: !ruby/object:Gem::Version
+  version: 1.14.0
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2023-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: relaton-bib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.14.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.14.13
+- !ruby/object:Gem::Dependency
+  name: relaton-index
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+description: 'RelatonCcsds: retrive www.ccsds.org Standards'
+email:
+- open.source@ribose.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/rake.yml"
+- ".github/workflows/release.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".vscode/launch.json"
+- Gemfile
+- README.adoc
+- Rakefile
+- bin/console
+- bin/rspec
+- bin/setup
+- grammars/basicdoc.rng
+- grammars/biblio-standoc.rng
+- grammars/biblio.rng
+- lib/relaton_ccsds.rb
+- lib/relaton_ccsds/bibliographic_item.rb
+- lib/relaton_ccsds/bibliography.rb
+- lib/relaton_ccsds/config.rb
+- lib/relaton_ccsds/data_fetcher.rb
+- lib/relaton_ccsds/data_parser.rb
+- lib/relaton_ccsds/hash_converter.rb
+- lib/relaton_ccsds/hit.rb
+- lib/relaton_ccsds/hit_collection.rb
+- lib/relaton_ccsds/processor.rb
+- lib/relaton_ccsds/util.rb
+- lib/relaton_ccsds/version.rb
+- lib/relaton_ccsds/xml_parser.rb
+- relaton_ccsds.gemspec
+homepage: https://github.com/metanorma/relaton-ccsds
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key: 
+specification_version: 4
+summary: 'RelatonCcsds: retrive www.ccsds.org Standards'
+test_files: []