relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/oasis/data_part_parser.rb

@@ -0,0 +1,239 @@
+module Relaton
+  module Oasis
+    # Parser for OASIS part document.
+    class DataPartParser
+      include DataParserUtils
+
+      #
+      # Initialize parser.
+      #
+      # @param [Nokogiri::HTML::Element] node document node
+      #
+      def initialize(node, errors = {}, agent: nil)
+        @node = node
+        @errors = errors
+        @agent = agent
+      end
+
+      def text
+        return @text if @text
+
+        sibling_xpath = "./strong/following-sibling::text()" \
+                        "|./span[strong]/following-sibling::text()"
+        if @node.at(sibling_xpath)
+          nodes_xpath = "./strong/following-sibling::node()" \
+                        "|./span[strong]/following-sibling::node()"
+          @text = @node.xpath(nodes_xpath).text.strip
+        else
+          @text = @node.xpath("./following-sibling::p[1][em]").text.strip
+        end
+      end
+
+      def title
+        return @title if @title
+
+        xpath = "./span[@class='citationTitle' " \
+                "or @class='citeTitle']|./em|./i"
+        t = @node.at(xpath)
+        @title = if t
+                   t.text
+                 else
+                   m = text.match(/(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/)
+                   m ? m[:content] : text
+                 end.strip
+      end
+
+      #
+      # Parse document.
+      #
+      # @return [ItemData] bibliographic item data
+      #
+      def parse # rubocop:disable Metrics/MethodLength
+        ItemData.new(
+          type: "standard",
+          title: parse_title,
+          docidentifier: parse_docid,
+          source: parse_link,
+          docnumber: parse_docnumber,
+          date: parse_date,
+          abstract: parse_abstract,
+          language: ["en"],
+          script: ["Latn"],
+          relation: parse_relation,
+          contributor: parse_contributor,
+          ext: create_ext,
+        )
+      end
+
+      #
+      # Parse title.
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title
+        result = [Bib::Title.new(type: "main", content: title,
+                                 language: "en", script: "Latn")]
+        @errors[:part_title] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse document number.
+      #
+      # @return [String] document number
+      #
+      def parse_docnumber
+        ref = @node.at("./span/strong|./strong|./b/span")
+        num = ref.text.match(/[^\[\]]+/).to_s
+        id = parse_errata(num)
+        # some part refs need "Pt" to distinguish from root doc
+        id += "-Pt" if %w[CMIS-v1.1 DocBook-5.0 XACML-V3.0 mqtt-v3.1.1
+                          OData-JSON-Format-v4.0].include?(id)
+        result = parse_part parse_spec id
+        @errors[:part_docnumber] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse link.
+      #
+      # @return [Array<Bib::Uri>] link
+      #
+      def parse_link
+        result = [Bib::Uri.new(type: "src", content: link_node[:href])]
+        @errors[:part_link] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse date.
+      #
+      # @return [Array<Bib::Date>] bibliographic dates
+      #
+      def parse_date
+        match = text.match(/(?<on>\d{1,2}\s\w+\s\d{4})/)
+        result = match ? [Bib::Date.new(at: Date.parse(match[:on]).to_s, type: "issued")] : []
+        @errors[:part_date] &&= result.empty?
+        result
+      end
+
+      def parse_abstract # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Abstract')]][1]"
+                   page.xpath(xpath).map do |p|
+                     cnt = p.text.gsub(/[\r\n]+/, " ").strip
+                     Bib::Abstract.new(
+                       content: cnt, language: "en", script: "Latn",
+                     )
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_abstract] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse editorial group as contributors.
+      #
+      # @return [Array<Bib::Contributor>] editorial group contributors
+      #
+      def parse_editorialgroup_contributor # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Technical')]][1]//a"
+                   tcs = page.xpath(xpath)
+                   if tcs.empty?
+                     []
+                   else
+                     subdivisions = tcs.map do |a|
+                       name = [Bib::TypedLocalizedString.new(
+                         content: a.text.strip,
+                       )]
+                       Bib::Subdivision.new(
+                         type: "technical-committee", name: name,
+                       )
+                     end
+                     org = Bib::Organization.new(
+                       name: [Bib::TypedLocalizedString.new(
+                         content: "OASIS",
+                       )],
+                       subdivision: subdivisions,
+                     )
+                     desc = [Bib::LocalizedMarkedUpString.new(
+                       content: "committee",
+                     )]
+                     role = Bib::Contributor::Role.new(
+                       type: "author", description: desc,
+                     )
+                     [Bib::Contributor.new(organization: org,
+                                           role: [role])]
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_editorialgroup_contributor] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse relation.
+      #
+      # @return [Array<Bib::Relation>] document relations
+      #
+      def parse_relation
+        parser = DataParser.new(@node.at("./ancestor::details"), @errors, agent: @agent)
+        fref = parser.parse_docid[0].content
+        bib = ItemData.new(formattedref: Bib::Formattedref.new(content: fref))
+        result = [Bib::Relation.new(type: "partOf", bibitem: bib)]
+        @errors[:part_relation] &&= result.empty?
+        result
+      end
+
+      def parse_authorizer # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Technical')]][1]//a"
+                   page.xpath(xpath).map do |a|
+                     org_name = a.text.gsub(/[\r\n]+/, " ").strip
+                     org = Bib::Organization.new(
+                       name: [Bib::TypedLocalizedString.new(
+                         content: org_name,
+                       )],
+                       uri: [Bib::Uri.new(type: "uri",
+                                          content: a[:href])],
+                     )
+                     desc = [Bib::LocalizedMarkedUpString.new(
+                       content: "Committee",
+                     )]
+                     role = Bib::Contributor::Role.new(
+                       type: "authorizer", description: desc,
+                     )
+                     Bib::Contributor.new(organization: org,
+                                          role: [role])
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_authorizer] &&= result.empty?
+        result
+      end
+
+      def link_node
+        @link_node = @node.at("./a|./following-sibling::p[1]/a")
+      end
+
+      #
+      # Parse technology area.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area
+        result = super(@node.at("./ancestor::details"))
+        @errors[:part_technology_area] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/oasis/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Oasis
+    class Doctype <  Bib::Doctype
+      attribute :content, :string, values: %w[specification memorandum resolution standard]
+    end
+  end
+end

data/lib/relaton/oasis/ext.rb

@@ -0,0 +1,19 @@
+require_relative "doctype"
+
+module Relaton
+  module Oasis
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :technology_area, :string, collection: true, values: %w[
+        Cloud Content-Technologies Cybersecurity e-Business eGov/Legal Emergency-Management
+        Energy Information-Modeling IoT Lifecycle-Integration Localization Messaging
+        Privacy/Identity Security SOA Web-Services Software-Development Virtualization
+      ]
+
+      xml { map_element "technology-area", to: :technology_area }
+      key_value { map "technology_area", to: :technology_area }
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-oasis"]
+    end
+  end
+end

data/lib/relaton/oasis/item.rb

@@ -0,0 +1,11 @@
+require_relative "ext"
+
+module Relaton
+  module Oasis
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/oasis/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Oasis
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/oasis/processor.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Oasis
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_oasis
+        @prefix = "OASIS"
+        @defaultprefix = %r{^OASIS\s}
+        @idtype = "OASIS"
+        @datasets = %w[oasis-open]
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Oasis::Item]
+      def get(code, date, opts)
+        require_relative "../oasis"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from https://www.oasis-open.org/standards/
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Oasis::Item]
+      def from_xml(xml)
+        require_relative "../oasis"
+        Item.from_xml(xml)
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Oasis::Item]
+      def from_yaml(yaml)
+        require_relative "../oasis"
+        Item.from_yaml(yaml)
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../oasis"
+        @grammar_hash ||= Relaton::Oasis.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../oasis"
+        Relaton::Index.find_or_create(:oasis, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/oasis/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-oasis".freeze
+    end
+  end
+end

data/lib/relaton/oasis.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "oasis/util"
+require_relative "oasis/item_data"
+require_relative "oasis/item"
+require_relative "oasis/bibitem"
+require_relative "oasis/bibdata"
+require_relative "oasis/bibliography"
+
+module Relaton
+  module Oasis
+    INDEXFILE = "index-v1"
+    class Error < StandardError; end
+    # Your code goes here...
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/ogc/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ogc
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/ogc/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ogc
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/ogc/bibliography.rb

@@ -0,0 +1,64 @@
+module Relaton
+  module Ogc
+    module Bibliography
+      extend self
+
+      # @param text [String]
+      # @param year [String, nil]
+      # @return [Relaton::Ogc::HitCollection]
+      def search(text, year = nil, _opts = {})
+        code = text.sub(/^OGC\s/, "")
+        HitCollection.new(code, year).find
+      rescue Faraday::ConnectionFailed, Faraday::SSLError
+        raise Relaton::RequestError, HitCollection::ENDPOINT
+      end
+
+      # @param code [String]
+      # @param year [String, nil]
+      # @param opts [Hash]
+      # @return [Relaton::Ogc::ItemData, nil]
+      def get(code, year = nil, opts = {})
+        result = bib_search_filter(code, year, opts) || (return nil)
+        ret = bib_results_filter(result, year)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: code
+          ret[:ret]
+        else
+          fetch_ref_err(code, year, ret[:years])
+        end
+      end
+
+      private
+
+      def bib_search_filter(code, year, opts)
+        Util.info "Fetching from Relaton repository ...", key: code
+        search(code, year, opts)
+      end
+
+      def bib_results_filter(result, year)
+        missed_years = []
+        result.each do |r|
+          item = r.item
+          return { ret: item } unless year
+
+          item.date.select { |d| d.type == "published" }.each do |d|
+            date_year = ::Date.parse(d.at.to_s).year
+            return { ret: item } if year.to_i == date_year
+
+            missed_years << date_year
+          end
+        end
+        { years: missed_years }
+      end
+
+      def fetch_ref_err(code, year, missed_years)
+        Util.info "Not found.", key: code
+        unless missed_years.empty?
+          Util.info "There was no match for `#{year}`, though there " \
+                    "were matches found for `#{missed_years.join('`, `')}`.", key: code
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/ogc/data_fetcher.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "faraday"
+require "json"
+require_relative "../ogc"
+require_relative "scraper"
+
+module Relaton
+  module Ogc
+    class DataFetcher < Core::DataFetcher
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/NamingAuthority/master/definitions/docs/docs.json"
+
+      def initialize(output, format)
+        super
+        @etagfile = File.join output, "etag.txt"
+        @docids = []
+        @dupids = Set.new
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create :ogc, file: "#{INDEXFILE}.yaml"
+      end
+
+      def fetch(_source = nil) # rubocop:disable Metrics/AbcSize
+        get_data do |etag, json|
+          no_errors = true
+          json.each_value { |hit| fetch_doc(hit) || no_errors = false }
+          if @dupids.any?
+            Util.warn "Duplicated documents: #{@dupids.to_a.join(', ')}"
+          end
+          self.etag = etag if no_errors
+          index.save
+          report_errors
+        end
+      end
+
+      def fetch_doc(hit)
+        return if hit["type"] == "CC"
+
+        bib = Scraper.parse_page hit, @errors
+        write_document bib
+        true
+      rescue StandardError => e
+        Util.error "Fetching document: #{hit['identifier']}\n" \
+                   "#{e.class} #{e.message}\n#{e.backtrace}"
+        false
+      end
+
+      def write_document(bib) # rubocop:disable Metrics/AbcSize
+        docid = bib.docidentifier[0].content
+        if @docids.include?(docid)
+          @dupids << docid
+          return
+        end
+
+        @docids << docid
+        file = file_name bib
+        index.add_or_update docid, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      def file_name(bib)
+        name = bib.docidentifier[0].content.upcase.gsub(/[\s:.]/, "_")
+        "#{@output}/#{name}.#{@ext}"
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml bib
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml bib
+      end
+
+      def to_bibxml(_bib)
+        raise NotImplementedError, "OGC does not support bibxml format"
+      end
+
+      # @return [String, nil]
+      def etag
+        @etag ||= if File.exist? @etagfile
+                    File.read @etagfile, encoding: "UTF-8"
+                  end
+      end
+
+      # @param e_tag [String]
+      def etag=(e_tag)
+        File.write @etagfile, e_tag, encoding: "UTF-8"
+      end
+
+      def get_data # rubocop:disable Metrics/AbcSize
+        h = {}
+        h["If-None-Match"] = etag if etag
+        resp = Faraday.new(ENDPOINT, headers: h).get
+        case resp.status
+        when 200
+          json = JSON.parse(resp.body)
+          block_given? ? yield(resp[:etag], json) : json
+        when 304 then []
+        else raise Relaton::RequestError, "Could not access #{ENDPOINT}"
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ogc/docidentifier.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Ogc
+    class Docidentifier < Bib::Docidentifier
+    end
+  end
+end

data/lib/relaton/ogc/doctype.rb

@@ -0,0 +1,12 @@
+module Relaton
+  module Ogc
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[
+        abstract-specification-topic best-practice change-request-supporting-document
+        community-practice community-standard discussion-paper engineering-report
+        other policy reference-model release-notes standard user-guide white-paper
+        test-suite draft-standard
+      ]
+    end
+  end
+end

data/lib/relaton/ogc/ext.rb

@@ -0,0 +1,15 @@
+require_relative "doctype"
+
+module Relaton
+  module Ogc
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :subdoctype, :string, values: %w[
+        conceptual-model conceptual-model-and-encoding conceptual-model-and-implementation
+        encoding extension implementation profile profile-with-extension general
+      ]
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-ogc"]
+    end
+  end
+end

data/lib/relaton/ogc/hit.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Ogc
+    class Hit < Core::Hit
+      # @return [Relaton::Ogc::ItemData]
+      def item
+        @item ||= begin
+          url = "#{HitCollection::ENDPOINT}#{hit[:file]}"
+          resp = Faraday.get(url) { |req| req.options.timeout = 10 }
+          return unless resp.status == 200
+
+          item = Item.from_yaml resp.body
+          item.fetched = Date.today.to_s
+          item
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ogc/hit_collection.rb

@@ -0,0 +1,36 @@
+require "faraday"
+require_relative "hit"
+
+module Relaton
+  module Ogc
+    class HitCollection < Core::HitCollection
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ogc/v2/".freeze
+
+      # @return [self]
+      def find
+        return self if ref.nil? || ref.empty?
+
+        row = index.search(ref).min_by { |r| r[:id] }
+        return self unless row
+
+        url = "#{ENDPOINT}#{row[:file]}"
+        resp = Faraday.get(url) { |req| req.options.timeout = 10 }
+        return self unless resp.status == 200
+
+        item = Item.from_yaml resp.body
+        item.fetched = Date.today.to_s
+        hit = Hit.new({ code: item.docidentifier[0]&.content, file: row[:file] }, self)
+        hit.instance_variable_set(:@item, item)
+        @array = [hit]
+        self
+      end
+
+      # @return [Relaton::Index]
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :ogc, url: "#{ENDPOINT}index-v1.zip", file: "#{INDEXFILE}.yaml",
+        )
+      end
+    end
+  end
+end