relaton-ogc 1.9.1 → 1.10.1

data/grammars/ogc.rng

@@ -62,6 +62,19 @@
         </optional>
       </element>
     </define>
+    <define name="DocumentSubtype">
+      <choice>
+        <value>conceptual-model</value>
+        <value>conceptual-model-and-encoding</value>
+        <value>conceptual-model-and-implementation</value>
+        <value>encoding</value>
+        <value>extension</value>
+        <value>implementation</value>
+        <value>profile</value>
+        <value>profile-with-extension</value>
+        <value>general</value>
+      </choice>
+    </define>
   </include>
   <define name="TextElement" combine="choice">
     <ref name="hi"/>
@@ -79,19 +92,6 @@
       </zeroOrMore>
     </element>
   </define>
-  <define name="DocumentSubtype">
-    <choice>
-      <value>conceptual-model</value>
-      <value>conceptual-model-and-encoding</value>
-      <value>conceptual-model-and-implementation</value>
-      <value>encoding</value>
-      <value>extension</value>
-      <value>implementation</value>
-      <value>profile</value>
-      <value>profile-with-extension</value>
-      <value>general</value>
-    </choice>
-  </define>
   <define name="submitters">
     <element name="submitters">
       <ref name="Basic-Section"/>

data/grammars/reqt.rng

@@ -64,9 +64,9 @@
     <optional>
       <ref name="label"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="subject"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="reqinherit"/>
     </zeroOrMore>
@@ -80,6 +80,7 @@
         <ref name="verification"/>
         <ref name="import"/>
         <ref name="description"/>
+        <ref name="component"/>
       </choice>
     </zeroOrMore>
     <optional>
@@ -105,12 +106,16 @@
   </define>
   <define name="subject">
     <element name="subject">
-      <text/>
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
     </element>
   </define>
   <define name="reqinherit">
     <element name="inherit">
-      <text/>
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
     </element>
   </define>
   <define name="measurementtarget">
@@ -138,6 +143,12 @@
       <ref name="RequirementSubpart"/>
     </element>
   </define>
+  <define name="component">
+    <element name="component">
+      <attribute name="class"/>
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
   <define name="reqt_references">
     <element name="references">
       <oneOrMore>

data/lib/relaton_ogc/data_fetcher.rb

@@ -0,0 +1,98 @@
+module RelatonOgc
+  class DataFetcher
+    module Utils
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+                 "NamingAuthority/master/incubation/bibliography/"\
+                 "bibliography.json".freeze
+
+      def get_data # rubocop:disable Metrics/AbcSize
+        h = {}
+        h["If-None-Match"] = etag if etag
+        resp = Faraday.new(ENDPOINT, headers: h).get
+        case resp.status
+        when 200
+          json = JSON.parse(resp.body)
+          block_given? ? yield(resp[:etag], json) : json
+        when 304 then [] # there aren't any changes since last fetching
+        else raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
+        end
+      end
+
+      #
+      # Read ETag form file
+      #
+      # @return [String, NilClass]
+      def etag
+        @etag ||= if File.exist? @etagfile
+                    File.read @etagfile, encoding: "UTF-8"
+                  end
+      end
+
+      #
+      # Save ETag to file
+      #
+      # @param tag [String]
+      def etag=(e_tag)
+        File.write @etagfile, e_tag, encoding: "UTF-8"
+      end
+    end
+
+    include Utils
+
+    #
+    # Create DataFetcher instance
+    #
+    # @param [String] output directory to save the documents
+    # @param [String] format output format "yaml" or "xmo"
+    #
+    def initialize(output, format)
+      @output = output
+      @etagfile = File.join output, "etag.txt"
+      @format = format
+      @docids = []
+      @dupids = []
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def fetch # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      get_data do |etag, json|
+        no_errors = true
+        json.each do |_, hit|
+          next if hit["type"] == "CC"
+
+          bib = Scrapper.parse_page hit
+          write_document bib
+        rescue StandardError => e
+          no_errors = false
+          warn "Fetching document: #{hit['identifier']}"
+          warn "#{e.class} #{e.message}"
+          warn e.backtrace
+        end
+        warn "[relaton-ogc] WARNING Duplicated documents: #{@dupids.uniq.join(', ')}" if @dupids.any?
+        self.etag = etag if no_errors
+      end
+    end
+
+    def write_document(bib) # rubocop:disable Metrics/AbcSize
+      if @docids.include?(bib.docidentifier[0].id)
+        @dupids << bib.docidentifier[0].id
+        return
+      end
+
+      @docids << bib.docidentifier[0].id
+      name = bib.docidentifier[0].id.upcase.gsub(/[\s:.]/, "_")
+      file = "#{@output}/#{name}.#{@format}"
+      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file, content, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton_ogc/hit.rb

@@ -1,9 +1,20 @@
 module RelatonOgc
   class Hit < RelatonBib::Hit
+    #
+    # <Description>
+    #
+    # @param [RelatonOgc::OgcBibliographicItem] bibitem
+    # @param [RelatonOgc::HitCollection, nil] hitcoll
+    #
+    def initialize(bibitem, hitcoll = nil)
+      super({ id: bibitem.docidentifier[0].id}, hitcoll)
+      @fetch = bibitem
+    end
+
     # Parse page.
     # @return [RelatonNist::NistBliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page @hit
+      @fetch # ||= Scrapper.parse_page @hit
     end
   end
 end

data/lib/relaton_ogc/hit_collection.rb

@@ -4,86 +4,71 @@ require "fileutils"
 
 module RelatonOgc
   class HitCollection < RelatonBib::HitCollection
-    ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
-      "NamingAuthority/master/incubation/bibliography/bibliography.json".freeze
-    DATADIR = File.expand_path ".relaton/ogc/", Dir.home
-    DATAFILE = File.expand_path "bibliography.json", DATADIR
-    ETAGFILE = File.expand_path "etag.txt", DATADIR
+    # include DataFetcher::Utils
 
-    # @param ref [Strig]
+    # ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+    #            "NamingAuthority/master/incubation/bibliography/"\
+    #            "bibliography.json".freeze
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ogc/main/data/".freeze
+    # DATADIR = File.expand_path ".relaton/ogc/", Dir.home
+    # DATAFILE = File.expand_path "bibliography.json", DATADIR
+    # ETAGFILE = File.expand_path "etag.txt", DATADIR
+
+    # @param code [Strig]
     # @param year [String]
     # @param opts [Hash]
-    def initialize(ref, year = nil)
+    def initialize(code, year = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       super
-      @array = from_json(ref).sort_by do |hit|
-        begin
-          hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
-        rescue ArgumentError
-          Date.parse "0000-01-01"
-        end
-      end.reverse
+      # @etagfile = File.expand_path "etag.txt", DATADIR
+      # @array = from_json(ref).sort_by do |hit|
+      #   hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
+      # rescue ArgumentError
+      #   Date.parse "0000-01-01"
+      # end.reverse
+      resp = Faraday.get "#{ENDPOINT}#{code.upcase.gsub(/[\s:.]/, '_')}.yaml"
+      @array = case resp.status
+               when 200
+                 bib = OgcBibliographicItem.from_hash YAML.safe_load(resp.body)
+                 [Hit.new(bib, self)]
+               else []
+               end
     end
 
-    private
+    # private
 
     #
     # Fetch data form json
     #
     # @param docid [String]
-    def from_json(docid, **_opts)
-      ref = docid.sub(/^OGC\s/, "").strip
-      return [] if ref.empty?
+    # def from_json(docid, **_opts)
+    #   ref = docid.sub(/^OGC\s/, "").strip
+    #   return [] if ref.empty?
 
-      data.select do |_k, doc|
-        doc["type"] != "CC" && doc["identifier"].include?(ref)
-      end.map { |_k, h| Hit.new(h, self) }
-    end
+    #   data.select do |_k, doc|
+    #     doc["type"] != "CC" && doc["identifier"].include?(ref)
+    #   end.map { |_k, h| Hit.new(h, self) }
+    # end
 
     #
     # Fetches json data
     #
     # @return [Hash]
-    def data
-      ctime = File.ctime DATAFILE if File.exist? DATAFILE
-      fetch_data if !ctime || ctime.to_date < Date.today
-      @data ||= JSON.parse File.read(DATAFILE, encoding: "UTF-8")
-    end
+    # def data
+    #   ctime = File.ctime DATAFILE if File.exist? DATAFILE
+    #   fetch_data if !ctime || ctime.to_date < Date.today
+    #   @data ||= JSON.parse File.read(DATAFILE, encoding: "UTF-8")
+    # end
 
     #
     # fetch data form server and save it to file.
     #
-    def fetch_data # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      h = {}
-      h["If-None-Match"] = etag if etag
-      resp = Faraday.new(ENDPOINT, headers: h).get
-      # return if there aren't any changes since last fetching
-      return if resp.status == 304
-      unless resp.status == 200
-        raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
-      end
-
-      FileUtils.mkdir_p DATADIR unless Dir.exist? DATADIR
-      self.etag = resp[:etag]
-      @data = JSON.parse resp.body
-      File.write DATAFILE, @data.to_json, encoding: "UTF-8"
-    end
-
-    #
-    # Read ETag form file
-    #
-    # @return [String, NilClass]
-    def etag
-      @etag ||= if File.exist? ETAGFILE
-                  File.read ETAGFILE, encoding: "UTF-8"
-                end
-    end
+    # def fetch_data
+    #   json = get_data
+    #   return unless json
 
-    #
-    # Save ETag to file
-    #
-    # @param tag [String]
-    def etag=(e_tag)
-      File.write ETAGFILE, e_tag, encoding: "UTF-8"
-    end
+    #   FileUtils.mkdir_p DATADIR unless Dir.exist? DATADIR
+    #   @data = json
+    #   File.write DATAFILE, @data.to_json, encoding: "UTF-8"
+    # end
   end
 end

data/lib/relaton_ogc/ogc_bibliography.rb

@@ -4,7 +4,8 @@ module RelatonOgc
       # @param text [String]
       # @return [RelatonOgc::HitCollection]
       def search(text, year = nil, _opts = {})
-        HitCollection.new text, year
+        code = text.sub(/^OGC\s/, "")
+        HitCollection.new code, year
       rescue Faraday::ConnectionFailed
         raise RelatonBib::RequestError, HitCollection::ENDPOINT
       end

data/lib/relaton_ogc/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonOgc
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ogc
       @prefix = "OGC"
       @defaultprefix = %r{^OGC\s}
       @idtype = "OGC"
+      @datasets = %w[ogc-naming-authority]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonOgc
       ::RelatonOgc::OgcBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonOgc::OgcBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ogc/scrapper.rb

@@ -13,7 +13,7 @@ module RelatonOgc
       "IPR" => { type: "engineering-report" },
       "IS" => { type: "standard", subtype: "implementation" },
       "ISC" => { type: "standard", subtype: "implementation" },
-      "ISx" => { type: "standard", subtype: "extesion" },
+      "ISx" => { type: "standard", subtype: "extension" },
       "Notes" => { type: "other" },
       "ORM" => { type: "reference-model" },
       "PC" => { type: "standard", subtype: "profile" },
@@ -70,7 +70,7 @@ module RelatonOgc
       # @param identifier [String]
       # @return [Array<RelatonBib::DocumentIdentifier>]
       def fetch_docid(identifier)
-        [RelatonBib::DocumentIdentifier.new(id: identifier, type: "OGC")]
+        [RelatonBib::DocumentIdentifier.new(id: identifier, type: "OGC", primary: true)]
       end
 
       # @param url [String]
@@ -88,7 +88,7 @@ module RelatonOgc
       # @param stage [String]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(stage)
-        stage && RelatonBib::DocunentStatus.new(stage: stage)
+        stage && RelatonBib::DocumentStatus.new(stage: stage)
       end
 
       # @param identifier [String]
@@ -138,6 +138,8 @@ module RelatonOgc
       # @param date [String]
       # @return [Array<RelatonBib::BibliographicDate>]
       def fetch_date(date)
+        return [] unless date
+
         [RelatonBib::BibliographicDate.new(type: "published", on: date)]
       end
     end

data/lib/relaton_ogc/version.rb

@@ -1,3 +1,3 @@
 module RelatonOgc
-  VERSION = "1.9.1".freeze
+  VERSION = "1.10.1".freeze
 end

data/lib/relaton_ogc.rb

@@ -2,6 +2,7 @@ require "relaton_iso_bib"
 require "relaton_ogc/version"
 require "relaton_ogc/ogc_bibliographic_item"
 require "relaton_ogc/ogc_bibliography"
+require "relaton_ogc/data_fetcher"
 require "relaton_ogc/hit_collection"
 require "relaton_ogc/scrapper"
 require "relaton_ogc/xml_parser"

data/relaton_ogc.gemspec

@@ -35,5 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock"
 
   spec.add_dependency "faraday", "~> 1.1"
-  spec.add_dependency "relaton-iso-bib", "~> 1.9.0"
+  spec.add_dependency "relaton-iso-bib", "~> 1.10.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ogc
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.10.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-07 00:00:00.000000000 Z
+date: 2022-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.10.1
 description: 'RelatonOgc: retrieve OGC Standards for bibliographic use using the OgcBibliographicItem
   model'
 email:
@@ -175,6 +175,7 @@ files:
 - grammars/ogc.rng
 - grammars/reqt.rng
 - lib/relaton_ogc.rb
+- lib/relaton_ogc/data_fetcher.rb
 - lib/relaton_ogc/editorial_group.rb
 - lib/relaton_ogc/hash_converter.rb
 - lib/relaton_ogc/hit.rb