relaton-ogc 1.7.3 → 1.9.2

data/grammars/ogc.rng

@@ -63,21 +63,35 @@
       </element>
     </define>
   </include>
-  <define name="docsubtype">
-    <element name="docsubtype">
-      <choice>
-        <value>conceptual-model</value>
-        <value>conceptual-model-and-encoding</value>
-        <value>conceptual-model-and-implementation</value>
-        <value>encoding</value>
-        <value>extension</value>
-        <value>implementation</value>
-        <value>profile</value>
-        <value>profile-with-extension</value>
-        <value>general</value>
-      </choice>
+  <define name="TextElement" combine="choice">
+    <ref name="hi"/>
+  </define>
+  <define name="PureTextElement" combine="choice">
+    <ref name="hi"/>
+  </define>
+  <define name="hi">
+    <element name="hi">
+      <zeroOrMore>
+        <choice>
+          <ref name="PureTextElement"/>
+          <ref name="stem"/>
+        </choice>
+      </zeroOrMore>
     </element>
   </define>
+  <define name="DocumentSubtype">
+    <choice>
+      <value>conceptual-model</value>
+      <value>conceptual-model-and-encoding</value>
+      <value>conceptual-model-and-implementation</value>
+      <value>encoding</value>
+      <value>extension</value>
+      <value>implementation</value>
+      <value>profile</value>
+      <value>profile-with-extension</value>
+      <value>general</value>
+    </choice>
+  </define>
   <define name="submitters">
     <element name="submitters">
       <ref name="Basic-Section"/>
@@ -115,6 +129,9 @@
       <zeroOrMore>
         <ref name="termdocsource"/>
       </zeroOrMore>
+      <optional>
+        <ref name="misccontainer"/>
+      </optional>
       <optional>
         <ref name="boilerplate"/>
       </optional>
@@ -126,6 +143,9 @@
         <ref name="annex"/>
       </zeroOrMore>
       <ref name="bibliography"/>
+      <zeroOrMore>
+        <ref name="indexsect"/>
+      </zeroOrMore>
     </element>
   </define>
 </grammar>

data/grammars/reqt.rng

@@ -30,15 +30,34 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
     <optional>
       <attribute name="subsequence"/>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <attribute name="id">
       <data type="ID"/>
     </attribute>
     <optional>
       <attribute name="filename"/>
     </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
     <optional>
       <ref name="reqtitle"/>
     </optional>
@@ -48,9 +67,9 @@
     <optional>
       <ref name="subject"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="reqinherit"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="classification"/>
     </zeroOrMore>
@@ -135,6 +154,16 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <oneOrMore>
       <ref name="BasicBlock"/>
     </oneOrMore>

data/lib/relaton_ogc/data_fetcher.rb

@@ -0,0 +1,96 @@
+module RelatonOgc
+  class DataFetcher
+    module Utils
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+                 "NamingAuthority/master/incubation/bibliography/"\
+                 "bibliography.json".freeze
+
+      def get_data # rubocop:disable Metrics/AbcSize
+        h = {}
+        h["If-None-Match"] = etag if etag
+        resp = Faraday.new(ENDPOINT, headers: h).get
+        case resp.status
+        when 200
+          json = JSON.parse(resp.body)
+          block_given? ? yield(resp[:etag], json) : json
+        when 304 then [] # there aren't any changes since last fetching
+        else raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
+        end
+      end
+
+      #
+      # Read ETag form file
+      #
+      # @return [String, NilClass]
+      def etag
+        @etag ||= if File.exist? @etagfile
+                    File.read @etagfile, encoding: "UTF-8"
+                  end
+      end
+
+      #
+      # Save ETag to file
+      #
+      # @param tag [String]
+      def etag=(e_tag)
+        File.write @etagfile, e_tag, encoding: "UTF-8"
+      end
+    end
+
+    include Utils
+
+    #
+    # Create DataFetcher instance
+    #
+    # @param [String] output directory to save the documents
+    # @param [String] format output format "yaml" or "xmo"
+    #
+    def initialize(output, format)
+      @output = output
+      @etagfile = File.join output, "etag.txt"
+      @format = format
+      @docids = []
+      @dupids = []
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def fetch # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      get_data do |etag, json|
+        no_errors = true
+        json.each do |_, hit|
+          bib = Scrapper.parse_page hit
+          write_document bib
+        rescue StandardError => e
+          no_errors = false
+          warn "Fetching document: #{hit['identifier']}"
+          warn "#{e.class} #{e.message}"
+          warn e.backtrace
+        end
+        warn "[relaton-ogc] WARNING Duplicated documents: #{@dupids.uniq.join(', ')}" if @dupids.any?
+        self.etag = etag if no_errors
+      end
+    end
+
+    def write_document(bib) # rubocop:disable Metrics/AbcSize
+      if @docids.include?(bib.docidentifier[0].id)
+        @dupids << bib.docidentifier[0].id
+        return
+      end
+
+      @docids << bib.docidentifier[0].id
+      name = bib.docidentifier[0].id.upcase.gsub(/[\s:.]/, "_")
+      file = "#{@output}/#{name}.#{@format}"
+      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file, content, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton_ogc/editorial_group.rb

@@ -74,7 +74,7 @@ module RelatonOgc
     # @return {RelatonIsoBib::IsoSubgroup}
     def subgroup(group)
       if group.is_a?(Hash)
-        RelatonIsoBib::IsoSubgroup.new(**group)
+        RelatonBib::WorkGroup.new(**group)
       else group
       end
     end

data/lib/relaton_ogc/hash_converter.rb

@@ -12,7 +12,7 @@ module RelatonOgc
           committee: eg[:committee],
           subcommittee: eg[:subcommittee],
           workgroup: eg[:workgroup],
-          secretariat: eg[:secretariat]
+          secretariat: eg[:secretariat],
         )
       end
     end

data/lib/relaton_ogc/hit_collection.rb

@@ -4,23 +4,25 @@ require "fileutils"
 
 module RelatonOgc
   class HitCollection < RelatonBib::HitCollection
-    ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
-      "NamingAuthority/master/incubation/bibliography/bibliography.json".freeze
+    include DataFetcher::Utils
+
+    # ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+    #            "NamingAuthority/master/incubation/bibliography/"\
+    #            "bibliography.json".freeze
     DATADIR = File.expand_path ".relaton/ogc/", Dir.home
     DATAFILE = File.expand_path "bibliography.json", DATADIR
-    ETAGFILE = File.expand_path "etag.txt", DATADIR
+    # ETAGFILE = File.expand_path "etag.txt", DATADIR
 
     # @param ref [Strig]
     # @param year [String]
     # @param opts [Hash]
     def initialize(ref, year = nil)
       super
+      @etagfile = File.expand_path "etag.txt", DATADIR
       @array = from_json(ref).sort_by do |hit|
-        begin
-          hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
-        rescue ArgumentError
-          Date.parse "0000-01-01"
-        end
+        hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
+      rescue ArgumentError
+        Date.parse "0000-01-01"
       end.reverse
     end
 
@@ -52,38 +54,13 @@ module RelatonOgc
     #
     # fetch data form server and save it to file.
     #
-    def fetch_data # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      h = {}
-      h["If-None-Match"] = etag if etag
-      resp = Faraday.new(ENDPOINT, headers: h).get
-      # return if there aren't any changes since last fetching
-      return if resp.status == 304
-      unless resp.status == 200
-        raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
-      end
+    def fetch_data
+      json = get_data
+      return unless json
 
       FileUtils.mkdir_p DATADIR unless Dir.exist? DATADIR
-      self.etag = resp[:etag]
-      @data = JSON.parse resp.body
+      @data = json
       File.write DATAFILE, @data.to_json, encoding: "UTF-8"
     end
-
-    #
-    # Read ETag form file
-    #
-    # @return [String, NilClass]
-    def etag
-      @etag ||= if File.exist? ETAGFILE
-                  File.read ETAGFILE, encoding: "UTF-8"
-                end
-    end
-
-    #
-    # Save ETag to file
-    #
-    # @param tag [String]
-    def etag=(e_tag)
-      File.write ETAGFILE, e_tag, encoding: "UTF-8"
-    end
   end
 end

data/lib/relaton_ogc/ogc_bibliographic_item.rb

@@ -14,17 +14,13 @@ module RelatonOgc
       profile profile-with-extension general
     ].freeze
 
-    # @return [String]
-    attr_reader :docsubtype
-
-    # @param docsubtype [String]
     def initialize(**args)
-      if args[:docsubtype] && !SUBTYPES.include?(args[:docsubtype])
+      if args[:subdoctype] && !SUBTYPES.include?(args[:subdoctype])
         warn "[relaton-ogc] WARNING: invalid document "\
-        "subtype: #{args[:docsubtype]}"
+             "subtype: #{args[:subdoctype]}"
       end
 
-      @docsubtype = args.delete :docsubtype
+      # @docsubtype = args.delete :docsubtype
       # @doctype = args.delete :doctype
       super
     end
@@ -33,15 +29,15 @@ module RelatonOgc
     # @return [RelatonOgc::OgcBibliographicItem]
     def self.from_hash(hash)
       item_hash = ::RelatonOgc::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @return [Hash]
-    def to_hash
-      hash = super
-      hash["docsubtype"] = docsubtype if docsubtype
-      hash
-    end
+    # def to_hash
+    #   hash = super
+    #   hash["docsubtype"] = docsubtype if docsubtype
+    #   hash
+    # end
 
     # @param opts [Hash]
     # @option opts [Nokogiri::XML::Builder] :builder XML builder
@@ -50,10 +46,10 @@ module RelatonOgc
     # @option opts [String, Symbol] :lang language
     # @return [String] XML
     def to_xml(**opts)
-      super **opts do |b|
+      super(**opts) do |b|
         b.ext do
           b.doctype doctype if doctype
-          b.docsubtype docsubtype if docsubtype
+          b.subdoctype subdoctype if subdoctype
           editorialgroup&.to_xml b
           ics.each { |i| i.to_xml b }
         end
@@ -62,11 +58,11 @@ module RelatonOgc
 
     # @param prefix [String]
     # @return [String]
-    def to_asciibib(prefix = "")
-      pref = prefix.empty? ? prefix : prefix + "."
-      out = super
-      out += "#{pref}docsubtype:: #{docsubtype}\n" if docsubtype
-      out
-    end
+    # def to_asciibib(prefix = "")
+    #   pref = prefix.empty? ? prefix : prefix + "."
+    #   out = super
+    #   out += "#{pref}docsubtype:: #{docsubtype}\n" if docsubtype
+    #   out
+    # end
   end
 end

data/lib/relaton_ogc/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonOgc
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ogc
       @prefix = "OGC"
       @defaultprefix = %r{^OGC\s}
       @idtype = "OGC"
+      @datasets = %w[ogc-naming-authority]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonOgc
       ::RelatonOgc::OgcBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonOgc::OgcBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ogc/scrapper.rb

@@ -13,7 +13,7 @@ module RelatonOgc
       "IPR" => { type: "engineering-report" },
       "IS" => { type: "standard", subtype: "implementation" },
       "ISC" => { type: "standard", subtype: "implementation" },
-      "ISx" => { type: "standard", subtype: "extesion" },
+      "ISx" => { type: "standard", subtype: "extension" },
       "Notes" => { type: "other" },
       "ORM" => { type: "reference-model" },
       "PC" => { type: "standard", subtype: "profile" },
@@ -34,7 +34,7 @@ module RelatonOgc
     class << self
       # papam hit [Hash]
       # @return [RelatonOgc::OrcBibliographicItem]
-      def parse_page(hit)
+      def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         type = fetch_type(hit["type"])
         OgcBibliographicItem.new(
           fetched: Date.today.to_s,
@@ -43,7 +43,7 @@ module RelatonOgc
           docid: fetch_docid(hit["identifier"]),
           link: fetch_link(hit["URL"]),
           doctype: type[:type],
-          docsubtype: type[:subtype],
+          subdoctype: type[:subtype],
           docstatus: fetch_status(type[:stage]),
           edition: fetch_edition(hit["identifier"]),
           abstract: fetch_abstract(hit["description"]),
@@ -88,7 +88,7 @@ module RelatonOgc
       # @param stage [String]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(stage)
-        stage && RelatonBib::DocunentStatus.new(stage: stage)
+        stage && RelatonBib::DocumentStatus.new(stage: stage)
       end
 
       # @param identifier [String]
@@ -138,6 +138,8 @@ module RelatonOgc
       # @param date [String]
       # @return [Array<RelatonBib::BibliographicDate>]
       def fetch_date(date)
+        return [] unless date
+
         [RelatonBib::BibliographicDate.new(type: "published", on: date)]
       end
     end