relaton-ogc 1.9.1 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fa7cf7cb4a2e7b21cb49a4fbb482f08119776626d18b6ab186080905783d516
-  data.tar.gz: d4bb20a2c3f35b1c25ac99882c5a3f7a405a420cc5d3dbcb2f6359b0f2f6166f
+  metadata.gz: d9ddd36656e221ad7ceb82adf637197b15648451b42fbd78069bc02078082c05
+  data.tar.gz: 9179d1d896e37f26841c392701c500990eafbfdb1060a4120bb85189014d788e
 SHA512:
-  metadata.gz: d5679900661ae943e2e929500314cfbb98fe416ebe8c809fccd0daa6f5d2c6232d9c85844d406372ab41af3c413de914944fd7b0433e24c7582a80942fe7d710
-  data.tar.gz: c187307b81cc069bc652f38e494ed01de8dfe96de9c00b7fc4c719f73dcff62335cc3d4167a4d32b1b402d31ac9df7e76fe5a77e6e2295cf4c6f1937c6b1f4a5
+  metadata.gz: 0ab0f2b5ed50fd8f218c92ae1a8589df818f92bb2480fffc450fbe930c16185cc9855d683dc4aafb5d34ad4f808ffd1cc44ba11b19adc6550d62dc029e7c0b11
+  data.tar.gz: c62733901e8f8790eaaccf90345503fd187bba8e8a08655400fdd77aa508df640ec7a4f11ef1146e41b0e456c538954769cbb4ef2d6eacba3b6b2cac73a70e01

data/README.adoc

@@ -113,6 +113,25 @@ RelatonOgc::OgcBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/opengeospatial/NamingAuthority/master/incubation/bibliography/bibliography.json dataset as a data sources.
+
+The method `RelatonOgc::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonOgc::DataFetcher.fetch
+Started at: 2021-09-14 11:21:46 +0200
+[relaton-ogc] WARNING Duplicated documents: 15-113r5, 08-094r1, 10-025r1, 12-128r14, 16-079, 16-007r3, 13-026r8, 12-128r12, 15-078r6, 12-176r7, 09-102r3, 14-095, 14-115, 07-147r2, 12-000, 12-006, 09-025r1, 07-036, 07-110r4, 03-105r1, 06-042, 07-165r1, 12-066, 06-104r4, 11-122r1, 09-000, 04-094, 07-006r1, 06-035r1, 03-006r3, 05-134, 04-021r3, 02-058, 01-009
+Stopped at: 2021-09-14 11:21:48 +0200
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_ogc/data_fetcher.rb

@@ -0,0 +1,96 @@
+module RelatonOgc
+  class DataFetcher
+    module Utils
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+                 "NamingAuthority/master/incubation/bibliography/"\
+                 "bibliography.json".freeze
+
+      def get_data # rubocop:disable Metrics/AbcSize
+        h = {}
+        h["If-None-Match"] = etag if etag
+        resp = Faraday.new(ENDPOINT, headers: h).get
+        case resp.status
+        when 200
+          json = JSON.parse(resp.body)
+          block_given? ? yield(resp[:etag], json) : json
+        when 304 then [] # there aren't any changes since last fetching
+        else raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
+        end
+      end
+
+      #
+      # Read ETag form file
+      #
+      # @return [String, NilClass]
+      def etag
+        @etag ||= if File.exist? @etagfile
+                    File.read @etagfile, encoding: "UTF-8"
+                  end
+      end
+
+      #
+      # Save ETag to file
+      #
+      # @param tag [String]
+      def etag=(e_tag)
+        File.write @etagfile, e_tag, encoding: "UTF-8"
+      end
+    end
+
+    include Utils
+
+    #
+    # Create DataFetcher instance
+    #
+    # @param [String] output directory to save the documents
+    # @param [String] format output format "yaml" or "xmo"
+    #
+    def initialize(output, format)
+      @output = output
+      @etagfile = File.join output, "etag.txt"
+      @format = format
+      @docids = []
+      @dupids = []
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    def fetch # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      get_data do |etag, json|
+        no_errors = true
+        json.each do |_, hit|
+          bib = Scrapper.parse_page hit
+          write_document bib
+        rescue StandardError => e
+          no_errors = false
+          warn "Fetching document: #{hit['identifier']}"
+          warn "#{e.class} #{e.message}"
+          warn e.backtrace
+        end
+        warn "[relaton-ogc] WARNING Duplicated documents: #{@dupids.uniq.join(', ')}" if @dupids.any?
+        self.etag = etag if no_errors
+      end
+    end
+
+    def write_document(bib) # rubocop:disable Metrics/AbcSize
+      if @docids.include?(bib.docidentifier[0].id)
+        @dupids << bib.docidentifier[0].id
+        return
+      end
+
+      @docids << bib.docidentifier[0].id
+      name = bib.docidentifier[0].id.upcase.gsub(/[\s:.]/, "_")
+      file = "#{@output}/#{name}.#{@format}"
+      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file, content, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton_ogc/hit_collection.rb

@@ -4,23 +4,25 @@ require "fileutils"
 
 module RelatonOgc
   class HitCollection < RelatonBib::HitCollection
-    ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
-      "NamingAuthority/master/incubation/bibliography/bibliography.json".freeze
+    include DataFetcher::Utils
+
+    # ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
+    #            "NamingAuthority/master/incubation/bibliography/"\
+    #            "bibliography.json".freeze
     DATADIR = File.expand_path ".relaton/ogc/", Dir.home
     DATAFILE = File.expand_path "bibliography.json", DATADIR
-    ETAGFILE = File.expand_path "etag.txt", DATADIR
+    # ETAGFILE = File.expand_path "etag.txt", DATADIR
 
     # @param ref [Strig]
     # @param year [String]
     # @param opts [Hash]
     def initialize(ref, year = nil)
       super
+      @etagfile = File.expand_path "etag.txt", DATADIR
       @array = from_json(ref).sort_by do |hit|
-        begin
-          hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
-        rescue ArgumentError
-          Date.parse "0000-01-01"
-        end
+        hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
+      rescue ArgumentError
+        Date.parse "0000-01-01"
       end.reverse
     end
 
@@ -52,38 +54,13 @@ module RelatonOgc
     #
     # fetch data form server and save it to file.
     #
-    def fetch_data # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      h = {}
-      h["If-None-Match"] = etag if etag
-      resp = Faraday.new(ENDPOINT, headers: h).get
-      # return if there aren't any changes since last fetching
-      return if resp.status == 304
-      unless resp.status == 200
-        raise RelatonBib::RequestError, "Could not access #{ENDPOINT}"
-      end
+    def fetch_data
+      json = get_data
+      return unless json
 
       FileUtils.mkdir_p DATADIR unless Dir.exist? DATADIR
-      self.etag = resp[:etag]
-      @data = JSON.parse resp.body
+      @data = json
       File.write DATAFILE, @data.to_json, encoding: "UTF-8"
     end
-
-    #
-    # Read ETag form file
-    #
-    # @return [String, NilClass]
-    def etag
-      @etag ||= if File.exist? ETAGFILE
-                  File.read ETAGFILE, encoding: "UTF-8"
-                end
-    end
-
-    #
-    # Save ETag to file
-    #
-    # @param tag [String]
-    def etag=(e_tag)
-      File.write ETAGFILE, e_tag, encoding: "UTF-8"
-    end
   end
 end

data/lib/relaton_ogc/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonOgc
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ogc
       @prefix = "OGC"
       @defaultprefix = %r{^OGC\s}
       @idtype = "OGC"
+      @datasets = %w[ogc-naming-authority]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonOgc
       ::RelatonOgc::OgcBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonOgc::OgcBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ogc/scrapper.rb

@@ -13,7 +13,7 @@ module RelatonOgc
       "IPR" => { type: "engineering-report" },
       "IS" => { type: "standard", subtype: "implementation" },
       "ISC" => { type: "standard", subtype: "implementation" },
-      "ISx" => { type: "standard", subtype: "extesion" },
+      "ISx" => { type: "standard", subtype: "extension" },
       "Notes" => { type: "other" },
       "ORM" => { type: "reference-model" },
       "PC" => { type: "standard", subtype: "profile" },
@@ -88,7 +88,7 @@ module RelatonOgc
       # @param stage [String]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(stage)
-        stage && RelatonBib::DocunentStatus.new(stage: stage)
+        stage && RelatonBib::DocumentStatus.new(stage: stage)
       end
 
       # @param identifier [String]
@@ -138,6 +138,8 @@ module RelatonOgc
       # @param date [String]
       # @return [Array<RelatonBib::BibliographicDate>]
       def fetch_date(date)
+        return [] unless date
+
         [RelatonBib::BibliographicDate.new(type: "published", on: date)]
       end
     end

data/lib/relaton_ogc/version.rb

@@ -1,3 +1,3 @@
 module RelatonOgc
-  VERSION = "1.9.1".freeze
+  VERSION = "1.9.2".freeze
 end

data/lib/relaton_ogc.rb

@@ -2,6 +2,7 @@ require "relaton_iso_bib"
 require "relaton_ogc/version"
 require "relaton_ogc/ogc_bibliographic_item"
 require "relaton_ogc/ogc_bibliography"
+require "relaton_ogc/data_fetcher"
 require "relaton_ogc/hit_collection"
 require "relaton_ogc/scrapper"
 require "relaton_ogc/xml_parser"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ogc
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-07 00:00:00.000000000 Z
+date: 2021-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -175,6 +175,7 @@ files:
 - grammars/ogc.rng
 - grammars/reqt.rng
 - lib/relaton_ogc.rb
+- lib/relaton_ogc/data_fetcher.rb
 - lib/relaton_ogc/editorial_group.rb
 - lib/relaton_ogc/hash_converter.rb
 - lib/relaton_ogc/hit.rb