relaton 1.9.3 → 1.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3599c52bacdd454603dc095055338b4a89b8264ee9391fa2a2b25f1724a46e2
-  data.tar.gz: aa64435911173af46c12a522b05d063c4cfa0644f1919c96ec3879f9b8f9af39
+  metadata.gz: daae758f0751cfb8c4682b4f74c4d67b4046290562d47e84bc7103df5da670d4
+  data.tar.gz: 1348bdfebcf234ac6b29c0ba5e09a357eb3d8885645671932d937a95e2528a91
 SHA512:
-  metadata.gz: 74fc610b35697515cf2f925349ff9ad076c50a3ec8e0fa08a9cbfa3fc82217c32b2f5e2869af604a58e12f70898de41addade73971a6fae9085b56eb7b3b5ff0
-  data.tar.gz: 8cda3116f9e84eb3f549dc1a4c5a74fb6957d4834d0cf140f587434f2b9ff84e7cae93946147b7151fa6cf1739f13aba1639f92c2f73fce552999e3a7dfe35fd
+  metadata.gz: 01c4d8fcc1c8a77e4425a87be0f28e5d178b758a1bedb19aaceacde9df7f9f138da4bd1df849d819d6ad2e6675a7e79479168ab9edce7800bd5491e4ee2e123d
+  data.tar.gz: 115ce0d39bf35ae82c973fc2043bc5f07d5b0d87ca5f6f1fc7e36cdc032c4e97c7aea9129d168c2e30f2073713f2d0128fd5d2f8de78eaa5f07344cb949723a4

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/docs/README.adoc

@@ -178,6 +178,7 @@ x = db.fetch_db("ISO 5749")
 => nil
 
 # Fetching asynchronously
+# RELATON_FETCH_PARALLEL environment variable can be used to ovveride default number of parallel fetches
 
 # prepare queue for results
 results = Queue.new

data/lib/relaton/db.rb

@@ -17,6 +17,7 @@ module Relaton
       @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
                                                       __dir__)
       @queues = {}
+      @semaphore = Mutex.new
     end
 
     # Move global or local caches to anothe dirs
@@ -104,7 +105,8 @@ module Relaton
       if stdclass
         unless @queues[stdclass]
           processor = @registry.processors[stdclass]
-          wp = WorkersPool.new(processor.threads) do |args|
+          threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
+          wp = WorkersPool.new(threads) do |args|
             args[3].call fetch(*args[0..2])
           rescue RelatonBib::RequestError => e
             args[3].call e
@@ -375,14 +377,20 @@ module Relaton
         return bib_retval(bibentry, stdclass)
       end
 
-      db.delete(id) unless db.valid_entry?(id, year)
+      @semaphore.synchronize do
+        db.delete(id) unless db.valid_entry?(id, year)
+      end
       if altdb
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
-        db.clone_entry id, altdb if altdb.valid_entry? id, year
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
-        altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        @semaphore.synchronize do
+          db.clone_entry id, altdb if altdb.valid_entry? id, year
+        end
+        entry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id) unless db[id]
+        @semaphore.synchronize do
+          db[id] ||= entry
+          altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        end
       else
         return bib_retval(db[id], stdclass) if opts[:fetch_db]
 
@@ -413,7 +421,7 @@ module Relaton
       if args[:db] && args[:id] &&
           bib_id && args[:id] !~ %r{#{Regexp.quote("(#{bib_id})")}}
         bid = std_id(bib.docidentifier.first.id, nil, {}, stdclass).first
-        args[:db][bid] ||= bib_entry bib
+        @semaphore.synchronize { args[:db][bid] ||= bib_entry bib }
         "redirection #{bid}"
       else bib_entry bib
       end

data/lib/relaton/registry.rb

@@ -9,7 +9,7 @@ module Relaton
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
       relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie relaton_bsi
-      relaton_cen
+      relaton_cen relaton_iana relaton_3gpp
     ].freeze
 
     include Singleton

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.9.3".freeze
+  VERSION = "1.9.7".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -4,9 +4,9 @@ module Relaton
   # Workers poll.
   class WorkersPool
     def initialize(workers = 2, &_block)
-      num_workers = workers < 2 ? 2 : workers
-      @queue = SizedQueue.new(num_workers * 2)
-      @threads = Array.new num_workers do
+      # num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(workers * 2)
+      @threads = Array.new workers do
         Thread.new do
           while item = @queue.pop; yield(item) end
         end

data/relaton.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
+  spec.add_dependency "relaton-3gpp", "~> 1.9.0"
   spec.add_dependency "relaton-bipm", "~> 1.9.0"
   spec.add_dependency "relaton-bsi", "~> 1.9.0"
   spec.add_dependency "relaton-calconnect", "~> 1.9.0"
@@ -36,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "relaton-cie", "~> 1.9.0"
   spec.add_dependency "relaton-ecma", "~> 1.9.0"
   spec.add_dependency "relaton-gb", "~> 1.9.0"
+  spec.add_dependency "relaton-iana", "~> 1.9.0"
   spec.add_dependency "relaton-iec", "~> 1.9.0"
   spec.add_dependency "relaton-ieee", "~> 1.9.0"
   spec.add_dependency "relaton-ietf", "~> 1.9.0"

data/spec/relaton/db_spec.rb

@@ -184,6 +184,16 @@ RSpec.describe Relaton::Db do
       end
       expect(result).to be_nil
     end
+
+    it "use threads number from RELATON_FETCH_PARALLEL" do
+      expect(ENV).to receive(:[]).with("RELATON_FETCH_PARALLEL").and_return(1)
+      allow(ENV).to receive(:[]).and_call_original
+      expect(Relaton::WorkersPool).to receive(:new).with(1).and_call_original
+      VCR.use_cassette "threads_from_env" do
+        db.fetch_async("ITU-T G.993.5") { |r| queue << r }
+        Timeout.timeout(5) { queue.pop }
+      end
+    end
   end
 
   context "fetch documents form static cache" do

data/spec/relaton/registry_spec.rb

@@ -87,6 +87,14 @@ RSpec.describe Relaton::Registry do
     it "CEN" do
       expect(Relaton::Registry.instance.by_type("CEN")).to be_instance_of RelatonCen::Processor
     end
+
+    it "IANA" do
+      expect(Relaton::Registry.instance.by_type("IANA")).to be_instance_of RelatonIana::Processor
+    end
+
+    it "3GPP" do
+      expect(Relaton::Registry.instance.by_type("3GPP")).to be_instance_of Relaton3gpp::Processor
+    end
   end
 
   it "find processot by dataset" do

data/spec/relaton_spec.rb

@@ -246,13 +246,13 @@ RSpec.describe Relaton::Db do
   end
 
   it "get W3C reference" do
-    w3c_fr = /\.relaton\/w3c\/bibliography\.yml/
-    expect(File).to receive(:exist?).with(w3c_fr).and_return false
-    expect(File).to receive(:exist?).and_call_original.at_least :once
-    expect(File).to receive(:write).with w3c_fr, kind_of(String), kind_of(Hash)
+    # w3c_fr = /\.relaton\/w3c\/bibliography\.yml/
+    # expect(File).to receive(:exist?).with(w3c_fr).and_return false
+    # expect(File).to receive(:exist?).and_call_original.at_least :once
+    # expect(File).to receive(:write).with w3c_fr, kind_of(String), kind_of(Hash)
     # expect(File).to receive(:write).and_call_original.at_least :once
     VCR.use_cassette "w3c_json_ld11" do
-      bib = @db.fetch "W3C JSON-LD 1.1", nil, {}
+      bib = @db.fetch "W3C REC-json-ld11-20200716", nil, {}
       expect(bib).to be_instance_of RelatonW3c::W3cBibliographicItem
     end
   end
@@ -299,6 +299,20 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "get IANA reference" do
+    VCR.use_cassette "iana_service_names_port_numbers" do
+      bib = @db.fetch "IANA service-names-port-numbers"
+      expect(bib).to be_instance_of RelatonBib::BibliographicItem
+    end
+  end
+
+  it "get 3GPP reference" do
+    VCR.use_cassette "3gpp_tr_00_01u_umts_3_0_0" do
+      bib = @db.fetch "3GPP TR 00.01U:UMTS/3.0.0"
+      expect(bib).to be_instance_of Relaton3gpp::BibliographicItem
+    end
+  end
+
   context "get combined documents" do
     context "ISO" do
       it "included" do

data/spec/vcr_cassetes/3gpp_tr_00_01u_umts_3_0_0.yml

@@ -0,0 +1,112 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://raw.githubusercontent.com/relaton/relaton-data-3gpp/main/data/TR_00.01U_UMTS_3.0.0.yaml
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Host:
+      - raw.githubusercontent.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '457'
+      Cache-Control:
+      - max-age=300
+      Content-Security-Policy:
+      - default-src 'none'; style-src 'unsafe-inline'; sandbox
+      Content-Type:
+      - text/plain; charset=utf-8
+      Etag:
+      - W/"5d2f491d96c7faf0fe7b769cefadb3f744327d06d70da694edc1386de14fee7a"
+      Strict-Transport-Security:
+      - max-age=31536000
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Github-Request-Id:
+      - 45B8:60DB:8C2BC7:933607:61B60A00
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Sun, 12 Dec 2021 14:41:05 GMT
+      Via:
+      - 1.1 varnish
+      X-Served-By:
+      - cache-sof1510064-SOF
+      X-Cache:
+      - MISS
+      X-Cache-Hits:
+      - '0'
+      X-Timer:
+      - S1639320065.893291,VS0,VE196
+      Vary:
+      - Authorization,Accept-Encoding,Origin
+      Access-Control-Allow-Origin:
+      - "*"
+      X-Fastly-Request-Id:
+      - 93f34b3047b3a5fc6b81f1b6684eeb810830e9b0
+      Expires:
+      - Sun, 12 Dec 2021 14:46:05 GMT
+      Source-Age:
+      - '0'
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        ---
+        id: 3GPPTR00.01U-UMTS/3.0.0
+        title:
+          type: main
+          content: Work programme for the standardization of Universal Mobile Telecommunications
+            System (UMTS)
+          format: text/plain
+        link:
+          content: http://www.3gpp.org/ftp/Specs/archive/00_series/00.01U/0001U-300.zip
+          type: src
+        type: standard
+        docid:
+        - id: 3GPP TR 00.01U:UMTS/3.0.0
+          type: 3GPP
+        - id: '2261'
+          type: rapporteurId
+        docnumber: TR 00.01U:UMTS/3.0.0
+        language: en
+        script: Latn
+        docstatus:
+          stage:
+            value: withdrawn
+        fetched: '2021-12-09'
+        doctype: TR
+        editorialgroup:
+          name: SMG5
+          type: prime
+        radiotechnology: 3G
+        release:
+          version2g: '3'
+          version3g: '3'
+          defunct: true
+          freeze_meeting: SMG-28
+          freeze_stage1_meeting: SMG-28
+          freeze_stage2_meeting: SMG-28
+          freeze_stage3_meeting: SMG-28
+          close_meeting: SP-28
+          project_end: '1999-02-12'
+    http_version:
+  recorded_at: Sun, 12 Dec 2021 14:41:05 GMT
+recorded_with: VCR 5.1.0