relaton 1.9.4 → 1.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 250207e6a1eaa872380c5b959d727be909d99874797832a4288a058259e96bb2
-  data.tar.gz: df287ddd627ff37b9ecbcd3207a4eb8216690b9d3df9db2a9b31ff2a3648a363
+  metadata.gz: b61ad383992583e255cf71895ebc1304432a0c3ffd1df095fd89938b3bf574c3
+  data.tar.gz: e9c36b287cb349c5d027833b8e9b022bde3c77a19b1c5accf73dc1ebcbb53dea
 SHA512:
-  metadata.gz: 75eecbdaa0c5d6bc954b790eecba13d987974be787181332343b23558534543fb4107366444fe49aab6fea2e54830dc46051449f482d494a9afad0f3bd502abf
-  data.tar.gz: 159b1e402d0b8521634598412edb68cf8e468009d24e96be56f678df98a62099b568d8de04c77a10ed72faed770f4d87d536fd1e152fa24f39786540d5e51643
+  metadata.gz: 52eb6e058e7eb0a995c97eded558d4323bc320d2aa4a6021c5e7b477433e1319fa07d99f4cb93513d1b9e88422d9e59dbe8a35995a4a29ddef6ce1ca40cc54ef
+  data.tar.gz: a84f478f11b661459e4c6477de8d2ca4e84ebd5d43489238480de3b0ea48833292935f96eeaff6ac5d4a58cb7e51c9450b413ae78224a53f07d05a7fdeca411a

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/docs/README.adoc

@@ -178,6 +178,7 @@ x = db.fetch_db("ISO 5749")
 => nil
 
 # Fetching asynchronously
+# RELATON_FETCH_PARALLEL environment variable can be used to ovveride default number of parallel fetches
 
 # prepare queue for results
 results = Queue.new

data/lib/relaton/db.rb

@@ -17,6 +17,7 @@ module Relaton
       @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
                                                       __dir__)
       @queues = {}
+      @semaphore = Mutex.new
     end
 
     # Move global or local caches to anothe dirs
@@ -104,7 +105,8 @@ module Relaton
       if stdclass
         unless @queues[stdclass]
           processor = @registry.processors[stdclass]
-          wp = WorkersPool.new(processor.threads) do |args|
+          threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
+          wp = WorkersPool.new(threads) do |args|
             args[3].call fetch(*args[0..2])
           rescue RelatonBib::RequestError => e
             args[3].call e
@@ -375,14 +377,20 @@ module Relaton
         return bib_retval(bibentry, stdclass)
       end
 
-      db.delete(id) unless db.valid_entry?(id, year)
+      @semaphore.synchronize do
+        db.delete(id) unless db.valid_entry?(id, year)
+      end
       if altdb
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
-        db.clone_entry id, altdb if altdb.valid_entry? id, year
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
-        altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        @semaphore.synchronize do
+          db.clone_entry id, altdb if altdb.valid_entry? id, year
+        end
+        entry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id) unless db[id]
+        @semaphore.synchronize do
+          db[id] ||= entry
+          altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        end
       else
         return bib_retval(db[id], stdclass) if opts[:fetch_db]
 
@@ -413,7 +421,7 @@ module Relaton
       if args[:db] && args[:id] &&
           bib_id && args[:id] !~ %r{#{Regexp.quote("(#{bib_id})")}}
         bid = std_id(bib.docidentifier.first.id, nil, {}, stdclass).first
-        args[:db][bid] ||= bib_entry bib
+        @semaphore.synchronize { args[:db][bid] ||= bib_entry bib }
         "redirection #{bid}"
       else bib_entry bib
       end

data/lib/relaton/registry.rb

@@ -9,7 +9,7 @@ module Relaton
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
       relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie relaton_bsi
-      relaton_cen relaton_iana
+      relaton_cen relaton_iana relaton_3gpp
     ].freeze
 
     include Singleton

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.9.4".freeze
+  VERSION = "1.9.8".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -4,9 +4,9 @@ module Relaton
   # Workers poll.
   class WorkersPool
     def initialize(workers = 2, &_block)
-      num_workers = workers < 2 ? 2 : workers
-      @queue = SizedQueue.new(num_workers * 2)
-      @threads = Array.new num_workers do
+      # num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(workers * 2)
+      @threads = Array.new workers do
         Thread.new do
           while item = @queue.pop; yield(item) end
         end

data/relaton.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
+  spec.add_dependency "relaton-3gpp", "~> 1.9.0"
   spec.add_dependency "relaton-bipm", "~> 1.9.0"
   spec.add_dependency "relaton-bsi", "~> 1.9.0"
   spec.add_dependency "relaton-calconnect", "~> 1.9.0"
@@ -39,7 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "relaton-iana", "~> 1.9.0"
   spec.add_dependency "relaton-iec", "~> 1.9.0"
   spec.add_dependency "relaton-ieee", "~> 1.9.0"
-  spec.add_dependency "relaton-ietf", "~> 1.9.0"
+  spec.add_dependency "relaton-ietf", ">= 1.9.7"
   spec.add_dependency "relaton-iho", "~> 1.9.0"
   spec.add_dependency "relaton-iso", "~> 1.9.0"
   spec.add_dependency "relaton-itu", "~> 1.9.0"

data/spec/relaton/db_spec.rb

@@ -184,6 +184,16 @@ RSpec.describe Relaton::Db do
       end
       expect(result).to be_nil
     end
+
+    it "use threads number from RELATON_FETCH_PARALLEL" do
+      expect(ENV).to receive(:[]).with("RELATON_FETCH_PARALLEL").and_return(1)
+      allow(ENV).to receive(:[]).and_call_original
+      expect(Relaton::WorkersPool).to receive(:new).with(1).and_call_original
+      VCR.use_cassette "threads_from_env" do
+        db.fetch_async("ITU-T G.993.5") { |r| queue << r }
+        Timeout.timeout(50) { queue.pop }
+      end
+    end
   end
 
   context "fetch documents form static cache" do

data/spec/relaton/registry_spec.rb

@@ -91,6 +91,10 @@ RSpec.describe Relaton::Registry do
     it "IANA" do
       expect(Relaton::Registry.instance.by_type("IANA")).to be_instance_of RelatonIana::Processor
     end
+
+    it "3GPP" do
+      expect(Relaton::Registry.instance.by_type("3GPP")).to be_instance_of Relaton3gpp::Processor
+    end
   end
 
   it "find processot by dataset" do

data/spec/relaton_spec.rb

@@ -196,10 +196,10 @@ RSpec.describe Relaton::Db do
       expect(File.exist?("testcache2")).to be true
       testcache = Relaton::DbCache.new "testcache"
       expect(testcache["IETF(RFC 8341)"]).to include "<docidentifier "\
-        "type=\"IETF\">RFC 8341</docidentifier>"
+        "type=\"IETF\" scope=\"anchor\">RFC8341</docidentifier>"
       testcache = Relaton::DbCache.new "testcache2"
       expect(testcache["IETF(RFC 8341)"]).to include "<docidentifier "\
-        "type=\"IETF\">RFC 8341</docidentifier>"
+        "type=\"IETF\" scope=\"anchor\">RFC8341</docidentifier>"
     end
   end
 
@@ -306,6 +306,13 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "get 3GPP reference" do
+    VCR.use_cassette "3gpp_tr_00_01u_umts_3_0_0" do
+      bib = @db.fetch "3GPP TR 00.01U:UMTS/3.0.0"
+      expect(bib).to be_instance_of Relaton3gpp::BibliographicItem
+    end
+  end
+
   context "get combined documents" do
     context "ISO" do
       it "included" do

data/spec/vcr_cassetes/3gpp_tr_00_01u_umts_3_0_0.yml

@@ -0,0 +1,121 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://raw.githubusercontent.com/relaton/relaton-data-3gpp/main/data/TR_00.01U_UMTS_3.0.0.yaml
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Host:
+      - raw.githubusercontent.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '549'
+      Cache-Control:
+      - max-age=300
+      Content-Security-Policy:
+      - default-src 'none'; style-src 'unsafe-inline'; sandbox
+      Content-Type:
+      - text/plain; charset=utf-8
+      Etag:
+      - W/"c70f6628f95ac3979a5237c54995637171f2f370a6090b835ecd229ef159fb99"
+      Strict-Transport-Security:
+      - max-age=31536000
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Github-Request-Id:
+      - 5F84:9E53:173E127:186CA31:61CAF6F2
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 28 Dec 2021 11:37:22 GMT
+      Via:
+      - 1.1 varnish
+      X-Served-By:
+      - cache-sof1510075-SOF
+      X-Cache:
+      - MISS
+      X-Cache-Hits:
+      - '0'
+      X-Timer:
+      - S1640691442.275517,VS0,VE245
+      Vary:
+      - Authorization,Accept-Encoding,Origin
+      Access-Control-Allow-Origin:
+      - "*"
+      X-Fastly-Request-Id:
+      - f1d9e96d0649f362bbac0579ea916a9009733903
+      Expires:
+      - Tue, 28 Dec 2021 11:42:22 GMT
+      Source-Age:
+      - '0'
+    body:
+      encoding: ASCII-8BIT
+      string: |
+        ---
+        id: 3GPPTR00.01U-UMTS/3.0.0
+        title:
+          type: main
+          content: Work programme for the standardization of Universal Mobile Telecommunications
+            System (UMTS)
+          format: text/plain
+        link:
+          content: http://www.3gpp.org/ftp/Specs/archive/00_series/00.01U/0001U-300.zip
+          type: src
+        type: standard
+        docid:
+          id: 3GPP TR 00.01U:UMTS/3.0.0
+          type: 3GPP
+        docnumber: TR 00.01U:UMTS/3.0.0
+        contributor:
+          person:
+            name:
+              completename:
+                content: RAPELI, Juha
+                language: en
+                script: Latn
+            affiliation:
+              organization:
+                name: PHILIPS CONSUMER COMMUNICATION
+          role: author
+        language: en
+        script: Latn
+        docstatus:
+          stage:
+            value: withdrawn
+        fetched: '2021-12-24'
+        doctype: TR
+        editorialgroup:
+          name: SMG5
+          type: prime
+        radiotechnology: 3G
+        release:
+          version2g: '3'
+          version3g: '3'
+          defunct: true
+          freeze_meeting: SMG-28
+          freeze_stage1_meeting: SMG-28
+          freeze_stage2_meeting: SMG-28
+          freeze_stage3_meeting: SMG-28
+          close_meeting: SP-28
+          project_end: '1999-02-12'
+    http_version:
+  recorded_at: Tue, 28 Dec 2021 11:37:22 GMT
+recorded_with: VCR 5.1.0