relaton 1.9.2 → 1.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00617ed8db3b679a9c44d7b8854b17c16d48f10b84a30bbc2e175113d02e9025
-  data.tar.gz: 198a87943ebe8f2ffdeabfc0c1ec12a9499ec9642d90ca393a8c26440f3acf3f
+  metadata.gz: '0856681e37a3dfb7fef7e84084c49990f4f980efcdf2bd3be6e019c9baf8c3e6'
+  data.tar.gz: 9c53c38d1b422435f5969ce02f7bee7b05d1195e57b0526a958a83abcaf2eea6
 SHA512:
-  metadata.gz: 2bf33ca669f3b719eea8e9add8a107e9f54e86ffcfda53d3845fe5dd109f08cde651b08024dda622f5d3438f8dce51d142c8b1f886e5b45b7ef481242b9212ea
-  data.tar.gz: bf242855dfb87ff3920907cf69616b2831c8ce35e65342327d0cfee3182b252bbceca244fbd30cdb097f09befc90769c9c4b72847b52cea852ddd3dc9c2d7441
+  metadata.gz: fc981fcd5df1fc7de922df2a2e979963d26ef784e4a6bc88448a632a8b09a0483292edbd2ecbe70caa129d59a7c9254c95c9307d01bd5dba6d0e28cd17832fbb
+  data.tar.gz: 27cb0dd8f208880222fdc4d8960ef644ca9ec1b1ed678621807ca319fe0596e7a560815c78504b4ebd4de35198f91a054125e5632273a068d44781abfa62a65e

data/docs/README.adoc

@@ -178,6 +178,7 @@ x = db.fetch_db("ISO 5749")
 => nil
 
 # Fetching asynchronously
+# RELATON_FETCH_PARALLEL environment variable can be used to ovveride default number of parallel fetches
 
 # prepare queue for results
 results = Queue.new
@@ -187,8 +188,8 @@ refs = ["ISO 19011", "ISO 19115"]
 
 # fetch documents
 refs.each do |ref|
-  db.fetch_async(ref, ref: ref) do |doc, other_args|
-    results << [doc, other_args[:ref]]
+  db.fetch_async(ref) do |doc|
+    results << [doc, ref]
   end
 end
 

data/lib/relaton/db.rb

@@ -17,6 +17,7 @@ module Relaton
       @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
                                                       __dir__)
       @queues = {}
+      @semaphore = Mutex.new
     end
 
     # Move global or local caches to anothe dirs
@@ -99,20 +100,21 @@ module Relaton
     end
 
     # Fetch asynchronously
-    def fetch_async(code, year = nil, opts = {}, **others, &_block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def fetch_async(code, year = nil, opts = {}, &block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       stdclass = standard_class code
       if stdclass
         unless @queues[stdclass]
           processor = @registry.processors[stdclass]
-          wp = WorkersPool.new(processor.threads) do |args|
-            yield fetch(*args[0..2]), **args[3]
+          threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
+          wp = WorkersPool.new(threads) do |args|
+            args[3].call fetch(*args[0..2])
           rescue RelatonBib::RequestError => e
-            yield e, **args[3]
+            args[3].call e
           end
           @queues[stdclass] = { queue: Queue.new, workers_pool: wp }
           Thread.new { process_queue @queues[stdclass] }
         end
-        @queues[stdclass][:queue] << [code, year, opts, others]
+        @queues[stdclass][:queue] << [code, year, opts, block]
       else yield nil
       end
     end
@@ -375,14 +377,20 @@ module Relaton
         return bib_retval(bibentry, stdclass)
       end
 
-      db.delete(id) unless db.valid_entry?(id, year)
+      @semaphore.synchronize do
+        db.delete(id) unless db.valid_entry?(id, year)
+      end
       if altdb
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
-        db.clone_entry id, altdb if altdb.valid_entry? id, year
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
-        altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        @semaphore.synchronize do
+          db.clone_entry id, altdb if altdb.valid_entry? id, year
+        end
+        entry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id) unless db[id]
+        @semaphore.synchronize do
+          db[id] ||= entry
+          altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
+        end
       else
         return bib_retval(db[id], stdclass) if opts[:fetch_db]
 
@@ -413,7 +421,7 @@ module Relaton
       if args[:db] && args[:id] &&
           bib_id && args[:id] !~ %r{#{Regexp.quote("(#{bib_id})")}}
         bid = std_id(bib.docidentifier.first.id, nil, {}, stdclass).first
-        args[:db][bid] ||= bib_entry bib
+        @semaphore.synchronize { args[:db][bid] ||= bib_entry bib }
         "redirection #{bid}"
       else bib_entry bib
       end

data/lib/relaton/registry.rb

@@ -9,7 +9,7 @@ module Relaton
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
       relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie relaton_bsi
-      relaton_cen
+      relaton_cen relaton_iana
     ].freeze
 
     include Singleton

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.9.2".freeze
+  VERSION = "1.9.6".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -4,9 +4,9 @@ module Relaton
   # Workers poll.
   class WorkersPool
     def initialize(workers = 2, &_block)
-      num_workers = workers < 2 ? 2 : workers
-      @queue = SizedQueue.new(num_workers * 2)
-      @threads = Array.new num_workers do
+      # num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(workers * 2)
+      @threads = Array.new workers do
         Thread.new do
           while item = @queue.pop; yield(item) end
         end

data/relaton.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "relaton-cie", "~> 1.9.0"
   spec.add_dependency "relaton-ecma", "~> 1.9.0"
   spec.add_dependency "relaton-gb", "~> 1.9.0"
+  spec.add_dependency "relaton-iana", "~> 1.9.0"
   spec.add_dependency "relaton-iec", "~> 1.9.0"
   spec.add_dependency "relaton-ieee", "~> 1.9.0"
   spec.add_dependency "relaton-ietf", "~> 1.9.0"

data/spec/relaton/db_spec.rb

@@ -163,8 +163,8 @@ RSpec.describe Relaton::Db do
       results = []
       VCR.use_cassette "async_fetch", match_requests_on: %i[method uri body] do
         refs.each do |ref|
-          db.fetch_async(ref, ref: ref) do |r, otherargs|
-            queue << [r, otherargs[:ref]]
+          db.fetch_async(ref) do |r|
+            queue << [r, ref]
           end
         end
         Timeout.timeout(60) do
@@ -184,6 +184,16 @@ RSpec.describe Relaton::Db do
       end
       expect(result).to be_nil
     end
+
+    it "use threads number from RELATON_FETCH_PARALLEL" do
+      expect(ENV).to receive(:[]).with("RELATON_FETCH_PARALLEL").and_return(1)
+      allow(ENV).to receive(:[]).and_call_original
+      expect(Relaton::WorkersPool).to receive(:new).with(1).and_call_original
+      VCR.use_cassette "threads_from_env" do
+        db.fetch_async("ITU-T G.993.5") { |r| queue << r }
+        Timeout.timeout(5) { queue.pop }
+      end
+    end
   end
 
   context "fetch documents form static cache" do

data/spec/relaton/registry_spec.rb

@@ -87,6 +87,10 @@ RSpec.describe Relaton::Registry do
     it "CEN" do
       expect(Relaton::Registry.instance.by_type("CEN")).to be_instance_of RelatonCen::Processor
     end
+
+    it "IANA" do
+      expect(Relaton::Registry.instance.by_type("IANA")).to be_instance_of RelatonIana::Processor
+    end
   end
 
   it "find processot by dataset" do

data/spec/relaton_spec.rb

@@ -246,13 +246,13 @@ RSpec.describe Relaton::Db do
   end
 
   it "get W3C reference" do
-    w3c_fr = /\.relaton\/w3c\/bibliography\.yml/
-    expect(File).to receive(:exist?).with(w3c_fr).and_return false
-    expect(File).to receive(:exist?).and_call_original.at_least :once
-    expect(File).to receive(:write).with w3c_fr, kind_of(String), kind_of(Hash)
+    # w3c_fr = /\.relaton\/w3c\/bibliography\.yml/
+    # expect(File).to receive(:exist?).with(w3c_fr).and_return false
+    # expect(File).to receive(:exist?).and_call_original.at_least :once
+    # expect(File).to receive(:write).with w3c_fr, kind_of(String), kind_of(Hash)
     # expect(File).to receive(:write).and_call_original.at_least :once
     VCR.use_cassette "w3c_json_ld11" do
-      bib = @db.fetch "W3C JSON-LD 1.1", nil, {}
+      bib = @db.fetch "W3C REC-json-ld11-20200716", nil, {}
       expect(bib).to be_instance_of RelatonW3c::W3cBibliographicItem
     end
   end
@@ -299,6 +299,13 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "get IANA reference" do
+    VCR.use_cassette "iana_service_names_port_numbers" do
+      bib = @db.fetch "IANA service-names-port-numbers"
+      expect(bib).to be_instance_of RelatonBib::BibliographicItem
+    end
+  end
+
   context "get combined documents" do
     context "ISO" do
       it "included" do