relaton 1.7.2 → 1.7.7

data/lib/relaton/db_cache.rb

@@ -1,4 +1,5 @@
 require "fileutils"
+require "timeout"
 
 module Relaton
   class DbCache
@@ -14,6 +15,26 @@ module Relaton
       # set_version # unless File.exist? file_version
     end
 
+    # Move caches to anothe dir
+    # @param new_dir [String, nil]
+    # @return [String, nil]
+    def mv(new_dir)
+      return unless new_dir && @ext == "xml"
+
+      if File.exist? new_dir
+        warn "[relaton] WARNING: target directory exists \"#{new_dir}\""
+        return
+      end
+
+      FileUtils.mv dir, new_dir
+      @dir = new_dir
+    end
+
+    # Clear database
+    def clear
+      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # unless it's static DB
+    end
+
     # Save item
     # @param key [String]
     # @param value [String] Bibitem xml serialization
@@ -76,8 +97,9 @@ module Relaton
     # Returns all items
     # @return [Array<String>]
     def all
-      Dir.glob("#{@dir}/**/*.xml").sort.map do |f|
-        File.read(f, encoding: "utf-8")
+      Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").sort.map do |f|
+        content = File.read(f, encoding: "utf-8")
+        block_given? ? yield(f, content) : content
       end
     end
 
@@ -193,7 +215,7 @@ module Relaton
     # @content [String]
     def file_safe_write(file, content)
       File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
-        Timeout.timeout(1) { f.flock File::LOCK_EX }
+        Timeout.timeout(10) { f.flock File::LOCK_EX }
         f.write content
       end
     end

data/lib/relaton/processor.rb

@@ -20,5 +20,16 @@ module Relaton
     def hash_to_bib(_hash)
       raise "This is an abstract class!"
     end
+
+    def grammar_hash
+      raise "This is an abstract class!"
+    end
+
+    # Retuns default number of workers. Should be overraded by childred classes if need.
+    #
+    # @return [Integer] nuber of wokrers
+    def threads
+      10
+    end
   end
 end

data/lib/relaton/registry.rb

@@ -8,7 +8,7 @@ module Relaton
     SUPPORTED_GEMS = %w[
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
-      relaton_ieee relaton_iho relaton_bipm relaton_ecma
+      relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie
     ].freeze
 
     include Singleton

data/lib/relaton/util.rb

@@ -4,7 +4,7 @@ module Relaton
       log_types = Relaton.configuration.logs.map(&:to_s) || []
 
       if log_types.include?(type.to_s)
-        puts(message)
+        warn(message)
       end
     end
   end

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.7.2".freeze
+  VERSION = "1.7.7".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Relaton
+  # Workers poll.
+  class WorkersPool
+    def initialize(workers = 2, &_block)
+      num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(num_workers * 2)
+      @threads = Array.new num_workers do
+        Thread.new do
+          while item = @queue.pop
+            yield(item)
+          end
+        end
+      end
+    end
+
+    def <<(item)
+      @queue << item
+      self
+    end
+  end
+end

data/relaton.gemspec

@@ -32,9 +32,10 @@ Gem::Specification.new do |spec|
   # spec.add_dependency "algoliasearch"
   spec.add_dependency "relaton-bipm", "~> 1.7.0"
   spec.add_dependency "relaton-calconnect", "~> 1.7.0"
-  spec.add_dependency "relaton-ecma", "1.7.pre1"
+  spec.add_dependency "relaton-cie", "~> 1.7.pre1"
+  spec.add_dependency "relaton-ecma", "~> 1.7.pre1"
   spec.add_dependency "relaton-gb", "~> 1.7.0"
-  spec.add_dependency "relaton-iec", ">= 1.7.3"
+  spec.add_dependency "relaton-iec", ">= 1.7.6"
   spec.add_dependency "relaton-ieee", "~> 1.7.0"
   spec.add_dependency "relaton-ietf", "~> 1.7.0"
   spec.add_dependency "relaton-iho", "~> 1.7.0"
@@ -47,15 +48,15 @@ Gem::Specification.new do |spec|
   spec.add_dependency "relaton-w3c", "~> 1.7.0"
 
   spec.add_development_dependency "byebug", "~> 11.0"
-  spec.add_development_dependency "debase"
+  # spec.add_development_dependency "debase"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-rspec", "~> 4.7"
   spec.add_development_dependency "pry-byebug", "~> 3.9.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
   spec.add_development_dependency "rubocop", "= 0.54.0"
-  spec.add_development_dependency "ruby-debug-ide"
+  # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 5"

data/spec/relaton/config_spec.rb

@@ -17,6 +17,6 @@ RSpec.describe Relaton::Config do
   end
 
   def restore_to_default_config
-    Relaton.configuration.logs = %i(warning error)
+    Relaton.configuration.logs = %i(info error)
   end
 end

data/spec/relaton/db_spec.rb

@@ -1,6 +1,121 @@
 RSpec.describe Relaton::Db do
   before(:each) { FileUtils.rm_rf %w[testcache testcache2] }
 
+  context "modifing database" do
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
+      db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
+    end
+
+    context "move to new dir" do
+      let(:db) { Relaton::Db.new "global_cache", "local_cache" }
+
+      after(:each) do
+        FileUtils.rm_rf "global_cache"
+        FileUtils.rm_rf "local_cache"
+      end
+
+      it "global cache" do
+        expect(File.exist?("global_cache")).to be true
+        expect(db.mv("testcache")).to eq "testcache"
+        expect(File.exist?("testcache")).to be true
+        expect(File.exist?("global_cache")).to be false
+      end
+
+      it "local cache" do
+        expect(File.exist?("local_cache")).to be true
+        expect(db.mv("testcache2", type: :local)).to eq "testcache2"
+        expect(File.exist?("testcache2")).to be true
+        expect(File.exist?("local_cache")).to be false
+      end
+    end
+
+    it "warn if moving in existed dir" do
+      expect(File).to receive(:exist?).with("new_cache_dir")
+        .and_return true
+      expect do
+        expect(db.mv("new_cache_dir")).to be_nil
+      end.to output(/\[relaton\] WARNING: target directory exists/).to_stderr
+    end
+
+    it "clear" do
+      expect(File.exist?("testcache/iso")).to be true
+      expect(File.exist?("testcache2/iso")).to be true
+      db.clear
+      expect(File.exist?("testcache/iso")).to be false
+      expect(File.exist?("testcache2/iso")).to be false
+    end
+  end
+
+  context "query in local DB" do
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
+      db.save_entry "ISO(ISO 123)", <<~DOC
+        <bibitem id='ISO123'>
+          <title>The first test</title><edition>2</edition><date type="published"><on>2011-10-12</on></date>
+        </bibitem>
+      DOC
+      db.save_entry "IEC(IEC 123)", <<~DOC
+        <bibitem id="IEC123">
+          <title>The second test</title><edition>1</edition><date type="published"><on>2015-12</on></date>
+        </bibitem>
+      DOC
+    end
+
+    after(:each) { db.clear }
+
+    it "one document" do
+      item = db.fetch_db "ISO((ISO 124)"
+      expect(item).to be_nil
+      item = db.fetch_db "ISO(ISO 123)"
+      expect(item).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    it "all documents" do
+      items = db.fetch_all
+      expect(items.size).to be 9
+      expect(items[7]).to be_instance_of RelatonIec::IecBibliographicItem
+      expect(items[8]).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    context "search for text" do
+      it do
+        items = db.fetch_all "test"
+        expect(items.size).to eq 2
+        items = db.fetch_all "first"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "in attributes" do
+        items = db.fetch_all "123"
+        expect(items.size).to eq 2
+        items = db.fetch_all "ISO"
+        expect(items.size).to eq 8
+        expect(items[7].id).to eq "ISO123"
+      end
+
+      it "and fail" do
+        items = db.fetch_all "bibitem"
+        expect(items.size).to eq 0
+      end
+
+      it "and edition" do
+        items = db.fetch_all "123", edition: "2"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "and year" do
+        items = db.fetch_all "123", year: 2015
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "IEC123"
+      end
+    end
+  end
+
   it "returns docid type" do
     db = Relaton::Db.new "testcache", "testcache2"
     expect(db.docid_type("CN(GB/T 1.1)")).to eq ["Chinese Standard", "GB/T 1.1"]
@@ -30,6 +145,36 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "fetch document with net retries" do
+    db = Relaton::Db.new nil, nil
+    expect(db.instance_variable_get(:@registry).processors[:relaton_ietf]).to receive(:get)
+      .and_raise(RelatonBib::RequestError).exactly(3).times
+    expect { db.fetch "RFC 8341", nil, retries: 3 }.to raise_error RelatonBib::RequestError
+  end
+
+  context "async fetch" do
+    let(:db) { Relaton::Db.new nil, nil }
+    let(:queue) { Queue.new }
+
+    it "success" do
+      result = nil
+      VCR.use_cassette "rfc_8341" do
+        db.fetch_async("RFC 8341") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_instance_of RelatonIetf::IetfBibliographicItem
+    end
+
+    it "prefix not found" do
+      result = ""
+      VCR.use_cassette "rfc_unsuccess" do
+        db.fetch_async("ABC 123456") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_nil
+    end
+  end
+
   context "fetch documents form static cache" do
     let(:db) { Relaton::Db.new nil, nil }
 

data/spec/relaton/processor_spec.rb

@@ -23,5 +23,9 @@ RSpec.describe Relaton::Processor do
     it "hash_to_bib method should be implemented" do
       expect { subject.hash_to_bib({}) }.to raise_error StandardError
     end
+
+    it "grammar_hash method should be implemented" do
+      expect { subject.grammar_hash }.to raise_error StandardError
+    end
   end
 end

data/spec/relaton/regirtry_spec.rb

@@ -3,7 +3,7 @@ RSpec.describe Relaton::Registry do
     stub_const "Relaton::Registry::SUPPORTED_GEMS", ["not_supported_gem"]
     expect { Relaton::Registry.clone.instance }.to output(
       /backend not_supported_gem not present/
-    ).to_stdout
+    ).to_stderr
   end
 
   it "finds ISO processor" do
@@ -15,63 +15,69 @@ RSpec.describe Relaton::Registry do
     expect(Relaton::Registry.instance.supported_processors).to include :relaton_iso
   end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("CN")).to be_instance_of RelatonGb::Processor
-  end
+  context "finds processor by type" do
+    it "CN" do
+      expect(Relaton::Registry.instance.by_type("CN")).to be_instance_of RelatonGb::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IEC")).to be_instance_of RelatonIec::Processor
-  end
+    it "IEC" do
+      expect(Relaton::Registry.instance.by_type("IEC")).to be_instance_of RelatonIec::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IETF")).to be_instance_of RelatonIetf::Processor
-  end
+    it "IETF" do
+      expect(Relaton::Registry.instance.by_type("IETF")).to be_instance_of RelatonIetf::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("ISO")).to be_instance_of RelatonIso::Processor
-  end
+    it "ISO" do
+      expect(Relaton::Registry.instance.by_type("ISO")).to be_instance_of RelatonIso::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("ITU")).to be_instance_of RelatonItu::Processor
-  end
+    it "ITU" do
+      expect(Relaton::Registry.instance.by_type("ITU")).to be_instance_of RelatonItu::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("NIST")).to be_instance_of RelatonNist::Processor
-  end
+    it "NIST" do
+      expect(Relaton::Registry.instance.by_type("NIST")).to be_instance_of RelatonNist::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("OGC")).to be_instance_of RelatonOgc::Processor
-  end
+    it "OGC" do
+      expect(Relaton::Registry.instance.by_type("OGC")).to be_instance_of RelatonOgc::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("CC")).to be_instance_of RelatonCalconnect::Processor
-  end
+    it "CC" do
+      expect(Relaton::Registry.instance.by_type("CC")).to be_instance_of RelatonCalconnect::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("OMG")).to be_instance_of RelatonOmg::Processor
-  end
+    it "OMG" do
+      expect(Relaton::Registry.instance.by_type("OMG")).to be_instance_of RelatonOmg::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("UN")).to be_instance_of RelatonUn::Processor
-  end
+    it "UN" do
+      expect(Relaton::Registry.instance.by_type("UN")).to be_instance_of RelatonUn::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("W3C")).to be_instance_of RelatonW3c::Processor
-  end
+    it "W3C" do
+      expect(Relaton::Registry.instance.by_type("W3C")).to be_instance_of RelatonW3c::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IEEE")).to be_instance_of RelatonIeee::Processor
-  end
+    it "IEEE" do
+      expect(Relaton::Registry.instance.by_type("IEEE")).to be_instance_of RelatonIeee::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IHO")).to be_instance_of RelatonIho::Processor
-  end
+    it "IHO" do
+      expect(Relaton::Registry.instance.by_type("IHO")).to be_instance_of RelatonIho::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("BIPM")).to be_instance_of RelatonBipm::Processor
-  end
+    it "BIPM" do
+      expect(Relaton::Registry.instance.by_type("BIPM")).to be_instance_of RelatonBipm::Processor
+    end
+
+    it "ECMA" do
+      expect(Relaton::Registry.instance.by_type("ECMA")).to be_instance_of RelatonEcma::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("ECMA")).to be_instance_of RelatonEcma::Processor
+    it "CIE" do
+      expect(Relaton::Registry.instance.by_type("CIE")).to be_instance_of RelatonCie::Processor
+    end
   end
 end