relaton 1.6.0 → 1.7.pre7

data/lib/relaton/db_cache.rb

@@ -1,4 +1,5 @@
 require "fileutils"
+require "timeout"
 
 module Relaton
   class DbCache
@@ -14,6 +15,26 @@ module Relaton
       # set_version # unless File.exist? file_version
     end
 
+    # Move caches to anothe dir
+    # @param new_dir [String, nil]
+    # @return [String, nil]
+    def mv(new_dir)
+      return unless new_dir && @ext == "xml"
+
+      if File.exist? new_dir
+        warn "[relaton] WARNING: target directory exists \"#{new_dir}\""
+        return
+      end
+
+      FileUtils.mv dir, new_dir
+      @dir = new_dir
+    end
+
+    # Clear database
+    def clear
+      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # unless it's static DB
+    end
+
     # Save item
     # @param key [String]
     # @param value [String] Bibitem xml serialization
@@ -26,7 +47,7 @@ module Relaton
       prefix_dir = "#{@dir}/#{prefix(key)}"
       FileUtils::mkdir_p prefix_dir unless Dir.exist? prefix_dir
       set_version prefix_dir
-      File.write "#{filename(key)}.#{ext(value)}", value, encoding: "utf-8"
+      file_safe_write "#{filename(key)}.#{ext(value)}", value
     end
 
     # @param value [String]
@@ -65,7 +86,7 @@ module Relaton
       value = self[key]
       return unless value
 
-      if value =~ /^not_found/
+      if value.match? /^not_found/
         value.match(/\d{4}-\d{2}-\d{2}/).to_s
       else
         doc = Nokogiri::XML value
@@ -76,8 +97,9 @@ module Relaton
     # Returns all items
     # @return [Array<String>]
     def all
-      Dir.glob("#{@dir}/**/*.xml").sort.map do |f|
-        File.read(f, encoding: "utf-8")
+      Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").sort.map do |f|
+        content = File.read(f, encoding: "utf-8")
+        block_given? ? yield(f, content) : content
       end
     end
 
@@ -106,7 +128,7 @@ module Relaton
     def set_version(fdir)
       file_version = "#{fdir}/version"
       unless File.exist? file_version
-        File.write file_version, grammar_hash(fdir), encoding: "utf-8"
+        file_safe_write file_version, grammar_hash(fdir)
       end
       self
     end
@@ -159,7 +181,7 @@ module Relaton
     def filename(key)
       prefcode = key.downcase.match /^(?<prefix>[^\(]+)\((?<code>[^\)]+)/
       fn = if prefcode
-             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze("_")}"
+             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze('_')}"
            else
              key.gsub(/[-:\s]/, "_")
            end
@@ -189,6 +211,15 @@ module Relaton
       key.downcase.match(/^[^\(]+(?=\()/).to_s
     end
 
+    # @param file [String]
+    # @content [String]
+    def file_safe_write(file, content)
+      File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+        Timeout.timeout(10) { f.flock File::LOCK_EX }
+        f.write content
+      end
+    end
+
     class << self
       private
 
@@ -209,7 +240,7 @@ module Relaton
       # local_cache: local cache name; none created if nil; "relaton" created
       # if empty global_cache: boolean to create global_cache
       # flush_caches: flush caches
-      def init_bib_caches(opts)
+      def init_bib_caches(opts) # rubocop:disable Metrics/CyclomaticComplexity
         globalname = global_bibliocache_name if opts[:global_cache]
         localname = local_bibliocache_name(opts[:local_cache])
         localname = "relaton" if localname&.empty?

data/lib/relaton/processor.rb

@@ -20,5 +20,16 @@ module Relaton
     def hash_to_bib(_hash)
       raise "This is an abstract class!"
     end
+
+    def grammar_hash
+      raise "This is an abstract class!"
+    end
+
+    # Retuns default number of workers. Should be overraded by childred classes if need.
+    #
+    # @return [Integer] nuber of wokrers
+    def threads
+      10
+    end
   end
 end

data/lib/relaton/registry.rb

@@ -8,7 +8,7 @@ module Relaton
     SUPPORTED_GEMS = %w[
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
-      relaton_ieee relaton_iho relaton_bipm
+      relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie
     ].freeze
 
     include Singleton

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.6.0".freeze
+  VERSION = "1.7.pre7".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Relaton
+  # Workers poll.
+  class WorkersPool
+    def initialize(workers = 2, &_block)
+      num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(num_workers * 2)
+      @threads = Array.new num_workers do
+        Thread.new do
+          while item = @queue.pop
+            yield(item)
+          end
+        end
+      end
+    end
+
+    def <<(item)
+      @queue << item
+      self
+    end
+  end
+end

data/relaton.gemspec

@@ -30,23 +30,25 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
 
   # spec.add_dependency "algoliasearch"
-  spec.add_dependency "relaton-bipm", "~> 1.6.0"
-  spec.add_dependency "relaton-calconnect", "~> 1.6.0"
-  spec.add_dependency "relaton-gb", "~> 1.6.0"
-  spec.add_dependency "relaton-iec", "~> 1.6.0"
-  spec.add_dependency "relaton-ieee", "~> 1.6.0"
-  spec.add_dependency "relaton-ietf", "~> 1.6.0"
-  spec.add_dependency "relaton-iho", "~> 1.6.0"
-  spec.add_dependency "relaton-iso", "~> 1.6.0"
-  spec.add_dependency "relaton-itu", "~> 1.6.0"
-  spec.add_dependency "relaton-nist", "~> 1.6.0"
-  spec.add_dependency "relaton-ogc", "~> 1.6.0"
-  spec.add_dependency "relaton-omg", "~> 1.6.0"
-  spec.add_dependency "relaton-un", "~> 1.6.0"
-  spec.add_dependency "relaton-w3c", "~> 1.6.0"
+  spec.add_dependency "relaton-bipm", "~> 1.7.0"
+  spec.add_dependency "relaton-calconnect", "~> 1.7.0"
+  spec.add_dependency "relaton-cie", "~> 1.7.pre1"
+  spec.add_dependency "relaton-ecma", "~> 1.7.pre1"
+  spec.add_dependency "relaton-gb", "~> 1.7.0"
+  spec.add_dependency "relaton-iec", ">= 1.7.6"
+  spec.add_dependency "relaton-ieee", "~> 1.7.0"
+  spec.add_dependency "relaton-ietf", "~> 1.7.0"
+  spec.add_dependency "relaton-iho", "~> 1.7.0"
+  spec.add_dependency "relaton-iso", ">= 1.7.1"
+  spec.add_dependency "relaton-itu", ">= 1.7.2"
+  spec.add_dependency "relaton-nist", ">= 1.7.1"
+  spec.add_dependency "relaton-ogc", "~> 1.7.0"
+  spec.add_dependency "relaton-omg", "~> 1.7.0"
+  spec.add_dependency "relaton-un", "~> 1.7.0"
+  spec.add_dependency "relaton-w3c", "~> 1.7.0"
 
   spec.add_development_dependency "byebug", "~> 11.0"
-  spec.add_development_dependency "debase"
+  # spec.add_development_dependency "debase"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-rspec", "~> 4.7"
@@ -54,7 +56,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.6"
   spec.add_development_dependency "rubocop", "= 0.54.0"
-  spec.add_development_dependency "ruby-debug-ide"
+  # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "vcr", "~> 5"

data/spec/relaton/db_cache_spec.rb

@@ -1,13 +1,38 @@
+require "fileutils"
+require "timeout"
+
 RSpec.describe Relaton::DbCache do
   it "creates default caches" do
     cache_path = File.expand_path("~/.relaton/cache")
     FileUtils.mv cache_path, "relaton1/cache", force: true
     FileUtils.rm_rf %w(relaton)
     Relaton::DbCache.init_bib_caches(
-      global_cache: true, local_cache: "", flush_caches: true,
+      global_cache: true, local_cache: "", flush_caches: true
     )
     expect(File.exist?(cache_path)).to be true
     expect(File.exist?("relaton")).to be true
     FileUtils.mv "relaton1/cache", cache_path if File.exist? "relaton1"
   end
+
+  it "write same file by concurent processes" do
+    dir = "testcache/iso"
+    FileUtils.mkdir_p dir unless File.exist? dir
+    file_name = File.join dir, "iso_123.xml"
+    file = File.open file_name, File::RDWR | File::CREAT, encoding: "UTF-8"
+    file.flock File::LOCK_EX
+    command = <<~RBY
+      require "relaton"
+      cache = Relaton::DbCache.new "testcache"
+      cache["ISO(ISO 123)"] = "test 1"
+    RBY
+    pid = spawn RbConfig.ruby, "-e #{command}"
+    sleep 0.1
+    file.write "test 2"
+    file.flock File::LOCK_UN
+    file.close
+    Process.waitpid pid, 0
+    expect($?.exitstatus).to eq 0
+    expect(File.read(file_name)).to eq "test 1"
+    FileUtils.rm_rf "testcache"
+  end
 end

data/spec/relaton/db_spec.rb

@@ -1,6 +1,121 @@
 RSpec.describe Relaton::Db do
   before(:each) { FileUtils.rm_rf %w[testcache testcache2] }
 
+  context "modifing database" do
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
+      db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
+    end
+
+    context "move to new dir" do
+      let(:db) { Relaton::Db.new "global_cache", "local_cache" }
+
+      after(:each) do
+        FileUtils.rm_rf "global_cache"
+        FileUtils.rm_rf "local_cache"
+      end
+
+      it "global cache" do
+        expect(File.exist?("global_cache")).to be true
+        expect(db.mv("testcache")).to eq "testcache"
+        expect(File.exist?("testcache")).to be true
+        expect(File.exist?("global_cache")).to be false
+      end
+
+      it "local cache" do
+        expect(File.exist?("local_cache")).to be true
+        expect(db.mv("testcache2", type: :local)).to eq "testcache2"
+        expect(File.exist?("testcache2")).to be true
+        expect(File.exist?("local_cache")).to be false
+      end
+    end
+
+    it "warn if moving in existed dir" do
+      expect(File).to receive(:exist?).with("new_cache_dir")
+        .and_return true
+      expect do
+        expect(db.mv("new_cache_dir")).to be_nil
+      end.to output(/\[relaton\] WARNING: target directory exists/).to_stderr
+    end
+
+    it "clear" do
+      expect(File.exist?("testcache/iso")).to be true
+      expect(File.exist?("testcache2/iso")).to be true
+      db.clear
+      expect(File.exist?("testcache/iso")).to be false
+      expect(File.exist?("testcache2/iso")).to be false
+    end
+  end
+
+  context "query in local DB" do
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
+      db.save_entry "ISO(ISO 123)", <<~DOC
+        <bibitem id='ISO123'>
+          <title>The first test</title><edition>2</edition><date type="published"><on>2011-10-12</on></date>
+        </bibitem>
+      DOC
+      db.save_entry "IEC(IEC 123)", <<~DOC
+        <bibitem id="IEC123">
+          <title>The second test</title><edition>1</edition><date type="published"><on>2015-12</on></date>
+        </bibitem>
+      DOC
+    end
+
+    after(:each) { db.clear }
+
+    it "one document" do
+      item = db.fetch_db "ISO((ISO 124)"
+      expect(item).to be_nil
+      item = db.fetch_db "ISO(ISO 123)"
+      expect(item).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    it "all documents" do
+      items = db.fetch_all
+      expect(items.size).to be 9
+      expect(items[7]).to be_instance_of RelatonIec::IecBibliographicItem
+      expect(items[8]).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    context "search for text" do
+      it do
+        items = db.fetch_all "test"
+        expect(items.size).to eq 2
+        items = db.fetch_all "first"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "in attributes" do
+        items = db.fetch_all "123"
+        expect(items.size).to eq 2
+        items = db.fetch_all "ISO"
+        expect(items.size).to eq 8
+        expect(items[7].id).to eq "ISO123"
+      end
+
+      it "and fail" do
+        items = db.fetch_all "bibitem"
+        expect(items.size).to eq 0
+      end
+
+      it "and edition" do
+        items = db.fetch_all "123", edition: "2"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "and year" do
+        items = db.fetch_all "123", year: 2015
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "IEC123"
+      end
+    end
+  end
+
   it "returns docid type" do
     db = Relaton::Db.new "testcache", "testcache2"
     expect(db.docid_type("CN(GB/T 1.1)")).to eq ["Chinese Standard", "GB/T 1.1"]
@@ -30,6 +145,36 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "fetch document with net retries" do
+    db = Relaton::Db.new nil, nil
+    expect(db.instance_variable_get(:@registry).processors[:relaton_ietf]).to receive(:get)
+      .and_raise(RelatonBib::RequestError).exactly(3).times
+    expect { db.fetch "RFC 8341", nil, retries: 3 }.to raise_error RelatonBib::RequestError
+  end
+
+  context "async fetch" do
+    let(:db) { Relaton::Db.new nil, nil }
+    let(:queue) { Queue.new }
+
+    it "success" do
+      result = nil
+      VCR.use_cassette "rfc_8341" do
+        db.fetch_async("RFC 8341") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_instance_of RelatonIetf::IetfBibliographicItem
+    end
+
+    it "prefix not found" do
+      result = ""
+      VCR.use_cassette "rfc_unsuccess" do
+        db.fetch_async("ABC 123456") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_nil
+    end
+  end
+
   context "fetch documents form static cache" do
     let(:db) { Relaton::Db.new nil, nil }
 

data/spec/relaton/processor_spec.rb

@@ -23,5 +23,9 @@ RSpec.describe Relaton::Processor do
     it "hash_to_bib method should be implemented" do
       expect { subject.hash_to_bib({}) }.to raise_error StandardError
     end
+
+    it "grammar_hash method should be implemented" do
+      expect { subject.grammar_hash }.to raise_error StandardError
+    end
   end
 end

data/spec/relaton/regirtry_spec.rb

@@ -2,72 +2,82 @@ RSpec.describe Relaton::Registry do
   it "outputs backend not present" do
     stub_const "Relaton::Registry::SUPPORTED_GEMS", ["not_supported_gem"]
     expect { Relaton::Registry.clone.instance }.to output(
-      /backend not_supported_gem not present/,
+      /backend not_supported_gem not present/
     ).to_stdout
   end
 
   it "finds ISO processor" do
-    expect(Relaton::Registry.instance.find_processor("relaton_iso")).
-      to be_instance_of RelatonIso::Processor
+    expect(Relaton::Registry.instance.find_processor("relaton_iso"))
+      .to be_instance_of RelatonIso::Processor
   end
 
   it "returns supported processors" do
     expect(Relaton::Registry.instance.supported_processors).to include :relaton_iso
   end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("CN")).to be_instance_of RelatonGb::Processor
-  end
+  context "finds processor by type" do
+    it "CN" do
+      expect(Relaton::Registry.instance.by_type("CN")).to be_instance_of RelatonGb::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IEC")).to be_instance_of RelatonIec::Processor
-  end
+    it "IEC" do
+      expect(Relaton::Registry.instance.by_type("IEC")).to be_instance_of RelatonIec::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IETF")).to be_instance_of RelatonIetf::Processor
-  end
+    it "IETF" do
+      expect(Relaton::Registry.instance.by_type("IETF")).to be_instance_of RelatonIetf::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("ISO")).to be_instance_of RelatonIso::Processor
-  end
+    it "ISO" do
+      expect(Relaton::Registry.instance.by_type("ISO")).to be_instance_of RelatonIso::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("ITU")).to be_instance_of RelatonItu::Processor
-  end
+    it "ITU" do
+      expect(Relaton::Registry.instance.by_type("ITU")).to be_instance_of RelatonItu::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("NIST")).to be_instance_of RelatonNist::Processor
-  end
+    it "NIST" do
+      expect(Relaton::Registry.instance.by_type("NIST")).to be_instance_of RelatonNist::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("OGC")).to be_instance_of RelatonOgc::Processor
-  end
+    it "OGC" do
+      expect(Relaton::Registry.instance.by_type("OGC")).to be_instance_of RelatonOgc::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("CC")).to be_instance_of RelatonCalconnect::Processor
-  end
+    it "CC" do
+      expect(Relaton::Registry.instance.by_type("CC")).to be_instance_of RelatonCalconnect::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("OMG")).to be_instance_of RelatonOmg::Processor
-  end
+    it "OMG" do
+      expect(Relaton::Registry.instance.by_type("OMG")).to be_instance_of RelatonOmg::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("UN")).to be_instance_of RelatonUn::Processor
-  end
+    it "UN" do
+      expect(Relaton::Registry.instance.by_type("UN")).to be_instance_of RelatonUn::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("W3C")).to be_instance_of RelatonW3c::Processor
-  end
+    it "W3C" do
+      expect(Relaton::Registry.instance.by_type("W3C")).to be_instance_of RelatonW3c::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IEEE")).to be_instance_of RelatonIeee::Processor
-  end
+    it "IEEE" do
+      expect(Relaton::Registry.instance.by_type("IEEE")).to be_instance_of RelatonIeee::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("IHO")).to be_instance_of RelatonIho::Processor
-  end
+    it "IHO" do
+      expect(Relaton::Registry.instance.by_type("IHO")).to be_instance_of RelatonIho::Processor
+    end
+
+    it "BIPM" do
+      expect(Relaton::Registry.instance.by_type("BIPM")).to be_instance_of RelatonBipm::Processor
+    end
+
+    it "ECMA" do
+      expect(Relaton::Registry.instance.by_type("ECMA")).to be_instance_of RelatonEcma::Processor
+    end
 
-  it "finds processor by type" do
-    expect(Relaton::Registry.instance.by_type("BIPM")).to be_instance_of RelatonBipm::Processor
+    it "CIE" do
+      expect(Relaton::Registry.instance.by_type("CIE")).to be_instance_of RelatonCie::Processor
+    end
   end
 end