relaton 1.6.0 → 1.7.4

data/lib/relaton/db_cache.rb

@@ -1,4 +1,5 @@
 require "fileutils"
+require "timeout"
 
 module Relaton
   class DbCache
@@ -14,6 +15,20 @@ module Relaton
       # set_version # unless File.exist? file_version
     end
 
+    # Move caches to anothe dir
+    # @param new_dir [String, nil]
+    def mv(new_dir)
+      return unless new_dir && @ext == "xml"
+
+      FileUtils.mv dir, new_dir
+      @dir = new_dir
+    end
+
+    # Clear database
+    def clear
+      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # unless it's static DB
+    end
+
     # Save item
     # @param key [String]
     # @param value [String] Bibitem xml serialization
@@ -26,7 +41,7 @@ module Relaton
       prefix_dir = "#{@dir}/#{prefix(key)}"
       FileUtils::mkdir_p prefix_dir unless Dir.exist? prefix_dir
       set_version prefix_dir
-      File.write "#{filename(key)}.#{ext(value)}", value, encoding: "utf-8"
+      file_safe_write "#{filename(key)}.#{ext(value)}", value
     end
 
     # @param value [String]
@@ -65,7 +80,7 @@ module Relaton
       value = self[key]
       return unless value
 
-      if value =~ /^not_found/
+      if value.match? /^not_found/
         value.match(/\d{4}-\d{2}-\d{2}/).to_s
       else
         doc = Nokogiri::XML value
@@ -76,8 +91,9 @@ module Relaton
     # Returns all items
     # @return [Array<String>]
     def all
-      Dir.glob("#{@dir}/**/*.xml").sort.map do |f|
-        File.read(f, encoding: "utf-8")
+      Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").sort.map do |f|
+        content = File.read(f, encoding: "utf-8")
+        block_given? ? yield(f, content) : content
       end
     end
 
@@ -106,7 +122,7 @@ module Relaton
     def set_version(fdir)
       file_version = "#{fdir}/version"
       unless File.exist? file_version
-        File.write file_version, grammar_hash(fdir), encoding: "utf-8"
+        file_safe_write file_version, grammar_hash(fdir)
       end
       self
     end
@@ -159,7 +175,7 @@ module Relaton
     def filename(key)
       prefcode = key.downcase.match /^(?<prefix>[^\(]+)\((?<code>[^\)]+)/
       fn = if prefcode
-             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze("_")}"
+             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze('_')}"
            else
              key.gsub(/[-:\s]/, "_")
            end
@@ -189,6 +205,15 @@ module Relaton
       key.downcase.match(/^[^\(]+(?=\()/).to_s
     end
 
+    # @param file [String]
+    # @content [String]
+    def file_safe_write(file, content)
+      File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+        Timeout.timeout(10) { f.flock File::LOCK_EX }
+        f.write content
+      end
+    end
+
     class << self
       private
 
@@ -209,7 +234,7 @@ module Relaton
       # local_cache: local cache name; none created if nil; "relaton" created
       # if empty global_cache: boolean to create global_cache
       # flush_caches: flush caches
-      def init_bib_caches(opts)
+      def init_bib_caches(opts) # rubocop:disable Metrics/CyclomaticComplexity
         globalname = global_bibliocache_name if opts[:global_cache]
         localname = local_bibliocache_name(opts[:local_cache])
         localname = "relaton" if localname&.empty?

data/lib/relaton/processor.rb

@@ -20,5 +20,16 @@ module Relaton
     def hash_to_bib(_hash)
       raise "This is an abstract class!"
     end
+
+    def grammar_hash
+      raise "This is an abstract class!"
+    end
+
+    # Retuns default number of workers. Should be overraded by childred classes if need.
+    #
+    # @return [Integer] nuber of wokrers
+    def threads
+      10
+    end
   end
 end

data/lib/relaton/registry.rb

@@ -8,7 +8,7 @@ module Relaton
     SUPPORTED_GEMS = %w[
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
-      relaton_ieee relaton_iho relaton_bipm
+      relaton_ieee relaton_iho relaton_bipm relaton_ecma
     ].freeze
 
     include Singleton

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.6.0".freeze
+  VERSION = "1.7.4".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Relaton
+  # Workers poll.
+  class WorkersPool
+    def initialize(workers = 2, &_block)
+      num_workers = workers < 2 ? 2 : workers
+      @queue = SizedQueue.new(num_workers * 2)
+      @threads = Array.new num_workers do
+        Thread.new do
+          while item = @queue.pop
+            yield(item)
+          end
+        end
+      end
+    end
+
+    def <<(item)
+      @queue << item
+      self
+    end
+  end
+end

data/localcache/iec/iec_60050_102_2007.xml

@@ -0,0 +1,58 @@
+<bibdata type="standard">
+  <fetched>2021-02-03</fetched>
+  <title type="title-main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV)</title>
+  <title type="title-part" format="text/plain" language="en" script="Latn">Part 102: Mathematics -- General concepts and linear algebra</title>
+  <title type="main" format="text/plain" language="en" script="Latn">International Electrotechnical Vocabulary (IEV) - Part 102: Mathematics -- General concepts and linear algebra</title>
+  <uri type="src">https://webstore.iec.ch/publication/160</uri>
+  <uri type="obp">/preview/info_iec60050-102%7Bed1.0%7Db.pdf</uri>
+  <docidentifier type="IEC">IEC 60050-102:2007</docidentifier>
+  <docidentifier type="URN">urn:iec:std:iec:60050-102:2007:::en</docidentifier>
+  <date type="published">
+    <on>2007-08-27</on>
+  </date>
+  <contributor>
+    <role type="publisher"/>
+    <organization>
+      <name>International Electrotechnical Commission</name>
+      <abbreviation>IEC</abbreviation>
+      <uri>www.iec.ch</uri>
+    </organization>
+  </contributor>
+  <edition>1.0</edition>
+  <language>en</language>
+  <script>Latn</script>
+  <abstract format="text/plain" language="en" script="Latn">This part of IEC 60050 gives the general mathematical terminology used in the fields of electricity, electronics and telecommunications, together with basic concepts in linear algebra. It maintains a clear distinction between mathematical concepts and physical concepts, even if some terms are used in both cases. Another part will deal with functions.&#13;
+It has the status of a horizontal standard in accordance with IEC Guide 108.</abstract>
+  <status>
+    <stage>60</stage>
+    <substage>60</substage>
+  </status>
+  <copyright>
+    <from>2007</from>
+    <owner>
+      <organization>
+        <name>International Electrotechnical Commission</name>
+        <abbreviation>IEC</abbreviation>
+        <uri>www.iec.ch</uri>
+      </organization>
+    </owner>
+  </copyright>
+  <place>Geneva</place>
+  <ext>
+    <doctype>international-standard</doctype>
+    <editorialgroup>
+      <technical-committee number="1" type="technicalCommittee">TC 1 - Terminology</technical-committee>
+    </editorialgroup>
+    <ics>
+      <code>01.040.07</code>
+      <text>Natural and applied sciences (Vocabularies)</text>
+    </ics>
+    <ics>
+      <code>07.020</code>
+      <text>Mathematics</text>
+    </ics>
+    <structuredidentifier type="IEC">
+      <project-number>60050</project-number>
+    </structuredidentifier>
+  </ext>
+</bibdata>

data/localcache/iec/version

@@ -0,0 +1 @@
+407d6f5afdc9ae9ea7abe6e84d27f40b

data/relaton.gemspec

@@ -30,20 +30,21 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
 
   # spec.add_dependency "algoliasearch"
-  spec.add_dependency "relaton-bipm", "~> 1.6.0"
-  spec.add_dependency "relaton-calconnect", "~> 1.6.0"
-  spec.add_dependency "relaton-gb", "~> 1.6.0"
-  spec.add_dependency "relaton-iec", "~> 1.6.0"
-  spec.add_dependency "relaton-ieee", "~> 1.6.0"
-  spec.add_dependency "relaton-ietf", "~> 1.6.0"
-  spec.add_dependency "relaton-iho", "~> 1.6.0"
-  spec.add_dependency "relaton-iso", "~> 1.6.0"
-  spec.add_dependency "relaton-itu", "~> 1.6.0"
-  spec.add_dependency "relaton-nist", "~> 1.6.0"
-  spec.add_dependency "relaton-ogc", "~> 1.6.0"
-  spec.add_dependency "relaton-omg", "~> 1.6.0"
-  spec.add_dependency "relaton-un", "~> 1.6.0"
-  spec.add_dependency "relaton-w3c", "~> 1.6.0"
+  spec.add_dependency "relaton-bipm", "~> 1.7.0"
+  spec.add_dependency "relaton-calconnect", "~> 1.7.0"
+  spec.add_dependency "relaton-ecma", "1.7.pre1"
+  spec.add_dependency "relaton-gb", "~> 1.7.0"
+  spec.add_dependency "relaton-iec", ">= 1.7.6"
+  spec.add_dependency "relaton-ieee", "~> 1.7.0"
+  spec.add_dependency "relaton-ietf", "~> 1.7.0"
+  spec.add_dependency "relaton-iho", "~> 1.7.0"
+  spec.add_dependency "relaton-iso", ">= 1.7.1"
+  spec.add_dependency "relaton-itu", ">= 1.7.2"
+  spec.add_dependency "relaton-nist", ">= 1.7.1"
+  spec.add_dependency "relaton-ogc", "~> 1.7.0"
+  spec.add_dependency "relaton-omg", "~> 1.7.0"
+  spec.add_dependency "relaton-un", "~> 1.7.0"
+  spec.add_dependency "relaton-w3c", "~> 1.7.0"
 
   spec.add_development_dependency "byebug", "~> 11.0"
   spec.add_development_dependency "debase"

data/spec/relaton/db_cache_spec.rb

@@ -1,13 +1,38 @@
+require "fileutils"
+require "timeout"
+
 RSpec.describe Relaton::DbCache do
   it "creates default caches" do
     cache_path = File.expand_path("~/.relaton/cache")
     FileUtils.mv cache_path, "relaton1/cache", force: true
     FileUtils.rm_rf %w(relaton)
     Relaton::DbCache.init_bib_caches(
-      global_cache: true, local_cache: "", flush_caches: true,
+      global_cache: true, local_cache: "", flush_caches: true
     )
     expect(File.exist?(cache_path)).to be true
     expect(File.exist?("relaton")).to be true
     FileUtils.mv "relaton1/cache", cache_path if File.exist? "relaton1"
   end
+
+  it "write same file by concurent processes" do
+    dir = "testcache/iso"
+    FileUtils.mkdir_p dir unless File.exist? dir
+    file_name = File.join dir, "iso_123.xml"
+    file = File.open file_name, File::RDWR | File::CREAT, encoding: "UTF-8"
+    file.flock File::LOCK_EX
+    command = <<~RBY
+      require "relaton"
+      cache = Relaton::DbCache.new "testcache"
+      cache["ISO(ISO 123)"] = "test 1"
+    RBY
+    pid = spawn RbConfig.ruby, "-e #{command}"
+    sleep 0.1
+    file.write "test 2"
+    file.flock File::LOCK_UN
+    file.close
+    Process.waitpid pid, 0
+    expect($?.exitstatus).to eq 0
+    expect(File.read(file_name)).to eq "test 1"
+    FileUtils.rm_rf "testcache"
+  end
 end

data/spec/relaton/db_spec.rb

@@ -1,6 +1,98 @@
 RSpec.describe Relaton::Db do
   before(:each) { FileUtils.rm_rf %w[testcache testcache2] }
 
+  context "modifing database" do
+    it "move to new dir" do
+      db = Relaton::Db.new "global_cache", "local_cache"
+      db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
+      expect(File.exist?("global_cache")).to be true
+      expect(File.exist?("local_cache")).to be true
+      db.mv "testcache", "testcache2"
+      expect(File.exist?("testcache")).to be true
+      expect(File.exist?("global_cache")).to be false
+      expect(File.exist?("testcache2")).to be true
+      expect(File.exist?("local_cache")).to be false
+    end
+
+    it "clear" do
+      db = Relaton::Db.new "testcache", "testcache2"
+      db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
+      expect(File.exist?("testcache/iso")).to be true
+      expect(File.exist?("testcache2/iso")).to be true
+      db.clear
+      expect(File.exist?("testcache/iso")).to be false
+      expect(File.exist?("testcache2/iso")).to be false
+    end
+  end
+
+  context "query in local DB" do
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
+      db.save_entry "ISO(ISO 123)", <<~DOC
+        <bibitem id='ISO123'>
+          <title>The first test</title><edition>2</edition><date type="published"><on>2011-10-12</on></date>
+        </bibitem>
+      DOC
+      db.save_entry "IEC(IEC 123)", <<~DOC
+        <bibitem id="IEC123">
+          <title>The second test</title><edition>1</edition><date type="published"><on>2015-12</on></date>
+        </bibitem>
+      DOC
+    end
+
+    after(:each) { db.clear }
+
+    it "one document" do
+      item = db.fetch_db "ISO((ISO 124)"
+      expect(item).to be_nil
+      item = db.fetch_db "ISO(ISO 123)"
+      expect(item).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    it "all documents" do
+      items = db.fetch_all
+      expect(items.size).to be 9
+      expect(items[7]).to be_instance_of RelatonIec::IecBibliographicItem
+      expect(items[8]).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+
+    context "search for text" do
+      it do
+        items = db.fetch_all "test"
+        expect(items.size).to eq 2
+        items = db.fetch_all "first"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "in attributes" do
+        items = db.fetch_all "123"
+        expect(items.size).to eq 2
+        items = db.fetch_all "ISO"
+        expect(items.size).to eq 8
+        expect(items[7].id).to eq "ISO123"
+      end
+
+      it "and fail" do
+        items = db.fetch_all "bibitem"
+        expect(items.size).to eq 0
+      end
+
+      it "and edition" do
+        items = db.fetch_all "123", edition: "2"
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "ISO123"
+      end
+
+      it "and year" do
+        items = db.fetch_all "123", year: 2015
+        expect(items.size).to eq 1
+        expect(items[0].id).to eq "IEC123"
+      end
+    end
+  end
+
   it "returns docid type" do
     db = Relaton::Db.new "testcache", "testcache2"
     expect(db.docid_type("CN(GB/T 1.1)")).to eq ["Chinese Standard", "GB/T 1.1"]
@@ -30,6 +122,36 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "fetch document with net retries" do
+    db = Relaton::Db.new nil, nil
+    expect(db.instance_variable_get(:@registry).processors[:relaton_ietf]).to receive(:get)
+      .and_raise(RelatonBib::RequestError).exactly(3).times
+    expect { db.fetch "RFC 8341", nil, retries: 3 }.to raise_error RelatonBib::RequestError
+  end
+
+  context "async fetch" do
+    let(:db) { Relaton::Db.new nil, nil }
+    let(:queue) { Queue.new }
+
+    it "success" do
+      result = nil
+      VCR.use_cassette "rfc_8341" do
+        db.fetch_async("RFC 8341") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_instance_of RelatonIetf::IetfBibliographicItem
+    end
+
+    it "prefix not found" do
+      result = ""
+      VCR.use_cassette "rfc_unsuccess" do
+        db.fetch_async("ABC 123456") { |r| queue << r }
+        Timeout.timeout(5) { result = queue.pop }
+      end
+      expect(result).to be_nil
+    end
+  end
+
   context "fetch documents form static cache" do
     let(:db) { Relaton::Db.new nil, nil }
 

data/spec/relaton/processor_spec.rb

@@ -23,5 +23,9 @@ RSpec.describe Relaton::Processor do
     it "hash_to_bib method should be implemented" do
       expect { subject.hash_to_bib({}) }.to raise_error StandardError
     end
+
+    it "grammar_hash method should be implemented" do
+      expect { subject.grammar_hash }.to raise_error StandardError
+    end
   end
 end

data/spec/relaton/regirtry_spec.rb

@@ -2,13 +2,13 @@ RSpec.describe Relaton::Registry do
   it "outputs backend not present" do
     stub_const "Relaton::Registry::SUPPORTED_GEMS", ["not_supported_gem"]
     expect { Relaton::Registry.clone.instance }.to output(
-      /backend not_supported_gem not present/,
+      /backend not_supported_gem not present/
     ).to_stdout
   end
 
   it "finds ISO processor" do
-    expect(Relaton::Registry.instance.find_processor("relaton_iso")).
-      to be_instance_of RelatonIso::Processor
+    expect(Relaton::Registry.instance.find_processor("relaton_iso"))
+      .to be_instance_of RelatonIso::Processor
   end
 
   it "returns supported processors" do
@@ -70,4 +70,8 @@ RSpec.describe Relaton::Registry do
   it "finds processor by type" do
     expect(Relaton::Registry.instance.by_type("BIPM")).to be_instance_of RelatonBipm::Processor
   end
+
+  it "finds processor by type" do
+    expect(Relaton::Registry.instance.by_type("ECMA")).to be_instance_of RelatonEcma::Processor
+  end
 end