relaton 1.7.5 → 1.8.pre2

data/lib/relaton/util.rb

@@ -1,10 +1,12 @@
 module Relaton
   module Util
+    # @param message [String]
+    # @param type [Symbol] default :info
     def self.log(message, type = :info)
       log_types = Relaton.configuration.logs.map(&:to_s) || []
 
       if log_types.include?(type.to_s)
-        puts(message)
+        warn(message)
       end
     end
   end

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.7.5".freeze
+  VERSION = "1.8.pre2".freeze
 end

data/lib/relaton/workers_pool.rb

@@ -8,9 +8,7 @@ module Relaton
       @queue = SizedQueue.new(num_workers * 2)
       @threads = Array.new num_workers do
         Thread.new do
-          while item = @queue.pop
-            yield(item)
-          end
+          while item = @queue.pop; yield(item) end
         end
       end
     end

data/relaton.gemspec

@@ -27,25 +27,28 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.files         = `git ls-files`.split("\n")
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   # spec.add_dependency "algoliasearch"
-  spec.add_dependency "relaton-bipm", "~> 1.7.0"
-  spec.add_dependency "relaton-calconnect", "~> 1.7.0"
-  spec.add_dependency "relaton-cie", "1.7.pre1"
-  spec.add_dependency "relaton-ecma", "1.7.pre1"
-  spec.add_dependency "relaton-gb", "~> 1.7.0"
-  spec.add_dependency "relaton-iec", ">= 1.7.6"
-  spec.add_dependency "relaton-ieee", "~> 1.7.0"
-  spec.add_dependency "relaton-ietf", "~> 1.7.0"
-  spec.add_dependency "relaton-iho", "~> 1.7.0"
-  spec.add_dependency "relaton-iso", ">= 1.7.1"
-  spec.add_dependency "relaton-itu", ">= 1.7.2"
-  spec.add_dependency "relaton-nist", ">= 1.7.1"
-  spec.add_dependency "relaton-ogc", "~> 1.7.0"
-  spec.add_dependency "relaton-omg", "~> 1.7.0"
-  spec.add_dependency "relaton-un", "~> 1.7.0"
-  spec.add_dependency "relaton-w3c", "~> 1.7.0"
+  spec.add_dependency "aws-sdk-s3", "~> 1.96.0"
+  spec.add_dependency "relaton-bipm", "~> 1.8.0"
+  spec.add_dependency "relaton-bsi", "~> 1.8.0"
+  spec.add_dependency "relaton-calconnect", "~> 1.8.0"
+  spec.add_dependency "relaton-cen", "~> 1.8.pre1"
+  spec.add_dependency "relaton-cie", "~> 1.8.0"
+  spec.add_dependency "relaton-ecma", "~> 1.8.0"
+  spec.add_dependency "relaton-gb", "~> 1.8.0"
+  spec.add_dependency "relaton-iec", ">= 1.8.0"
+  spec.add_dependency "relaton-ieee", "~> 1.8.0"
+  spec.add_dependency "relaton-ietf", "~> 1.8.0"
+  spec.add_dependency "relaton-iho", "~> 1.8.0"
+  spec.add_dependency "relaton-iso", ">= 1.8.0"
+  spec.add_dependency "relaton-itu", ">= 1.8.0"
+  spec.add_dependency "relaton-nist", ">= 1.8.0"
+  spec.add_dependency "relaton-ogc", "~> 1.8.0"
+  spec.add_dependency "relaton-omg", "~> 1.8.0"
+  spec.add_dependency "relaton-un", "~> 1.8.0"
+  spec.add_dependency "relaton-w3c", "~> 1.8.0"
 
   spec.add_development_dependency "byebug", "~> 11.0"
   # spec.add_development_dependency "debase"
@@ -53,9 +56,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-rspec", "~> 4.7"
   spec.add_development_dependency "pry-byebug", "~> 3.9.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
-  spec.add_development_dependency "rubocop", "= 0.54.0"
+  spec.add_development_dependency "rubocop", "~> 1.17.0"
+  spec.add_development_dependency "rubocop-performance", "~> 1.11.0"
+  spec.add_development_dependency "rubocop-rails", "~> 2.10.0"
   # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"

data/spec/relaton/config_spec.rb

@@ -17,6 +17,6 @@ RSpec.describe Relaton::Config do
   end
 
   def restore_to_default_config
-    Relaton.configuration.logs = %i(warning error)
+    Relaton.configuration.logs = %i(info error)
   end
 end

data/spec/relaton/db_cache_spec.rb

@@ -6,8 +6,8 @@ RSpec.describe Relaton::DbCache do
     cache_path = File.expand_path("~/.relaton/cache")
     FileUtils.mv cache_path, "relaton1/cache", force: true
     FileUtils.rm_rf %w(relaton)
-    Relaton::DbCache.init_bib_caches(
-      global_cache: true, local_cache: "", flush_caches: true
+    Relaton::Db.init_bib_caches(
+      global_cache: true, local_cache: "", flush_caches: true,
     )
     expect(File.exist?(cache_path)).to be true
     expect(File.exist?("relaton")).to be true

data/spec/relaton/db_spec.rb

@@ -1,22 +1,62 @@
 RSpec.describe Relaton::Db do
   before(:each) { FileUtils.rm_rf %w[testcache testcache2] }
 
+  context "initialization in API mode" do
+    before(:each) { Relaton.configure { |c| c.api_mode = true } }
+    after(:each) { Relaton.configure { |c| c.api_mode = false } }
+
+    it do
+      db = Relaton::Db.new "cache", nil
+      cache = db.instance_variable_get :@db
+      expect(cache.dir).to eq "cache"
+    end
+
+    it "with default cache directory" do
+      db = Relaton::Db.init_bib_caches global_cache: true
+      cache = db.instance_variable_get :@db
+      expect(cache.dir).to eq "cache"
+    end
+  end
+
   context "modifing database" do
-    it "move to new dir" do
-      db = Relaton::Db.new "global_cache", "local_cache"
+    let(:db) { Relaton::Db.new "testcache", "testcache2" }
+
+    before(:each) do
       db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
-      expect(File.exist?("global_cache")).to be true
-      expect(File.exist?("local_cache")).to be true
-      db.mv "testcache", "testcache2"
-      expect(File.exist?("testcache")).to be true
-      expect(File.exist?("global_cache")).to be false
-      expect(File.exist?("testcache2")).to be true
-      expect(File.exist?("local_cache")).to be false
+    end
+
+    context "move to new dir" do
+      let(:db) { Relaton::Db.new "global_cache", "local_cache" }
+
+      after(:each) do
+        FileUtils.rm_rf "global_cache"
+        FileUtils.rm_rf "local_cache"
+      end
+
+      it "global cache" do
+        expect(File.exist?("global_cache")).to be true
+        expect(db.mv("testcache")).to eq "testcache"
+        expect(File.exist?("testcache")).to be true
+        expect(File.exist?("global_cache")).to be false
+      end
+
+      it "local cache" do
+        expect(File.exist?("local_cache")).to be true
+        expect(db.mv("testcache2", type: :local)).to eq "testcache2"
+        expect(File.exist?("testcache2")).to be true
+        expect(File.exist?("local_cache")).to be false
+      end
+    end
+
+    it "warn if moving in existed dir" do
+      expect(File).to receive(:exist?).with("new_cache_dir")
+        .and_return true
+      expect do
+        expect(db.mv("new_cache_dir")).to be_nil
+      end.to output(/\[relaton\] WARNING: target directory exists/).to_stderr
     end
 
     it "clear" do
-      db = Relaton::Db.new "testcache", "testcache2"
-      db.save_entry "ISO(ISO 123)", "<bibitem id='ISO123></bibitem>"
       expect(File.exist?("testcache/iso")).to be true
       expect(File.exist?("testcache2/iso")).to be true
       db.clear
@@ -134,12 +174,21 @@ RSpec.describe Relaton::Db do
     let(:queue) { Queue.new }
 
     it "success" do
-      result = nil
-      VCR.use_cassette "rfc_8341" do
-        db.fetch_async("RFC 8341") { |r| queue << r }
-        Timeout.timeout(5) { result = queue.pop }
+      refs = ["ITU-T G.993.5", "ITU-T G.994.1", "ITU-T H.264.1", "ITU-T H.740",
+              "ITU-T Y.1911", "ITU-T Y.2012", "ITU-T Y.2206", "ITU-T O.172",
+              "ITU-T G.780/Y.1351", "ITU-T G.711", "ITU-T G.1011"]
+      results = []
+      VCR.use_cassette "async_fetch", match_requests_on: %i[method uri body] do
+        refs.each do |ref|
+          db.fetch_async(ref) { |r| queue << r }
+        end
+        Timeout.timeout(60) do
+          refs.size.times { results << queue.pop }
+        end
+      end
+      results.each do |result|
+        expect(result).to be_instance_of RelatonItu::ItuBibliographicItem
       end
-      expect(result).to be_instance_of RelatonIetf::IetfBibliographicItem
     end
 
     it "prefix not found" do

data/spec/relaton/regirtry_spec.rb

@@ -2,8 +2,8 @@ RSpec.describe Relaton::Registry do
   it "outputs backend not present" do
     stub_const "Relaton::Registry::SUPPORTED_GEMS", ["not_supported_gem"]
     expect { Relaton::Registry.clone.instance }.to output(
-      /backend not_supported_gem not present/
-    ).to_stdout
+      /backend not_supported_gem not present/,
+    ).to_stderr
   end
 
   it "finds ISO processor" do
@@ -79,5 +79,13 @@ RSpec.describe Relaton::Registry do
     it "CIE" do
       expect(Relaton::Registry.instance.by_type("CIE")).to be_instance_of RelatonCie::Processor
     end
+
+    it "BSI" do
+      expect(Relaton::Registry.instance.by_type("BSI")).to be_instance_of RelatonBsi::Processor
+    end
+
+    it "CEN" do
+      expect(Relaton::Registry.instance.by_type("CEN")).to be_instance_of RelatonCen::Processor
+    end
   end
 end

data/spec/relaton/storage_spec.rb

@@ -0,0 +1,130 @@
+RSpec.describe Relaton::Storage do
+  context "API mode" do
+    before(:each) do
+      Relaton.configure do |config|
+        config.api_mode = true
+      end
+      Singleton.__init__ Relaton::Storage
+    end
+
+    after(:each) do
+      Relaton.configure do |config|
+        config.api_mode = false
+      end
+      Singleton.__init__ Relaton::Storage
+    end
+
+    let(:storage) { Relaton::Storage.instance }
+    let(:bucket) { "test-bucket" }
+
+    it "save document to S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc.xml"
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:head_object).with(bucket: bucket, key: "cahche/iso/version")
+      expect(client).to receive(:put_object).with(
+        bucket: bucket, key: key, body: body,
+        content_type: "text/plain; charset=utf-8"
+      )
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.save dir, key, body
+    end
+
+    it "read document from S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc"
+      client = double "AwsClient"
+      item = double "Item", key: "#{key}.xml"
+      list = double "List", contents: [item]
+      obj_body = double "Body", read: body
+      obj = double "Object", body: obj_body
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: "#{key}.").and_return list
+      expect(client).to receive(:get_object).with(bucket: bucket, key: "#{key}.xml").and_return obj
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      expect(storage.get(key)).to eq body
+    end
+
+    it "delete document from S3" do
+      key = "cache/iso/doc"
+      client = double "AwsClient"
+      item = double "Item", key: "#{key}.xml"
+      list = double "List", contents: [item]
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: "#{key}.").and_return list
+      expect(client).to receive(:delete_object).with(bucket: bucket, key: "#{key}.xml")
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.delete key
+    end
+
+    it "return all the documents from S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc.xml"
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      item = double "Item", key: key
+      list = double "List", contents: [item]
+      obj_body = double "Body", read: body
+      obj = double "Object", body: obj_body
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: dir).and_return list
+      expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_return obj
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      contents = storage.all dir
+      expect(contents).to eq [body]
+    end
+
+    it "set version" do
+      dir = "cahche/iso"
+      key = File.join dir, "version"
+      hash = Relaton::Registry.instance.by_type("iso").grammar_hash
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      # expect(ENV).to receive(:[]).and_call_original
+      error = Aws::S3::Errors::NotFound.new Seahorse::Client::RequestContext.new, ""
+      expect(client).to receive(:head_object).with(bucket: bucket, key: key)
+        .and_raise error
+      expect(client).to receive(:put_object).with(
+        bucket: bucket, key: key, body: hash,
+        content_type: "text/plain; charset=utf-8"
+      )
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.send :set_version, dir
+    end
+
+    context "check version" do
+      it "valid" do
+        dir = "cahche/iso"
+        key = File.join dir, "version"
+        hash = Relaton::Registry.instance.by_type("iso").grammar_hash
+        client = double "AwsClient"
+        obj_body = double "Body", read: hash
+        obj = double "Object", body: obj_body
+        expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_return obj
+        expect(Aws::S3::Client).to receive(:new).and_return client
+        expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket)
+        expect(storage.check_version?(dir)).to be true
+      end
+
+      it "version file doesn't exist" do
+        dir = "cahche/iso"
+        key = File.join dir, "version"
+        # hash = "invalid"
+        client = double "AwsClient"
+        # obj_body = double "Body", read: hash
+        # obj = double "Object", body: obj_body
+        error = Aws::S3::Errors::NoSuchKey.new Seahorse::Client::RequestContext.new, ""
+        expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_raise error
+        expect(Aws::S3::Client).to receive(:new).and_return client
+        expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket)
+        expect(storage.check_version?(dir)).to be false
+      end
+    end
+  end
+end

data/spec/relaton_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Relaton::Db do
 
   it "rejects an illegal reference prefix" do
     expect { @db.fetch("XYZ XYZ", nil, {}) }.to output(
-      /does not have a recognised prefix/
+      /does not have a recognised prefix/,
     ).to_stderr
   end
 
@@ -38,7 +38,7 @@ RSpec.describe Relaton::Db do
           'type="standard">'
         testcache = Relaton::DbCache.new "testcache"
         expect(
-          testcache.valid_entry?("ISO(ISO 19133:2005)", Date.today.year.to_s)
+          testcache.valid_entry?("ISO(ISO 19133:2005)", Date.today.year.to_s),
         ).to eq Date.today.year.to_s
       end
     end
@@ -60,8 +60,8 @@ RSpec.describe Relaton::Db do
     end
 
     it "gets sn ISO/AWI reference" do
-      VCR.use_cassette "iso_awi_24229" do
-        bib = @db.fetch "ISO/AWI 24229"
+      VCR.use_cassette "iso_awi_14093" do
+        bib = @db.fetch "ISO/AWI 14093"
         expect(bib).not_to be_nil
       end
     end
@@ -225,7 +225,7 @@ RSpec.describe Relaton::Db do
   end
 
   it "get OMG reference" do
-    VCR.use_cassette "ogm_ami4ccm_1_0" do
+    VCR.use_cassette "omg_ami4ccm_1_0" do
       bib = @db.fetch "OMG AMI4CCM 1.0", nil, {}
       expect(bib).to be_instance_of RelatonOmg::OmgBibliographicItem
     end
@@ -278,6 +278,20 @@ RSpec.describe Relaton::Db do
     end
   end
 
+  it "get BSI reference" do
+    VCR.use_cassette "bsi_bs_en_iso_8848" do
+      bib = @db.fetch "BSI BS EN ISO 8848"
+      expect(bib).to be_instance_of RelatonBsi::BsiBibliographicItem
+    end
+  end
+
+  it "get CEN reference" do
+    VCR.use_cassette "cen_en_10160_1999" do
+      bib = @db.fetch "CEN EN 10160:1999"
+      expect(bib).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+    end
+  end
+
   context "get combined documents" do
     context "ISO" do
       it "included" do
@@ -331,17 +345,17 @@ RSpec.describe Relaton::Db do
           expect(bib.relation[0].bibitem.docidentifier[0].id).to eq "ITU-T G.989.2"
           expect(bib.relation[1].type).to eq "complements"
           expect(bib.relation[1].description.content).to eq "amendment"
-          expect(bib.relation[1].bibitem.docidentifier[0].id).to eq "ITU-T G.989.2/Amd 1"
+          expect(bib.relation[1].bibitem.docidentifier[0].id).to eq "ITU-T G.989.2 Amd 1"
           expect(bib.relation[2].type).to eq "complements"
           expect(bib.relation[2].description.content).to eq "amendment"
-          expect(bib.relation[2].bibitem.docidentifier[0].id).to eq "ITU-T G.989.2/Amd 2"
+          expect(bib.relation[2].bibitem.docidentifier[0].id).to eq "ITU-T G.989.2 Amd 2"
         end
       end
     end
 
     context "HIST" do
       it "included" do
-        VCR.use_cassette "hist_cmbined_included" do
+        VCR.use_cassette "hist_combined_included" do
           bib = @db.fetch "NIST SP 800-38A, Add"
           expect(bib.docidentifier[0].id).to eq "NIST SP 800-38A, Add"
           expect(bib.relation[0].type).to eq "updates"
@@ -357,7 +371,7 @@ RSpec.describe Relaton::Db do
   context "version control" do
     before(:each) { @db.save_entry "iso(test_key)", "<bibitem><title>test_value</title></bibitem>" }
 
-    it "shoudn't clear cacho if version isn't changed" do
+    it "shoudn't clear cache if version isn't changed" do
       testcache = @db.instance_variable_get :@db
       expect(testcache.all).to be_any
       testcache = @db.instance_variable_get :@local_db
@@ -368,11 +382,45 @@ RSpec.describe Relaton::Db do
       expect(File.read("testcache/iso/version", encoding: "UTF-8")).not_to eq "new_version"
       expect(File.read("testcache2/iso/version", encoding: "UTF-8")).not_to eq "new_version"
       processor = double
-      expect(processor).to receive(:grammar_hash).and_return("new_version").exactly(4).times
-      expect(Relaton::Registry.instance).to receive(:by_type).and_return(processor).exactly(4).times
+      expect(processor).to receive(:grammar_hash).and_return("new_version").exactly(2).times
+      expect(Relaton::Registry.instance).to receive(:by_type).and_return(processor).exactly(2).times
       Relaton::Db.new "testcache", "testcache2"
-      expect(File.read("testcache/iso/version", encoding: "UTF-8")).to eq "new_version"
-      expect(File.read("testcache2/iso/version", encoding: "UTF-8")).to eq "new_version"
+      expect(File.exist?("testcache/iso/version")).to eq false
+      expect(File.exist?("testcache2/iso/version")).to eq false
+    end
+  end
+
+  context "api.relaton.org" do
+    before(:each) do
+      Relaton.configure do |config|
+        config.use_api = true
+        config.api_host = "http://0.0.0.0:9292"
+      end
+    end
+
+    after(:each) do
+      Relaton.configure do |config|
+        config.use_api = false
+      end
+    end
+
+    it "get document" do
+      VCR.use_cassette "api_relaton_org", re_record_interval: nil do
+        bib = @db.fetch "ISO 19115-2", "2019"
+        expect(bib).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+      end
+    end
+
+    it "if unavailable then get document directly" do
+      expect(Net::HTTP).to receive(:get_response).and_wrap_original do |m, *args|
+        raise Errno::ECONNREFUSED if args[0].host == "0.0.0.0"
+
+        m.call(*args)
+      end.at_least :once
+      VCR.use_cassette "api_relaton_org_unavailable" do
+        bib = @db.fetch "ISO 19115-2", "2019"
+        expect(bib).to be_instance_of RelatonIsoBib::IsoBibliographicItem
+      end
     end
   end
 end