relaton 1.7.9 → 1.8.pre2

data/lib/relaton/registry.rb

@@ -9,6 +9,7 @@ module Relaton
       relaton_gb relaton_iec relaton_ietf relaton_iso relaton_itu relaton_nist
       relaton_ogc relaton_calconnect relaton_omg relaton_un relaton_w3c
       relaton_ieee relaton_iho relaton_bipm relaton_ecma relaton_cie relaton_bsi
+      relaton_cen
     ].freeze
 
     include Singleton

data/lib/relaton/storage.rb

@@ -0,0 +1,186 @@
+require "aws-sdk-s3"
+
+module Relaton
+  #
+  # Man
+  #
+  class Storage
+    include Singleton
+
+    def initialize
+      if Relaton.configuration.api_mode
+        @s3 = Aws::S3::Client.new
+      end
+    end
+
+    #
+    # Save file to storage
+    #
+    # @param dir [String]
+    # @param key [String]
+    # @param value [String]
+    #
+    def save(dir, key, value)
+      set_version dir
+      if Relaton.configuration.api_mode then s3_write key, value
+      else file_safe_write key, value
+      end
+    end
+
+    #
+    # Returns all items
+    #
+    # @param dir [String]
+    #
+    # @return [Array<String>]
+    #
+    def all(dir, &block)
+      return all_s3 dir, &block if Relaton.configuration.api_mode
+
+      Dir.glob("#{dir}/**/*.{xml,yml,yaml}").sort.map do |f|
+        content = File.read(f, encoding: "utf-8")
+        block ? yield(f, content) : content
+      end
+    end
+
+    # Delete item
+    # @param key [String] path to file without extension
+    def delete(key)
+      f = search_ext(key)
+      if Relaton.configuration.api_mode && f
+        @s3.delete_object bucket: ENV["AWS_BUCKET"], key: f
+      elsif f then File.delete f
+      end
+    end
+
+    # Check if version of the DB match to the gem grammar hash.
+    # @param fdir [String] dir pathe to flavor cache
+    # @return [Boolean]
+    def check_version?(fdir)
+      file_version = "#{fdir}/version"
+      if Relaton.configuration.api_mode
+        check_version_s3? file_version, fdir
+      else
+        return false unless File.exist? fdir
+
+        v = File.read file_version, encoding: "utf-8"
+        v.strip == grammar_hash(fdir)
+      end
+    end
+
+    # Reads file by a key
+    #
+    # @param key [String]
+    # @return [String, NilClass]
+    def get(key, static: false)
+      return unless (f = search_ext(key, static: static))
+
+      if Relaton.configuration.api_mode && !static
+        s3_read f
+      else
+        File.read(f, encoding: "utf-8")
+      end
+    end
+
+    private
+
+    # Check if version of the DB match to the gem grammar hash.
+    # @param file_version [String] dir pathe to flover cache
+    # @param dir [String] fdir pathe to flavor cache
+    # @return [Boolean]
+    def check_version_s3?(file_version, fdir)
+      s3_read(file_version) == grammar_hash(fdir)
+    rescue Aws::S3::Errors::NoSuchKey
+      false
+    end
+
+    #
+    # Read file form AWS S#
+    #
+    # @param [String] key file name
+    #
+    # @return [String] content
+    #
+    def s3_read(key)
+      obj = @s3.get_object bucket: ENV["AWS_BUCKET"], key: key
+      obj.body.read
+    end
+
+    #
+    # Write file to AWS S3
+    #
+    # @param [String] key
+    # @param [String] value
+    #
+    def s3_write(key, value)
+      @s3.put_object(bucket: ENV["AWS_BUCKET"], key: key, body: value,
+                     content_type: "text/plain; charset=utf-8")
+    end
+
+    #
+    # Returns all items
+    #
+    # @param dir [String]
+    #
+    # @return [Array<String>]
+    #
+    def all_s3(dir)
+      list = @s3.list_objects_v2 bucket: ENV["AWS_BUCKET"], prefix: dir
+      list.contents.select { |i| i.key.match?(/\.xml$/) }.sort_by(&:key).map do |item|
+        content = s3_read item.key
+        block_given? ? yield(item.key, content) : content
+      end
+    end
+
+    #
+    # Checks if there is file with xml or txt extension and return filename with
+    # the extension.
+    #
+    # @param file [String]
+    # @return [String, NilClass]
+    def search_ext(file, static: false)
+      if Relaton.configuration.api_mode && !static
+        fs = @s3.list_objects_v2 bucket: ENV["AWS_BUCKET"], prefix: "#{file}."
+        fs.contents.first&.key
+      else
+        Dir["#{file}.*"].first
+      end
+    end
+
+    # Set version of the DB to the gem grammar hash.
+    # @param fdir [String] dir pathe to flover cache
+    def set_version(fdir)
+      file_version = "#{fdir}/version"
+      return set_version_s3 file_version, fdir if Relaton.configuration.api_mode
+
+      FileUtils::mkdir_p fdir unless Dir.exist?(fdir)
+      unless File.exist? file_version
+        file_safe_write file_version, grammar_hash(fdir)
+      end
+    end
+
+    # Set version of the DB to the gem grammar hash.
+    # @param fdir [String] dir pathe to flover cache
+    def set_version_s3(fver, dir)
+      @s3.head_object bucket: ENV["AWS_BUCKET"], key: fver
+    rescue Aws::S3::Errors::NotFound
+      s3_write fver, grammar_hash(dir)
+    end
+
+    # @param file [String]
+    # @content [String]
+    def file_safe_write(file, content)
+      File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+        Timeout.timeout(10) { f.flock File::LOCK_EX }
+        f.write content
+      end
+    end
+
+    # @param fdir [String] dir pathe to flover cache
+    # @return [String]
+    def grammar_hash(fdir)
+      type = fdir.split("/").last
+      Relaton::Registry.instance.by_type(type)&.grammar_hash
+    end
+  end
+end

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "1.7.9".freeze
+  VERSION = "1.8.pre2".freeze
 end

data/relaton.gemspec

@@ -27,26 +27,28 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.files         = `git ls-files`.split("\n")
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   # spec.add_dependency "algoliasearch"
-  spec.add_dependency "relaton-bipm", "~> 1.7.0"
-  spec.add_dependency "relaton-bsi", "~> 1.7.pre"
-  spec.add_dependency "relaton-calconnect", "~> 1.7.0"
-  spec.add_dependency "relaton-cie", "~> 1.7.pre1"
-  spec.add_dependency "relaton-ecma", "~> 1.7.pre1"
-  spec.add_dependency "relaton-gb", "~> 1.7.0"
-  spec.add_dependency "relaton-iec", ">= 1.7.6"
-  spec.add_dependency "relaton-ieee", "~> 1.7.0"
-  spec.add_dependency "relaton-ietf", "~> 1.7.0"
-  spec.add_dependency "relaton-iho", "~> 1.7.0"
-  spec.add_dependency "relaton-iso", ">= 1.7.1"
-  spec.add_dependency "relaton-itu", ">= 1.7.2"
-  spec.add_dependency "relaton-nist", ">= 1.7.1"
-  spec.add_dependency "relaton-ogc", "~> 1.7.0"
-  spec.add_dependency "relaton-omg", "~> 1.7.0"
-  spec.add_dependency "relaton-un", "~> 1.7.0"
-  spec.add_dependency "relaton-w3c", "~> 1.7.0"
+  spec.add_dependency "aws-sdk-s3", "~> 1.96.0"
+  spec.add_dependency "relaton-bipm", "~> 1.8.0"
+  spec.add_dependency "relaton-bsi", "~> 1.8.0"
+  spec.add_dependency "relaton-calconnect", "~> 1.8.0"
+  spec.add_dependency "relaton-cen", "~> 1.8.pre1"
+  spec.add_dependency "relaton-cie", "~> 1.8.0"
+  spec.add_dependency "relaton-ecma", "~> 1.8.0"
+  spec.add_dependency "relaton-gb", "~> 1.8.0"
+  spec.add_dependency "relaton-iec", ">= 1.8.0"
+  spec.add_dependency "relaton-ieee", "~> 1.8.0"
+  spec.add_dependency "relaton-ietf", "~> 1.8.0"
+  spec.add_dependency "relaton-iho", "~> 1.8.0"
+  spec.add_dependency "relaton-iso", ">= 1.8.0"
+  spec.add_dependency "relaton-itu", ">= 1.8.0"
+  spec.add_dependency "relaton-nist", ">= 1.8.0"
+  spec.add_dependency "relaton-ogc", "~> 1.8.0"
+  spec.add_dependency "relaton-omg", "~> 1.8.0"
+  spec.add_dependency "relaton-un", "~> 1.8.0"
+  spec.add_dependency "relaton-w3c", "~> 1.8.0"
 
   spec.add_development_dependency "byebug", "~> 11.0"
   # spec.add_development_dependency "debase"
@@ -56,7 +58,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug", "~> 3.9.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.6"
-  spec.add_development_dependency "rubocop", "= 0.54.0"
+  spec.add_development_dependency "rubocop", "~> 1.17.0"
+  spec.add_development_dependency "rubocop-performance", "~> 1.11.0"
+  spec.add_development_dependency "rubocop-rails", "~> 2.10.0"
   # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"

data/spec/relaton/db_cache_spec.rb

@@ -6,8 +6,8 @@ RSpec.describe Relaton::DbCache do
     cache_path = File.expand_path("~/.relaton/cache")
     FileUtils.mv cache_path, "relaton1/cache", force: true
     FileUtils.rm_rf %w(relaton)
-    Relaton::DbCache.init_bib_caches(
-      global_cache: true, local_cache: "", flush_caches: true
+    Relaton::Db.init_bib_caches(
+      global_cache: true, local_cache: "", flush_caches: true,
     )
     expect(File.exist?(cache_path)).to be true
     expect(File.exist?("relaton")).to be true

data/spec/relaton/db_spec.rb

@@ -1,6 +1,23 @@
 RSpec.describe Relaton::Db do
   before(:each) { FileUtils.rm_rf %w[testcache testcache2] }
 
+  context "initialization in API mode" do
+    before(:each) { Relaton.configure { |c| c.api_mode = true } }
+    after(:each) { Relaton.configure { |c| c.api_mode = false } }
+
+    it do
+      db = Relaton::Db.new "cache", nil
+      cache = db.instance_variable_get :@db
+      expect(cache.dir).to eq "cache"
+    end
+
+    it "with default cache directory" do
+      db = Relaton::Db.init_bib_caches global_cache: true
+      cache = db.instance_variable_get :@db
+      expect(cache.dir).to eq "cache"
+    end
+  end
+
   context "modifing database" do
     let(:db) { Relaton::Db.new "testcache", "testcache2" }
 

data/spec/relaton/regirtry_spec.rb

@@ -2,7 +2,7 @@ RSpec.describe Relaton::Registry do
   it "outputs backend not present" do
     stub_const "Relaton::Registry::SUPPORTED_GEMS", ["not_supported_gem"]
     expect { Relaton::Registry.clone.instance }.to output(
-      /backend not_supported_gem not present/
+      /backend not_supported_gem not present/,
     ).to_stderr
   end
 
@@ -79,5 +79,13 @@ RSpec.describe Relaton::Registry do
     it "CIE" do
       expect(Relaton::Registry.instance.by_type("CIE")).to be_instance_of RelatonCie::Processor
     end
+
+    it "BSI" do
+      expect(Relaton::Registry.instance.by_type("BSI")).to be_instance_of RelatonBsi::Processor
+    end
+
+    it "CEN" do
+      expect(Relaton::Registry.instance.by_type("CEN")).to be_instance_of RelatonCen::Processor
+    end
   end
 end

data/spec/relaton/storage_spec.rb

@@ -0,0 +1,130 @@
+RSpec.describe Relaton::Storage do
+  context "API mode" do
+    before(:each) do
+      Relaton.configure do |config|
+        config.api_mode = true
+      end
+      Singleton.__init__ Relaton::Storage
+    end
+
+    after(:each) do
+      Relaton.configure do |config|
+        config.api_mode = false
+      end
+      Singleton.__init__ Relaton::Storage
+    end
+
+    let(:storage) { Relaton::Storage.instance }
+    let(:bucket) { "test-bucket" }
+
+    it "save document to S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc.xml"
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:head_object).with(bucket: bucket, key: "cahche/iso/version")
+      expect(client).to receive(:put_object).with(
+        bucket: bucket, key: key, body: body,
+        content_type: "text/plain; charset=utf-8"
+      )
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.save dir, key, body
+    end
+
+    it "read document from S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc"
+      client = double "AwsClient"
+      item = double "Item", key: "#{key}.xml"
+      list = double "List", contents: [item]
+      obj_body = double "Body", read: body
+      obj = double "Object", body: obj_body
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: "#{key}.").and_return list
+      expect(client).to receive(:get_object).with(bucket: bucket, key: "#{key}.xml").and_return obj
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      expect(storage.get(key)).to eq body
+    end
+
+    it "delete document from S3" do
+      key = "cache/iso/doc"
+      client = double "AwsClient"
+      item = double "Item", key: "#{key}.xml"
+      list = double "List", contents: [item]
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: "#{key}.").and_return list
+      expect(client).to receive(:delete_object).with(bucket: bucket, key: "#{key}.xml")
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.delete key
+    end
+
+    it "return all the documents from S3" do
+      dir = "cahche/iso"
+      # file = "doc.xml"
+      body = "<doc>Test doc</doc>"
+      key = File.join dir, "doc.xml"
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      item = double "Item", key: key
+      list = double "List", contents: [item]
+      obj_body = double "Body", read: body
+      obj = double "Object", body: obj_body
+      expect(client).to receive(:list_objects_v2).with(bucket: bucket, prefix: dir).and_return list
+      expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_return obj
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      contents = storage.all dir
+      expect(contents).to eq [body]
+    end
+
+    it "set version" do
+      dir = "cahche/iso"
+      key = File.join dir, "version"
+      hash = Relaton::Registry.instance.by_type("iso").grammar_hash
+      client = double "AwsClient"
+      expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket).twice
+      # expect(ENV).to receive(:[]).and_call_original
+      error = Aws::S3::Errors::NotFound.new Seahorse::Client::RequestContext.new, ""
+      expect(client).to receive(:head_object).with(bucket: bucket, key: key)
+        .and_raise error
+      expect(client).to receive(:put_object).with(
+        bucket: bucket, key: key, body: hash,
+        content_type: "text/plain; charset=utf-8"
+      )
+      expect(Aws::S3::Client).to receive(:new).and_return client
+      storage.send :set_version, dir
+    end
+
+    context "check version" do
+      it "valid" do
+        dir = "cahche/iso"
+        key = File.join dir, "version"
+        hash = Relaton::Registry.instance.by_type("iso").grammar_hash
+        client = double "AwsClient"
+        obj_body = double "Body", read: hash
+        obj = double "Object", body: obj_body
+        expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_return obj
+        expect(Aws::S3::Client).to receive(:new).and_return client
+        expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket)
+        expect(storage.check_version?(dir)).to be true
+      end
+
+      it "version file doesn't exist" do
+        dir = "cahche/iso"
+        key = File.join dir, "version"
+        # hash = "invalid"
+        client = double "AwsClient"
+        # obj_body = double "Body", read: hash
+        # obj = double "Object", body: obj_body
+        error = Aws::S3::Errors::NoSuchKey.new Seahorse::Client::RequestContext.new, ""
+        expect(client).to receive(:get_object).with(bucket: bucket, key: key).and_raise error
+        expect(Aws::S3::Client).to receive(:new).and_return client
+        expect(ENV).to receive(:[]).with("AWS_BUCKET").and_return(bucket)
+        expect(storage.check_version?(dir)).to be false
+      end
+    end
+  end
+end