singleton-client-test 0.7.0.31 → 0.7.0.34

data/lib/sgtn-client/core/config.rb

@@ -1,8 +1,17 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
 require 'erb'
 require 'yaml'
 
+
 module SgtnClient
   #include Exceptions
+
+  module TranslationLoader
+    autoload :LoaderFactory, 'sgtn-client/loader/loader_factory'
+  end
+
   module Configuration
 
     def config
@@ -39,7 +48,7 @@ module SgtnClient
     :mode, :endpoint, :merchant_endpoint, :platform_endpoint, :ipn_endpoint,
     :rest_endpoint, :rest_token_endpoint, :client_id, :client_secret,
     :openid_endpoint, :openid_redirect_uri, :openid_client_id, :openid_client_secret,
-    :verbose_logging, :product_name, :version, :vip_server, :bundle_mode,
+    :verbose_logging, :product_name, :version, :vip_server,
     :translation_bundle, :source_bundle, :cache_expiry_period, :disable_cache, :default_language
 
 
@@ -149,6 +158,34 @@ module SgtnClient
           Logging.logger
         end
 
+
+        def loader
+          @loader ||= begin
+            config = SgtnClient::Config.configurations[SgtnClient::Config.default_environment]
+            SgtnClient::TranslationLoader::LoaderFactory.create(config)
+          end
+        end
+
+        def available_bundles
+          loader.available_bundles
+        rescue StandardError => e
+          SgtnClient.logger.error 'failed to get available bundles'
+          SgtnClient.logger.error e
+          Set.new
+        end
+
+        def available_locales
+          bundles = available_bundles
+          return Set.new if bundles.nil? || bundles.empty?
+
+          unless bundles.respond_to?(:locales)
+            def bundles.locales
+              @locales ||= reduce(Set.new) { |locales, id| locales << id.locale }
+            end
+          end
+          bundles.locales
+        end
+
         private
         # Read configurations from the given file name
         # === Arguments
@@ -158,9 +195,8 @@ module SgtnClient
           erb.filename = file_name
           YAML.load(erb.result)
         end       
-          
 
     end
   end
 
-end
+end

data/lib/sgtn-client/core/exceptions.rb

@@ -1,3 +1,6 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
 require 'multi_json'
 require 'pp'
 

data/lib/sgtn-client/core/logging.rb

@@ -1,3 +1,6 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
 require 'logger'
 
 module SgtnClient
@@ -47,4 +50,3 @@ module SgtnClient
   end
 
 end
-

data/lib/sgtn-client/exceptions.rb

@@ -0,0 +1,6 @@
+#  Copyright 2022 VMware, Inc.
+#  SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  class SingletonError < StandardError; end
+end

data/lib/sgtn-client/formatters/plurals/plural_formatter.rb

@@ -1,3 +1,6 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
 require 'json'
 
 module SgtnClient
@@ -35,4 +38,4 @@ module SgtnClient
               end
         end
     end
-end
+end

data/lib/sgtn-client/loader/cache.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  autoload :CacheUtil, 'sgtn-client/util/cache-util'
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    module Cache # :nodoc:
+      # get from cache, return expired data immediately
+      def get_bundle(component, locale)
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}] component=#{component}, locale=#{locale}"
+
+        key = SgtnClient::CacheUtil.get_cachekey(component, locale)
+        cache_item = SgtnClient::CacheUtil.get_cache(key)
+        if cache_item
+          if SgtnClient::CacheUtil.is_expired(cache_item)
+            Thread.new do # TODO: Use one thread # refresh in background
+              begin
+                load_bundle(component, locale)
+              rescue StandardError => e
+                SgtnClient.logger.error "an error occured while loading bundle: component=#{component}, locale=#{locale}"
+                SgtnClient.logger.error e
+              end
+            end
+          end
+          return cache_item.dig(:items)
+        end
+
+        load_bundle(component, locale) # refresh synchronously if not in cache
+      end
+
+      # load and save to cache
+      def load_bundle(component, locale)
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}] component=#{component}, locale=#{locale}"
+
+        key = SgtnClient::CacheUtil.get_cachekey(component, locale)
+        item = super
+        SgtnClient::CacheUtil.write_cache(key, item) if item
+        item
+      end
+
+      def available_bundles
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}]"
+
+        cache_item = SgtnClient::CacheUtil.get_cache(CONSTS::AVAILABLE_BUNDLES_KEY)
+        if cache_item
+          if SgtnClient::CacheUtil.is_expired(cache_item)
+            Thread.new do # TODO: Use one thread
+              begin
+                item = super
+                SgtnClient::CacheUtil.write_cache(CONSTS::AVAILABLE_BUNDLES_KEY, item) if item
+              rescue StandardError => e
+                SgtnClient.logger.error 'an error occured while loading available bundles.'
+                SgtnClient.logger.error e
+              end
+            end
+          end
+          return cache_item.dig(:items)
+        end
+
+        item = super
+        SgtnClient::CacheUtil.write_cache(CONSTS::AVAILABLE_BUNDLES_KEY, item) if item
+        item
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/chain_loader.rb

@@ -0,0 +1,49 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  module TranslationLoader
+    class Chain
+      attr_accessor :loaders
+
+      def initialize(*loaders)
+        self.loaders = loaders
+      end
+
+      def load_bundle(component, locale)
+        exception = nil
+
+        loaders.each do |loader|
+          begin
+            bundle = loader.load_bundle(component, locale)
+            return bundle if bundle
+          rescue StandardError => e
+            exception = e
+            SgtnClient.logger.error "[#{__FILE__}][#{__callee__}] {component: #{component},locale: #{locale}}, failed on #{loader.class}: #{e}"
+          end
+        end
+
+        raise exception || SgtnClient::SingletonError.new("can't load component: #{component}, locale: #{locale}")
+      end
+
+      def available_bundles
+        exception = nil
+        total_data = Set.new
+
+        loaders.each do |loader|
+          begin
+            item = loader.available_bundles
+            total_data += item
+          rescue StandardError => e
+            exception = e
+            SgtnClient.logger.error "[#{__FILE__}][#{__callee__}] failed on #{loader.class}: #{e}"
+          end
+        end
+
+        raise exception if total_data.empty? && exception
+
+        total_data
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/consts.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  module TranslationLoader
+    module CONSTS
+      OLD_SOURCE_LOCALE = 'old_source'
+      REAL_SOURCE_LOCALE = 'latest'
+
+      AVAILABLE_BUNDLES_KEY = 'available_bundles'
+    end
+  end
+end

data/lib/sgtn-client/loader/loader_factory.rb

@@ -0,0 +1,33 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  module TranslationLoader
+    autoload :Source, 'sgtn-client/loader/source'
+    autoload :SgtnServer, 'sgtn-client/loader/server'
+    autoload :LocalTranslation, 'sgtn-client/loader/local_translation'
+    autoload :Chain, 'sgtn-client/loader/chain_loader'
+    autoload :SourceComparer, 'sgtn-client/loader/source_comparer'
+    autoload :SingleLoader, 'sgtn-client/loader/single_loader'
+    autoload :Cache, 'sgtn-client/loader/cache'
+
+    module LoaderFactory
+      def self.create(config)
+        SgtnClient.logger.info "[#{method(__callee__).owner}.#{__callee__}] config=#{config}"
+
+        loaders = []
+        loaders << Source.new(config) if config['source_bundle']
+        loaders << SgtnServer.new(config) if config['vip_server']
+        loaders << LocalTranslation.new(config) if config['translation_bundle']
+        raise SgtnClient::SingletonError, 'no translation is available!' if loaders.empty?
+
+        chain_loader = Class.new(Chain)
+        chain_loader.include SourceComparer if config['source_bundle'] || config['vip_server']
+        chain_loader.include SingleLoader
+        chain_loader.include Cache
+
+        chain_loader.new(*loaders)
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/local_translation.rb

@@ -0,0 +1,50 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+require 'multi_json'
+
+module SgtnClient
+  module Common
+    autoload :BundleID, 'sgtn-client/common/data'
+  end
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    class LocalTranslation
+      BUNDLE_PREFIX = 'messages_'.freeze
+      BUNDLE_SUFFIX = '.json'.freeze
+
+      def initialize(config)
+        @base_path = Pathname.new(config['translation_bundle']) + config['product_name'] + config['version'].to_s
+      end
+
+      def load_bundle(component, locale)
+        return if locale == CONSTS::REAL_SOURCE_LOCALE # return when querying source
+
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}] component=#{component}, locale=#{locale}"
+
+        file_name = BUNDLE_PREFIX + locale + BUNDLE_SUFFIX
+        file_path = @base_path + component + file_name
+
+        bundle_data = JSON.parse(File.read(file_path))
+        messages = bundle_data['messages']
+
+        raise SgtnClient::SingletonError, "no messages in local bundle file: #{file_path}." unless messages
+
+        messages
+      end
+
+      def available_bundles
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}]"
+
+        @available_bundles ||= begin
+          @base_path.glob('*/*.json').reduce(Set.new) do |bundles, f|
+            locale = f.basename.to_s.sub(/\A#{BUNDLE_PREFIX}/i, '').sub(/#{BUNDLE_SUFFIX}\z/i, '')
+            bundles.add Common::BundleID.new(f.parent.basename.to_s, locale)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/server.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+require 'faraday'
+require 'faraday_middleware'
+
+module SgtnClient
+  module Common
+    autoload :BundleID, 'sgtn-client/common/data'
+  end
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    class SgtnServer
+      ERROR_ILLEGAL_DATA = 'server returned illegal data.'
+      ERROR_BUSINESS_ERROR = 'server returned business error.'
+
+      REQUEST_ARGUMENTS = { timeout: 10 }.freeze
+
+      def initialize(config)
+        @server_url = config['vip_server']
+
+        product_root = format('/i18n/api/v2/translation/products/%s/versions/%s', config['product_name'], config['version'])
+
+        @bundle_url = "#{product_root}/locales/%s/components/%s"
+        @locales_url = "#{product_root}/localelist"
+        @components_url = "#{product_root}/componentlist"
+      end
+
+      def load_bundle(component, locale)
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}] component=#{component}, locale=#{locale}"
+
+        messages = query_server(format(@bundle_url, locale, component), ['messages'])
+        messages
+      end
+
+      def available_bundles
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}]"
+
+        components_thread = Thread.new { available_components }
+        available_locales.reduce(Set.new) do |bundles, locale|
+          components_thread.value.reduce(bundles) do |inner_bundles, component|
+            inner_bundles << Common::BundleID.new(component, locale)
+          end
+        end
+      end
+
+      private
+
+      def available_locales
+        query_server(@locales_url, ['locales'])
+      end
+
+      def available_components
+        query_server(@components_url, ['components'])
+      end
+
+      def query_server(url, path_to_data = [], queries = nil, headers = nil)
+        conn = Faraday.new(@server_url, request: REQUEST_ARGUMENTS) do |f|
+          f.response :json # decode response bodies as JSON
+          f.use :gzip
+          f.response :raise_error
+          f.response :logger, SgtnClient.logger, { log_level: :debug }
+        end
+        resp = conn.get(url, queries, headers)
+
+        process_business_error(resp.body)
+        extract_data(resp.body, path_to_data)
+      end
+
+      def extract_data(parsedbody, path_to_data)
+        data = parsedbody.dig('data', *path_to_data)
+        raise SgtnClient::SingletonError, "no expected data in response. Body is: #{parsedbody}" unless data
+
+        data
+      end
+
+      def process_business_error(parsedbody)
+        b_code = parsedbody.dig('response', 'code')
+        unless b_code >= 200 && b_code < 300 || b_code >= 600 && b_code < 700
+          raise SgtnClient::SingletonError, "#{ERROR_BUSINESS_ERROR} #{parsedbody['response']}"
+        end
+
+        # 600 means a successful response, 6xx means partial successful.
+        SgtnClient.logger.warn "#{ERROR_BUSINESS_ERROR} #{parsedbody['response']}" if b_code > 600
+      rescue TypeError, ArgumentError, NoMethodError => e
+        raise SgtnClient::SingletonError, "#{ERROR_ILLEGAL_DATA} #{e}. Body is: #{parsedbody}"
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/single_loader.rb

@@ -0,0 +1,46 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  autoload :SingleOperation, 'sgtn-client/common/single_operation'
+  autoload :CacheUtil, 'sgtn-client/util/cache-util'
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    module SingleLoader
+      def load_bundle(component, locale)
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}] component=#{component}, locale=#{locale}"
+
+        @single_bundle_loader ||= single_loader { |c,l| super(c,l) }
+        id = CacheUtil.get_cachekey(component, locale)
+        @single_bundle_loader.operate(id, component, locale)&.value
+      ensure
+        # delete thread from hash after finish
+        @single_bundle_loader.remove_object(id)
+      end
+
+      def available_bundles
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}]"
+
+        @single_available_bundles_loader ||= single_loader { super }
+        @single_available_bundles_loader.operate(CONSTS::AVAILABLE_BUNDLES_KEY)&.value
+      ensure
+        @single_available_bundles_loader.remove_object(CONSTS::AVAILABLE_BUNDLES_KEY)
+      end
+
+      private
+      def single_loader(&block)
+        none_alive = proc { |_, thread| thread.nil? || thread.alive? == false }
+        creator = proc do |id, _, *args|
+          Thread.new do
+            SgtnClient.logger.debug "start single loading #{id}"
+            block.call(*args)
+          end
+        end
+
+        SgtnClient::SingleOperation.new(none_alive, &creator)
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/source.rb

@@ -0,0 +1,53 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  module Common
+    autoload :BundleID, 'sgtn-client/common/data'
+  end
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    class Source
+      def initialize(config)
+        @source_bundle_path = Pathname.new(config['source_bundle'])
+      end
+
+      def load_bundle(component, locale = nil)
+        return if locale && locale != CONSTS::REAL_SOURCE_LOCALE # return when NOT querying source
+
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}] component=#{component}"
+
+        total_messages = {}
+
+        (@source_bundle_path + component).glob('**/*.{yml, yaml}') do |f|
+          bundle = YAML.safe_load(File.read(f))
+          messages = bundle&.first&.last # TODO: Warn about inconsistent source locale
+          if messages.is_a?(Hash)
+            total_messages.merge!(messages)
+          else
+            SgtnClient.logger.error "[#{method(__callee__).owner}.#{__callee__}] invalid bundle data in #{f}"
+          end
+        end
+
+        raise SgtnClient::SingletonError, "no local source messages for component #{component}" if total_messages.empty?
+
+        total_messages
+      end
+
+      def available_bundles
+        SgtnClient.logger.debug "[#{method(__callee__).owner}.#{__callee__}]"
+
+        @available_bundles ||= begin
+          @source_bundle_path.children.select(&:directory?).reduce(Set.new) do |bundles, component|
+            component.glob('**/*.{yml, yaml}') do |_|
+              bundles << Common::BundleID.new(component.basename.to_s, SgtnClient::LocaleUtil.get_source_locale)
+              break bundles
+            end || bundles
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sgtn-client/loader/source_comparer.rb

@@ -0,0 +1,58 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
+module SgtnClient
+  autoload :StringUtil, 'sgtn-client/util/string-util'
+  autoload :LocaleUtil, 'sgtn-client/util/locale-util'
+
+  module TranslationLoader
+    autoload :CONSTS, 'sgtn-client/loader/consts'
+
+    module SourceComparer
+      def load_bundle(component, locale)
+        SgtnClient.logger.debug "[#{__FILE__}][#{__callee__}] component=#{component}, locale=#{locale}"
+
+        # source locale and old source locale don't need comparison because they are bases of comparison
+        real_locale = cache_to_real_map[locale]
+        return super(component, real_locale) if real_locale
+
+        old_source_bundle_thread = Thread.new { load_bundle(component, CONSTS::OLD_SOURCE_LOCALE) }
+        source_bundle_thread = Thread.new { load_bundle(component, LocaleUtil.get_source_locale) }
+        translation_bundle = super(component, locale)
+
+        begin
+          old_source_bundle = old_source_bundle_thread.value
+          source_bundle = source_bundle_thread.value
+        rescue StandardError => e
+          SgtnClient.logger.error "[#{__FILE__}][#{__callee__}] failed to load soruce(or old source) bundle. component:#{component}. error: #{e}"
+          return translation_bundle
+        end
+
+        compare_source(translation_bundle, old_source_bundle, source_bundle)
+      end
+
+      private
+
+      def compare_source(translation_bundle, old_source_bundle, source_bundle)
+        if !translation_bundle.is_a?(Hash) || !source_bundle.is_a?(Hash) || !old_source_bundle.is_a?(Hash)
+          SgtnClient.logger.warn "can't do source comparison because some bundle data are wrong."
+          return translation_bundle
+        end
+
+        source_bundle.each do |key, value|
+          if old_source_bundle[key] != value || translation_bundle[key].nil?
+            translation_bundle[key] = StringUtil.new(value, LocaleUtil.get_source_locale)
+          end
+        end
+        translation_bundle
+      end
+
+      def cache_to_real_map
+        @cache_to_real_map ||= {
+          LocaleUtil.get_source_locale => CONSTS::REAL_SOURCE_LOCALE,
+          CONSTS::OLD_SOURCE_LOCALE => LocaleUtil.get_source_locale
+        }.freeze
+      end
+    end
+  end
+end

data/lib/sgtn-client/sgtn-client.rb

@@ -1,4 +1,9 @@
+# Copyright 2022 VMware, Inc.
+# SPDX-License-Identifier: EPL-2.0
+
 module SgtnClient
+      LOGFILE_SHIFT_AGE = 4
+
       module Core
             autoload :Cache,        "sgtn-client/core/cache"
       end
@@ -12,6 +17,7 @@ module SgtnClient
       autoload :ValidateUtil,       "sgtn-client/util/validate-util"
       autoload :LocaleUtil,         "sgtn-client/util/locale-util"
       autoload :FileUtil,           "sgtn-client/util/file-util"
+      autoload :CacheUtil,          "sgtn-client/util/cache-util"
 
       module Formatters
             autoload :PluralFormatter,    "sgtn-client/formatters/plurals/plural_formatter"
@@ -44,7 +50,7 @@ module SgtnClient
                   end
                   file = File.open(file, 'a')
                   file.sync = true
-                  SgtnClient.logger = Logger.new(file)
+                  SgtnClient.logger = Logger.new(file, LOGFILE_SHIFT_AGE)
 
                   # Set log level for sandbox mode
                   env = SgtnClient::Config.default_environment