nexus_mods 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7858789972e8674c6ba24c53d36e4af87b09b8636e84faced240ff0edd1eb3fa
-  data.tar.gz: 29df7fc4849e885e6a9aa705d1ae4868215e1181bdff8814d806787ff948790d
+  metadata.gz: 79e4d81970dd1549db43105a2a155b6b275692df4f06827e2234ab02fce6276b
+  data.tar.gz: 4d96df8a39784c8a2eb6fb9bd72a247f3cafa2765aa03059dfc661afd9db7a38
 SHA512:
-  metadata.gz: 522c72b5d4cabff4d57609f9c593c5e75345c675343397e7e75c0bc10c7b29c8df72c26d7fa51bc73265bb8f0c025d3b929ef5a1810711928f416b851bedfef9
-  data.tar.gz: 17b804dfc30795f57c5e862a4990b39aec762f0b3058513956204443e2ebfe59df5285ceda0634a6644cdcd8e45619a8248964bf3028b2aa96852cbc4dd69f9f
+  metadata.gz: 9c13d188694e09dc06998846791322a4887bbb5163ff634d5e6f813abca187a6a2dc8aeb102e040b43812d6276feb6c1ee59665c5e5e408c6acbff4dca0539b3
+  data.tar.gz: 3e37eb153f00bb6335efb8e0b377065f4b0613cbef46066a1a5d7095b1e1767b1786d59f28681298e75be49e1cfeb739ea6ce2f7b13fefc6b28362b27d7536b5

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# [v0.4.0](https://github.com/Muriel-Salvan/nexus_mods/compare/v0.3.0...v0.4.0) (2023-04-10 10:15:41)
+
+### Features
+
+* [[Feature] Add persistent API caching in files](https://github.com/Muriel-Salvan/nexus_mods/commit/f124ecf58b0f478ecdca5f6ad2309779d1ab67ac)
+
 # [v0.3.0](https://github.com/Muriel-Salvan/nexus_mods/compare/v0.2.0...v0.3.0) (2023-04-10 09:13:43)
 
 ### Features

data/lib/nexus_mods/api_client.rb

@@ -12,6 +12,13 @@ class NexusMods
 
     include CacheableApi
 
+    # Default expiry times, in seconds
+    DEFAULT_API_CACHE_EXPIRY = {
+      games: 24 * 60 * 60,
+      mod: 24 * 60 * 60,
+      mod_files: 24 * 60 * 60
+    }
+
     # Constructor
     #
     # Parameters::
@@ -31,8 +38,9 @@ class NexusMods
       logger: Logger.new($stdout)
     )
       @api_key = api_key
-      ApiClient.api_cache_expiry = ApiClient.api_cache_expiry.merge(api_cache_expiry)
+      @api_cache_expiry = DEFAULT_API_CACHE_EXPIRY.merge(api_cache_expiry)
       @api_cache_file = api_cache_file
+      ApiClient.api_client = self
       @logger = logger
       # Initialize our HTTP client
       @http_cache = http_cache_file.nil? ? nil : FileCache.new(http_cache_file)
@@ -96,15 +104,15 @@ class NexusMods
         case key_components[0]
         when 'games'
           if key_components[1].nil?
-            ApiClient.api_cache_expiry[:games]
+            ApiClient.api_client.api_cache_expiry[:games]
           else
             case key_components[2]
             when 'mods'
               case key_components[4]
               when nil
-                ApiClient.api_cache_expiry[:mod]
+                ApiClient.api_client.api_cache_expiry[:mod]
               when 'files'
-                ApiClient.api_cache_expiry[:mod_files]
+                ApiClient.api_client.api_cache_expiry[:mod_files]
               else
                 raise "Unknown API path: #{key}"
               end
@@ -118,6 +126,9 @@ class NexusMods
         else
           raise "Unknown API path: #{key}"
         end
+      end,
+      on_cache_update: proc do
+        ApiClient.api_client.save_api_cache
       end
     )
 
@@ -146,26 +157,22 @@ class NexusMods
       return unless @api_cache_file
 
       FileUtils.mkdir_p(File.dirname(@api_cache_file))
-      Cacheable.cache_adapter.dump(@api_cache_file)
+      Cacheable.cache_adapter.save(@api_cache_file)
     end
 
+    # Some attributes exposed for the cacheable feature to work
+    attr_reader :api_cache_expiry
+
     private
 
     class << self
 
-      # Hash<Symbol,Integer>: Expiry time, per expiry key
-      attr_accessor :api_cache_expiry
+      # ApiClient: The API client to be used by the cacheable adapter (singleton pattern)
+      attr_accessor :api_client
 
     end
 
-    # Default expiry times, in seconds
-    DEFAULT_API_CACHE_EXPIRY = {
-      games: 24 * 60 * 60,
-      mod: 24 * 60 * 60,
-      mod_files: 24 * 60 * 60
-    }
-
-    @api_cache_expiry = DEFAULT_API_CACHE_EXPIRY
+    @api_client = nil
 
     # Get the real URI to query for a given API path
     #

data/lib/nexus_mods/cacheable_api.rb

@@ -28,7 +28,8 @@ class NexusMods
       #     * *key* (String): The key for which we want the expiry time in seconds
       #   * Result::
       #     * Integer: Corresponding expiry time
-      def cacheable_api(*original_method_names, expiry_from_key:)
+      # * *on_cache_update* (Proc): Proc called when the cache has been updated
+      def cacheable_api(*original_method_names, expiry_from_key:, on_cache_update:)
         cacheable_with_expiry(
           *original_method_names,
           key_format: lambda do |target, method_name, method_args, method_kwargs|
@@ -41,7 +42,12 @@ class NexusMods
                 method_kwargs.map { |key, value| "#{key}:#{value}" }
             ).join('/')
           end,
-          expiry_from_key:
+          expiry_from_key:,
+          cache_options: {
+            on_cache_update: proc do |_adapter, _key, _value, _options, _context|
+              on_cache_update.call
+            end
+          }
         )
       end
 

data/lib/nexus_mods/cacheable_with_expiry.rb

@@ -34,19 +34,21 @@ class NexusMods
           *original_method_names,
           **opts.merge(
             {
-              cache_options: {
-                expiry_from_key: opts[:expiry_from_key],
-                invalidate_if: proc do |key, options, context|
-                  next true unless context['invalidate_time']
+              cache_options: (opts[:cache_options] || {}).merge(
+                {
+                  expiry_from_key: opts[:expiry_from_key],
+                  invalidate_if: proc do |key, options, context|
+                    next true unless context['invalidate_time']
 
-                  # Find if we know already the expiry for this key
-                  expiry_cache[key] = options[:expiry_from_key].call(key) unless expiry_cache.key?(key)
-                  expiry_cache[key].nil? || (Time.now.utc - Time.parse(context['invalidate_time']).utc > expiry_cache[key])
-                end,
-                update_context_after_fetch: proc do |_key, _value, _options, context|
-                  context['invalidate_time'] = Time.now.utc.strftime('%FT%TUTC')
-                end
-              }
+                    # Find if we know already the expiry for this key
+                    expiry_cache[key] = options[:expiry_from_key].call(key) unless expiry_cache.key?(key)
+                    expiry_cache[key].nil? || (Time.now.utc - Time.parse(context['invalidate_time']).utc > expiry_cache[key])
+                  end,
+                  update_context_after_fetch: proc do |_key, _value, _options, context|
+                    context['invalidate_time'] = Time.now.utc.strftime('%FT%TUTC')
+                  end
+                }
+              )
             }
           )
         )

data/lib/nexus_mods/core_extensions/cacheable/cache_adapters/persistent_json_adapter.rb

@@ -18,19 +18,26 @@ module Cacheable
       # Parameters::
       # * *key* (String): Key to be fetched
       # * *options* (Hash): Cache options. The following options are interpreted by this fetch:
-      #   * *invalidate_if* (Proc): Code called to know if the cache should be invalidated for a given key:
+      #   * *invalidate_if* (Proc or nil): Optional code called to know if the cache should be invalidated for a given key:
       #     * Parameters::
       #       * *key* (String): The key for which we check the cache invalidation
       #       * *options* (Hash): Cache options linked to this key
       #       * *key_context* (Hash): Context linked to this key, that can be set using the update_context_after_fetch callback.
       #     * Result::
       #       * Boolean: Should we invalidate the cached value of this key?
-      #   * *update_context_after_fetch* (Proc): Code called when the value has been fetched for real (without cache), used to update the context
+      #   * *update_context_after_fetch* (Proc or nil): Optional code called when the value has been fetched for real (without cache), used to update the context
       #     * Parameters::
       #       * *key* (String): The key for which we just fetched the value
       #       * *value* (Object): The value that has just been fetched
       #       * *options* (Hash): Cache options linked to this key
       #       * *key_context* (Hash): Context linked to this key, that is supposed to be updated in place by this callback
+      #   * *on_cache_update* (Proc or nil): Optional code called once the cache has been updated
+      #     * Parameters::
+      #       * *adapter* (Object): Adapter that has the cache being updated
+      #       * *key* (String): The key for which we just fetched the value
+      #       * *value* (Object): The value that has just been fetched
+      #       * *options* (Hash): Cache options linked to this key
+      #       * *key_context* (Hash): Context linked to this key
       # * CodeBlock: Code called to fetch the value if not in the cache
       # Result::
       # * Object: The value for this key
@@ -43,6 +50,8 @@ module Cacheable
         value = yield
         options[:update_context_after_fetch]&.call(key, value, options, key_context)
         write(key, value)
+        options[:on_cache_update]&.call(self, key, value, options, key_context)
+        value
       end
 
       # Clear the cache
@@ -51,6 +60,33 @@ module Cacheable
         super
       end
 
+      # Save the cache and context into a JSON file
+      #
+      # Parameters::
+      # * *file* (String): The file to save to
+      def save(file)
+        # Remove from the context the keys that are not in the cache
+        File.write(
+          file,
+          JSON.dump(
+            {
+              'cache' => cache,
+              'context' => context.select { |key, _value| @cache.key?(key) }
+            }
+          )
+        )
+      end
+
+      # Load the cache and context from a JSON file
+      #
+      # Parameters::
+      # * *file* (String): The file to load from
+      def load(file)
+        loaded_content = JSON.parse(File.read(file))
+        @cache = loaded_content['cache']
+        @context = loaded_content['context']
+      end
+
       private
 
       attr_reader :context

data/lib/nexus_mods/version.rb

@@ -1,5 +1,5 @@
 class NexusMods
 
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 
 end

data/lib/nexus_mods.rb

@@ -44,6 +44,7 @@ class NexusMods
   #   * *games*: Expiry associated to queries on games [default: 1 day]
   #   * *mod*: Expiry associated to queries on mod [default: 1 day]
   #   * *mod_files*: Expiry associated to queries on mod files [default: 1 day]
+  # * *api_cache_file* (String): File used to store the NexusMods API cache, or nil for no cache [default: "#{Dir.tmpdir}/nexus_mods_api_cache.json"]
   # * *logger* (Logger): The logger to be used for log messages [default: Logger.new(STDOUT)]
   def initialize(
     api_key: nil,
@@ -52,6 +53,7 @@ class NexusMods
     file_id: 1,
     http_cache_file: "#{Dir.tmpdir}/nexus_mods_http_cache.json",
     api_cache_expiry: {},
+    api_cache_file: "#{Dir.tmpdir}/nexus_mods_api_cache.json",
     logger: Logger.new($stdout)
   )
     @game_domain_name = game_domain_name
@@ -63,6 +65,7 @@ class NexusMods
       api_key:,
       http_cache_file:,
       api_cache_expiry:,
+      api_cache_file:,
       logger:
     )
 

data/spec/nexus_mods_test/helpers.rb

@@ -55,6 +55,9 @@ module NexusModsTest
     def nexus_mods(**args)
       if @nexus_mods.nil?
         args[:api_key] = MOCKED_API_KEY unless args.key?(:api_key)
+        # By default running tests should not persistent cache files
+        args[:http_cache_file] = nil unless args.key?(:http_cache_file)
+        args[:api_cache_file] = nil unless args.key?(:api_cache_file)
         # Redirect any log into a string so that they don't pollute the tests output and they could be asserted.
         nexus_mods_logger = StringIO.new
         args[:logger] = Logger.new(nexus_mods_logger)
@@ -134,6 +137,19 @@ module NexusModsTest
       )
     end
 
+    # Setup an API cache file to be used (does not create it)
+    #
+    # Parameters::
+    # * CodeBlock: The code called when the API cache file is reserved
+    #   * Parameters::
+    #     * *api_cache_file* (String): File name to be used for the API cache file
+    def with_api_cache_file
+      api_cache_file = "#{Dir.tmpdir}/nexus_mods_test/api_cache.json"
+      FileUtils.mkdir_p(File.dirname(api_cache_file))
+      FileUtils.rm_f(api_cache_file)
+      yield api_cache_file
+    end
+
   end
 
 end

data/spec/nexus_mods_test/scenarios/nexus_mods_caching_spec.rb

@@ -1,3 +1,5 @@
+require 'fileutils'
+
 describe NexusMods do
 
   context 'when testing caching' do
@@ -83,6 +85,73 @@ describe NexusMods do
       expect(nexus_mods_instance.mod_files(game_domain_name: 'skyrimspecialedition', mod_id: 2014)).to eq(mod_files)
     end
 
+    context 'with file persistence' do
+
+      it 'persists API cache in a file' do
+        with_api_cache_file do |api_cache_file|
+          expect_validate_user
+          expect_http_call_to(
+            path: '/v1/games.json',
+            json: [
+              json_game100,
+              json_game101
+            ]
+          )
+          nexus_mods(api_cache_file:).games
+          expect(File.exist?(api_cache_file)).to be true
+          expect(File.size(api_cache_file)).to be > 0
+        end
+      end
+
+      it 'uses API cache from a file' do
+        with_api_cache_file do |api_cache_file|
+          expect_validate_user(times: 2)
+          expect_http_call_to(
+            path: '/v1/games.json',
+            json: [
+              json_game100,
+              json_game101
+            ]
+          )
+          # Generate the cache first
+          games = nexus_mods(api_cache_file:).games
+          # Force a new instance of NexusMods API to run
+          @nexus_mods = nil
+          expect(nexus_mods(api_cache_file:).games).to eq games
+        end
+      end
+
+      it 'completes the API cache from a file' do
+        with_api_cache_file do |api_cache_file|
+          expect_validate_user(times: 3)
+          # Generate the cache first for games only
+          expect_http_call_to(
+            path: '/v1/games.json',
+            json: [
+              json_game100,
+              json_game101
+            ]
+          )
+          games = nexus_mods(api_cache_file:).games
+          # Force a new instance of NexusMods API to run
+          @nexus_mods = nil
+          # Complete the cache with a mod
+          expect_http_call_to(
+            path: '/v1/games/skyrimspecialedition/mods/2014.json',
+            json: json_complete_mod
+          )
+          mod = nexus_mods(api_cache_file:).mod(game_domain_name: 'skyrimspecialedition', mod_id: 2014)
+          # Force a new instance of NexusMods API to run
+          @nexus_mods = nil
+          # Check that both API calls were cached correctly
+          nexus_mods_instance = nexus_mods(api_cache_file:)
+          expect(nexus_mods_instance.games).to eq games
+          expect(nexus_mods_instance.mod(game_domain_name: 'skyrimspecialedition', mod_id: 2014)).to eq mod
+        end
+      end
+
+    end
+
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nexus_mods
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Muriel Salvan