factorix 0.5.1 → 0.7.0

data/lib/factorix/cache/file_system.rb

@@ -2,6 +2,7 @@
 
 require "digest"
 require "fileutils"
+require "json"
 require "pathname"
 require "zlib"
 
@@ -12,7 +13,12 @@ module Factorix
     # Uses a two-level directory structure to store cached files,
     # with file locking to handle concurrent access and TTL support
     # for cache expiration.
-    class FileSystem
+    #
+    # Cache entries consist of:
+    # - Data file: the cached content (optionally compressed)
+    # - Metadata file (.metadata): JSON containing the logical key
+    # - Lock file (.lock): used for concurrent access control
+    class FileSystem < Base
       # @!parse
       #   # @return [Dry::Logger::Dispatcher]
       #   attr_reader :logger
@@ -29,52 +35,46 @@ module Factorix
       private_constant :ZLIB_CMF_BYTE
 
       # Initialize a new file system cache storage.
-      # Creates the cache directory if it doesn't exist
+      # Creates the cache directory if it doesn't exist.
+      # Cache directory is auto-calculated as: factorix_cache_dir / cache_type
       #
-      # @param cache_dir [Pathname] path to the cache directory
-      # @param ttl [Integer, nil] time-to-live in seconds (nil for unlimited)
+      # @param cache_type [Symbol] cache type for directory name (e.g., :api, :download)
       # @param max_file_size [Integer, nil] maximum file size in bytes (nil for unlimited)
       # @param compression_threshold [Integer, nil] compress data larger than this size in bytes
       #   (nil: no compression, 0: always compress, N: compress if >= N bytes)
-      def initialize(cache_dir, ttl: nil, max_file_size: nil, compression_threshold: nil, logger: nil)
-        super(logger:)
-        @cache_dir = cache_dir
-        @ttl = ttl
+      # @param ttl [Integer, nil] time-to-live in seconds (nil for unlimited)
+      def initialize(cache_type:, max_file_size: nil, compression_threshold: nil, **)
+        super(**)
+        @cache_dir = Container[:runtime].factorix_cache_dir / cache_type.to_s
         @max_file_size = max_file_size
         @compression_threshold = compression_threshold
         @cache_dir.mkpath
-        logger.info("Initializing cache", dir: @cache_dir.to_s, ttl: @ttl, max_size: @max_file_size, compression_threshold: @compression_threshold)
+        logger.info("Initializing cache", root: @cache_dir.to_s, ttl: @ttl, max_size: @max_file_size, compression_threshold: @compression_threshold)
       end
 
-      # Generate a cache key for the given URL string.
-      # Uses SHA1 to create a unique, deterministic key
-      #
-      # @param url_string [String] URL string to generate key for
-      # @return [String] cache key
-      # Use Digest(:SHA1) instead of Digest::SHA1 for thread-safety (Ruby 2.2+)
-      def key_for(url_string) = Digest(:SHA1).hexdigest(url_string)
-
       # Check if a cache entry exists and is not expired.
       # A cache entry is considered to exist if its file exists and is not expired
       #
-      # @param key [String] cache key to check
+      # @param key [String] logical cache key
       # @return [Boolean] true if the cache entry exists and is valid, false otherwise
       def exist?(key)
-        return false unless cache_path_for(key).exist?
+        internal_key = storage_key_for(key)
+        return false unless cache_path_for(internal_key).exist?
         return true if @ttl.nil?
 
         !expired?(key)
       end
 
-      # Fetch a cached file and copy it to the output path.
+      # Write cached content to a file.
       # If the cache entry doesn't exist or is expired, returns false without modifying the output path.
       # Automatically decompresses zlib-compressed cache entries.
       #
-      # @param key [String] cache key to fetch
-      # @param output [Pathname] path to copy the cached file to
-      # @return [Boolean] true if the cache entry was found and copied, false otherwise
-      def fetch(key, output)
-        path = cache_path_for(key)
+      # @param key [String] logical cache key
+      # @param output [Pathname] path to write the cached content to
+      # @return [Boolean] true if written successfully, false if not found/expired
+      def write_to(key, output)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
         unless path.exist?
           logger.debug("Cache miss", key:)
           return false
@@ -96,21 +96,21 @@ module Factorix
         true
       end
 
-      # Read a cached file as a string.
+      # Read a cached file as a binary string.
       # If the cache entry doesn't exist or is expired, returns nil.
       # Automatically decompresses zlib-compressed cache entries.
       #
-      # @param key [String] cache key to read
-      # @param encoding [Encoding, String] encoding to use (default: ASCII-8BIT for binary)
+      # @param key [String] logical cache key
       # @return [String, nil] cached content or nil if not found/expired
-      def read(key, encoding: Encoding::ASCII_8BIT)
-        path = cache_path_for(key)
+      def read(key)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
         return nil unless path.exist?
         return nil if expired?(key)
 
         data = path.binread
         data = Zlib.inflate(data) if zlib_compressed?(data)
-        data.force_encoding(encoding)
+        data
       end
 
       # Store a file in the cache.
@@ -118,7 +118,7 @@ module Factorix
       # Optionally compresses data based on compression_threshold setting.
       # If the (possibly compressed) size exceeds max_file_size, skips caching and returns false.
       #
-      # @param key [String] cache key to store under
+      # @param key [String] logical cache key
       # @param src [Pathname] path of the file to store
       # @return [Boolean] true if cached successfully, false if skipped due to size limit
       def store(key, src)
@@ -135,22 +135,30 @@ module Factorix
           return false
         end
 
-        path = cache_path_for(key)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
+        metadata_path = metadata_path_for(internal_key)
+
         path.dirname.mkpath
         path.binwrite(data)
+        metadata_path.write(JSON.generate({logical_key: key}))
         logger.debug("Stored in cache", key:, size_bytes: data.bytesize)
         true
       end
 
       # Delete a specific cache entry.
       #
-      # @param key [String] cache key to delete
+      # @param key [String] logical cache key
       # @return [Boolean] true if the entry was deleted, false if it didn't exist
       def delete(key)
-        path = cache_path_for(key)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
+        metadata_path = metadata_path_for(internal_key)
+
         return false unless path.exist?
 
         path.delete
+        metadata_path.delete if metadata_path.exist?
         logger.debug("Deleted from cache", key:)
         true
       end
@@ -160,13 +168,14 @@ module Factorix
       #
       # @return [void]
       def clear
-        logger.info("Clearing cache directory", dir: @cache_dir.to_s)
+        logger.info("Clearing cache directory", root: @cache_dir.to_s)
         count = 0
         @cache_dir.glob("**/*").each do |path|
-          if path.file?
-            path.delete
-            count += 1
-          end
+          next unless path.file?
+          next if path.extname == ".lock"
+
+          path.delete
+          count += 1
         end
         logger.info("Cache cleared", files_removed: count)
       end
@@ -174,10 +183,11 @@ module Factorix
       # Get the age of a cache entry in seconds.
       # Returns nil if the entry doesn't exist.
       #
-      # @param key [String] cache key
+      # @param key [String] logical cache key
       # @return [Float, nil] age in seconds, or nil if entry doesn't exist
       def age(key)
-        path = cache_path_for(key)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
         return nil unless path.exist?
 
         Time.now - path.mtime
@@ -186,7 +196,7 @@ module Factorix
       # Check if a cache entry has expired based on TTL.
       # Returns false if TTL is not set (unlimited) or if entry doesn't exist.
       #
-      # @param key [String] cache key
+      # @param key [String] logical cache key
       # @return [Boolean] true if expired, false otherwise
       def expired?(key)
         return false if @ttl.nil?
@@ -200,10 +210,11 @@ module Factorix
       # Get the size of a cached file in bytes.
       # Returns nil if the entry doesn't exist or is expired.
       #
-      # @param key [String] cache key
+      # @param key [String] logical cache key
       # @return [Integer, nil] file size in bytes, or nil if entry doesn't exist/expired
       def size(key)
-        path = cache_path_for(key)
+        internal_key = storage_key_for(key)
+        path = cache_path_for(internal_key)
         return nil unless path.exist?
         return nil if expired?(key)
 
@@ -211,13 +222,14 @@ module Factorix
       end
 
       # Executes the given block with a file lock.
-      # Uses flock for process-safe file locking and automatically removes stale locks
+      # Uses flock for process-safe file locking and automatically removes stale locks.
       #
-      # @param key [String] cache key to lock
+      # @param key [String] logical cache key
       # @yield Executes the block with exclusive file lock
       # @return [void]
       def with_lock(key)
-        lock_path = lock_path_for(key)
+        internal_key = storage_key_for(key)
+        lock_path = lock_path_for(internal_key)
         cleanup_stale_lock(lock_path)
 
         lock_path.dirname.mkpath
@@ -240,23 +252,83 @@ module Factorix
         end
       end
 
-      # Get the cache file path for the given key.
+      # Enumerate cache entries.
+      #
+      # Yields [key, entry] pairs similar to Hash#each.
+      # Skips entries without metadata files (legacy entries).
+      #
+      # @yield [key, entry] logical key and Entry object
+      # @yieldparam key [String] logical cache key
+      # @yieldparam entry [Entry] cache entry metadata
+      # @return [Enumerator] if no block given
+      def each
+        return enum_for(__method__) unless block_given?
+
+        @cache_dir.glob("**/*").each do |path|
+          next unless path.file?
+          next if path.extname == ".metadata" || path.extname == ".lock"
+
+          metadata_path = Pathname("#{path}.metadata")
+          next unless metadata_path.exist?
+
+          logical_key = JSON.parse(metadata_path.read)["logical_key"]
+          age = Time.now - path.mtime
+          entry = Entry.new(
+            size: path.size,
+            age:,
+            expired: @ttl ? age > @ttl : false
+          )
+
+          yield logical_key, entry
+        end
+      end
+
+      # Return backend-specific information.
+      #
+      # @return [Hash] backend configuration and status
+      def backend_info
+        {
+          type: "file_system",
+          directory: @cache_dir.to_s,
+          max_file_size: @max_file_size,
+          compression_threshold: @compression_threshold,
+          stale_locks: count_stale_locks
+        }
+      end
+
+      # Generate a storage key for the given logical key.
+      # Uses SHA1 to create a unique, deterministic key.
+      # Use Digest(:SHA1) instead of Digest::SHA1 for thread-safety (Ruby 2.2+)
+      #
+      # @param logical_key [String] logical key to generate storage key for
+      # @return [String] storage key (SHA1 hash)
+      private def storage_key_for(logical_key) = Digest(:SHA1).hexdigest(logical_key)
+
+      # Get the cache file path for the given internal key.
       # Uses a two-level directory structure to avoid too many files in one directory
       #
-      # @param key [String] cache key
+      # @param internal_key [String] internal storage key
       # @return [Pathname] path to the cache file
-      private def cache_path_for(key)
-        prefix = key[0, 2]
-        @cache_dir.join(prefix, key[2..])
+      private def cache_path_for(internal_key)
+        prefix = internal_key[0, 2]
+        @cache_dir.join(prefix, internal_key[2..])
+      end
+
+      # Get the metadata file path for the given internal key.
+      #
+      # @param internal_key [String] internal storage key
+      # @return [Pathname] path to the metadata file
+      private def metadata_path_for(internal_key)
+        Pathname("#{cache_path_for(internal_key)}.metadata")
       end
 
-      # Get the lock file path for the given key.
+      # Get the lock file path for the given internal key.
       # Lock files are stored alongside cache files with a .lock extension
       #
-      # @param key [String] cache key
+      # @param internal_key [String] internal storage key
       # @return [Pathname] path to the lock file
-      private def lock_path_for(key)
-        cache_path_for(key).sub_ext(".lock")
+      private def lock_path_for(internal_key)
+        cache_path_for(internal_key).sub_ext(".lock")
       end
 
       # Check if data should be compressed based on compression_threshold setting.
@@ -302,6 +374,14 @@ module Factorix
           nil
         end
       end
+
+      # Count stale lock files in the cache directory.
+      #
+      # @return [Integer] number of stale lock files
+      private def count_stale_locks
+        cutoff = Time.now - LOCK_FILE_LIFETIME
+        @cache_dir.glob("**/*.lock").count {|path| path.mtime < cutoff }
+      end
     end
   end
 end

data/lib/factorix/cache/redis.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+begin
+  require "redis"
+rescue LoadError
+  raise Factorix::Error, "redis gem is required for Redis cache backend. Add it to your Gemfile."
+end
+
+require "securerandom"
+
+module Factorix
+  module Cache
+    # Redis-based cache storage implementation.
+    #
+    # Stores cache entries in Redis with automatic namespace prefixing.
+    # Metadata (size, created_at) stored in separate hash keys.
+    # Supports distributed locking with Lua script for atomic release.
+    #
+    # @example Configuration
+    #   Factorix.configure do |config|
+    #     config.cache.api.backend = :redis
+    #     config.cache.api.redis.url = "redis://localhost:6379/0"
+    #     config.cache.api.redis.lock_timeout = 30
+    #   end
+    class Redis < Base
+      # @!parse
+      #   # @return [Dry::Logger::Dispatcher]
+      #   attr_reader :logger
+      include Import[:logger]
+
+      # Default timeout for distributed lock acquisition in seconds.
+      DEFAULT_LOCK_TIMEOUT = 30
+      public_constant :DEFAULT_LOCK_TIMEOUT
+
+      # TTL for distributed locks in seconds.
+      LOCK_TTL = 30
+      private_constant :LOCK_TTL
+
+      # Lua script for atomic lock release (only release if we own it).
+      RELEASE_LOCK_SCRIPT = <<~LUA
+        if redis.call("get", KEYS[1]) == ARGV[1] then
+          return redis.call("del", KEYS[1])
+        else
+          return 0
+        end
+      LUA
+      private_constant :RELEASE_LOCK_SCRIPT
+
+      # Initialize a new Redis cache storage.
+      #
+      # @param url [String, nil] Redis URL (defaults to REDIS_URL env)
+      # @param cache_type [String, Symbol] Cache type for namespace (e.g., :api, :download)
+      # @param lock_timeout [Integer] Timeout for lock acquisition in seconds
+      # @param ttl [Integer, nil] time-to-live in seconds (nil for unlimited)
+      def initialize(cache_type:, url: nil, lock_timeout: DEFAULT_LOCK_TIMEOUT, **)
+        super(**)
+        @url = url || ENV.fetch("REDIS_URL", nil)
+        @redis = ::Redis.new(url: @url)
+        @namespace = "factorix-cache:#{cache_type}"
+        @lock_timeout = lock_timeout
+        logger.info("Initializing Redis cache", namespace: @namespace, ttl: @ttl, lock_timeout: @lock_timeout)
+      end
+
+      # Check if a cache entry exists.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if the cache entry exists
+      def exist?(key) = @redis.exists?(data_key(key))
+
+      # Read a cached entry.
+      #
+      # @param key [String] logical cache key
+      # @return [String, nil] cached content or nil if not found
+      def read(key)
+        @redis.get(data_key(key))
+      end
+
+      # Write cached content to a file.
+      #
+      # @param key [String] logical cache key
+      # @param output [Pathname] path to write the cached content
+      # @return [Boolean] true if written successfully, false if not found
+      def write_to(key, output)
+        data = @redis.get(data_key(key))
+        return false if data.nil?
+
+        output.binwrite(data)
+        logger.debug("Cache hit", key:)
+        true
+      end
+
+      # Store data in the cache.
+      #
+      # @param key [String] logical cache key
+      # @param src [Pathname] path to the source file
+      # @return [Boolean] true if stored successfully
+      def store(key, src)
+        data = src.binread
+        data_k = data_key(key)
+        meta_k = meta_key(key)
+
+        @redis.multi do |tx|
+          tx.set(data_k, data)
+          tx.hset(meta_k, "size", data.bytesize, "created_at", Time.now.to_i)
+
+          if @ttl
+            tx.expire(data_k, @ttl)
+            tx.expire(meta_k, @ttl)
+          end
+        end
+
+        logger.debug("Stored in cache", key:, size_bytes: data.bytesize)
+        true
+      end
+
+      # Delete a cache entry.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if deleted, false if not found
+      def delete(key)
+        deleted = @redis.del(data_key(key), meta_key(key))
+        logger.debug("Deleted from cache", key:) if deleted.positive?
+        deleted.positive?
+      end
+
+      # Clear all cache entries in this namespace.
+      #
+      # @return [void]
+      def clear
+        logger.info("Clearing Redis cache namespace", namespace: @namespace)
+        count = 0
+        cursor = "0"
+        pattern = "#{@namespace}:*"
+
+        loop do
+          cursor, keys = @redis.scan(cursor, match: pattern, count: 100)
+          unless keys.empty?
+            @redis.del(*keys)
+            count += keys.size
+          end
+          break if cursor == "0"
+        end
+
+        logger.info("Cache cleared", keys_removed: count)
+      end
+
+      # Get the age of a cache entry in seconds.
+      #
+      # @param key [String] logical cache key
+      # @return [Integer, nil] age in seconds, or nil if entry doesn't exist
+      def age(key)
+        value = @redis.hget(meta_key(key), "created_at")
+        return nil if value.nil?
+
+        created_at = Integer(value, 10)
+        return nil if created_at.zero?
+
+        Time.now.to_i - created_at
+      end
+
+      # Check if a cache entry has expired.
+      # With Redis native EXPIRE, non-existent keys are considered expired.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if expired (or doesn't exist), false otherwise
+      def expired?(key) = !exist?(key)
+
+      # Get the size of a cached entry in bytes.
+      #
+      # @param key [String] logical cache key
+      # @return [Integer, nil] size in bytes, or nil if entry doesn't exist
+      def size(key)
+        return nil unless exist?(key)
+
+        value = @redis.hget(meta_key(key), "size")
+        value.nil? ? nil : Integer(value, 10)
+      end
+
+      # Execute a block with a distributed lock.
+      # Uses Redis SET NX EX for lock acquisition and Lua script for atomic release.
+      #
+      # @param key [String] logical cache key
+      # @yield block to execute with lock held
+      # @raise [LockTimeoutError] if lock cannot be acquired within timeout
+      def with_lock(key)
+        lkey = lock_key(key)
+        lock_value = SecureRandom.uuid
+        deadline = Time.now + @lock_timeout
+
+        until @redis.set(lkey, lock_value, nx: true, ex: LOCK_TTL)
+          raise LockTimeoutError, "Failed to acquire lock for key: #{key}" if Time.now > deadline
+
+          sleep 0.1
+        end
+
+        logger.debug("Acquired lock", key:)
+        begin
+          yield
+        ensure
+          @redis.eval(RELEASE_LOCK_SCRIPT, keys: [lkey], argv: [lock_value])
+          logger.debug("Released lock", key:)
+        end
+      end
+
+      # Enumerate cache entries.
+      #
+      # @yield [key, entry] logical key and Entry object
+      # @yieldparam key [String] logical cache key
+      # @yieldparam entry [Entry] cache entry metadata
+      # @return [Enumerator] if no block given
+      def each
+        return enum_for(__method__) unless block_given?
+
+        cursor = "0"
+        pattern = "#{@namespace}:*"
+
+        loop do
+          cursor, keys = @redis.scan(cursor, match: pattern, count: 100)
+
+          keys.each do |data_k|
+            next if data_k.include?(":meta:") || data_k.include?(":lock:")
+
+            logical_key = logical_key_from_data_key(data_k)
+            meta = @redis.hgetall(meta_key(logical_key))
+
+            entry = Entry.new(
+              size: meta["size"] ? Integer(meta["size"], 10) : 0,
+              age: meta["created_at"] ? Time.now.to_i - Integer(meta["created_at"], 10) : 0,
+              expired: false # Redis handles expiry natively
+            )
+
+            yield logical_key, entry
+          end
+
+          break if cursor == "0"
+        end
+      end
+
+      # Return backend-specific information.
+      #
+      # @return [Hash] backend configuration
+      def backend_info
+        {
+          type: "redis",
+          url: mask_url(@url),
+          namespace: @namespace,
+          lock_timeout: @lock_timeout
+        }
+      end
+
+      # Generate data key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] namespaced data key
+      private def data_key(logical_key) = "#{@namespace}:#{logical_key}"
+
+      # Generate metadata key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] namespaced metadata key
+      private def meta_key(logical_key) = "#{@namespace}:meta:#{logical_key}"
+
+      # Generate lock key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] namespaced lock key
+      private def lock_key(logical_key) = "#{@namespace}:lock:#{logical_key}"
+
+      # Extract logical key from data key.
+      #
+      # @param data_k [String] namespaced data key
+      # @return [String] logical key
+      private def logical_key_from_data_key(data_k) = data_k.delete_prefix("#{@namespace}:")
+
+      DEFAULT_URL = "redis://localhost:6379/0"
+      private_constant :DEFAULT_URL
+
+      # Mask credentials in Redis URL for safe display.
+      #
+      # @param url [String, nil] Redis URL
+      # @return [String] URL with credentials masked (defaults to redis://localhost:6379/0)
+      private def mask_url(url)
+        URI.parse(url || DEFAULT_URL).tap {|uri| uri.userinfo = "***:***" if uri.userinfo }.to_s
+      end
+    end
+  end
+end