factorix 0.5.1 → 0.7.0

data/lib/factorix/cache/s3.rb

@@ -0,0 +1,388 @@
+# frozen_string_literal: true
+
+begin
+  require "aws-sdk-s3"
+rescue LoadError
+  raise Factorix::Error, "aws-sdk-s3 gem is required for S3 cache backend. Add it to your Gemfile."
+end
+
+require "securerandom"
+
+module Factorix
+  module Cache
+    # S3-based cache storage implementation.
+    #
+    # Stores cache entries in AWS S3 with automatic prefix generation.
+    # TTL is managed via custom metadata on objects.
+    # Supports distributed locking using conditional PUT operations.
+    #
+    # @example Configuration
+    #   Factorix.configure do |config|
+    #     config.cache.download.backend = :s3
+    #     config.cache.download.s3.bucket = "factorix-develop"
+    #     config.cache.download.s3.region = "ap-northeast-1"
+    #     config.cache.download.s3.lock_timeout = 30
+    #   end
+    class S3 < Base
+      # @!parse
+      #   # @return [Dry::Logger::Dispatcher]
+      #   attr_reader :logger
+      include Import[:logger]
+
+      # Default timeout for distributed lock acquisition in seconds.
+      DEFAULT_LOCK_TIMEOUT = 30
+      public_constant :DEFAULT_LOCK_TIMEOUT
+
+      # TTL for distributed locks in seconds.
+      LOCK_TTL = 30
+      private_constant :LOCK_TTL
+
+      # Metadata key for storing expiration timestamp.
+      EXPIRES_AT_KEY = "expires-at"
+      private_constant :EXPIRES_AT_KEY
+
+      # Metadata key for storing logical key.
+      LOGICAL_KEY_KEY = "logical-key"
+      private_constant :LOGICAL_KEY_KEY
+
+      # Initialize a new S3 cache storage.
+      #
+      # @param bucket [String] S3 bucket name (required)
+      # @param region [String, nil] AWS region (defaults to AWS_REGION env or SDK default)
+      # @param cache_type [String, Symbol] Cache type for prefix (e.g., :api, :download)
+      # @param lock_timeout [Integer] Timeout for lock acquisition in seconds
+      # @param ttl [Integer, nil] time-to-live in seconds (nil for unlimited)
+      def initialize(bucket:, cache_type:, region: nil, lock_timeout: DEFAULT_LOCK_TIMEOUT, **)
+        super(**)
+        @client = Aws::S3::Client.new(**{region:}.compact)
+        @bucket = bucket
+        @prefix = "cache/#{cache_type}/"
+        @lock_timeout = lock_timeout
+        logger.info("Initializing S3 cache", bucket: @bucket, prefix: @prefix, ttl: @ttl, lock_timeout: @lock_timeout)
+      end
+
+      # Check if a cache entry exists and is not expired.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if the cache entry exists and is valid
+      def exist?(key)
+        head_object(key)
+        !expired?(key)
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      # Read a cached entry.
+      #
+      # @param key [String] logical cache key
+      # @return [String, nil] cached content or nil if not found/expired
+      def read(key)
+        return nil if expired?(key)
+
+        resp = @client.get_object(bucket: @bucket, key: storage_key(key))
+        resp.body.read
+      rescue Aws::S3::Errors::NoSuchKey
+        nil
+      end
+
+      # Write cached content to a file.
+      #
+      # @param key [String] logical cache key
+      # @param output [Pathname] path to write the cached content
+      # @return [Boolean] true if written successfully, false if not found/expired
+      def write_to(key, output)
+        return false if expired?(key)
+
+        @client.get_object(bucket: @bucket, key: storage_key(key), response_target: output.to_s)
+        logger.debug("Cache hit", key:)
+        true
+      rescue Aws::S3::Errors::NoSuchKey
+        false
+      end
+
+      # Store data in the cache.
+      #
+      # @param key [String] logical cache key
+      # @param src [Pathname] path to the source file
+      # @return [Boolean] true if stored successfully
+      def store(key, src)
+        metadata = {LOGICAL_KEY_KEY => key}
+        metadata[EXPIRES_AT_KEY] = (Time.now.to_i + @ttl).to_s if @ttl
+
+        @client.put_object(
+          bucket: @bucket,
+          key: storage_key(key),
+          body: src.binread,
+          metadata:
+        )
+
+        logger.debug("Stored in cache", key:, size_bytes: src.size)
+        true
+      end
+
+      # Delete a cache entry.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if deleted, false if not found
+      def delete(key)
+        return false unless exist_without_expiry_check?(key)
+
+        @client.delete_object(bucket: @bucket, key: storage_key(key))
+        logger.debug("Deleted from cache", key:)
+        true
+      end
+
+      # Clear all cache entries in this prefix.
+      #
+      # @return [void]
+      def clear
+        logger.info("Clearing S3 cache prefix", bucket: @bucket, prefix: @prefix)
+        count = 0
+
+        list_all_objects do |objects|
+          keys_to_delete = objects.filter_map {|obj| {key: obj.key} unless obj.key.end_with?(".lock") }
+          next if keys_to_delete.empty?
+
+          @client.delete_objects(bucket: @bucket, delete: {objects: keys_to_delete})
+          count += keys_to_delete.size
+        end
+
+        logger.info("Cache cleared", objects_removed: count)
+      end
+
+      # Get the age of a cache entry in seconds.
+      #
+      # @param key [String] logical cache key
+      # @return [Float, nil] age in seconds, or nil if entry doesn't exist
+      def age(key)
+        resp = head_object(key)
+        Time.now - resp.last_modified
+      rescue Aws::S3::Errors::NotFound
+        nil
+      end
+
+      # Check if a cache entry has expired based on TTL.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if expired, false otherwise
+      def expired?(key)
+        return false if @ttl.nil?
+
+        resp = head_object(key)
+        value = resp.metadata[EXPIRES_AT_KEY]
+        return false if value.nil?
+
+        Time.now.to_i > Integer(value, 10)
+      rescue Aws::S3::Errors::NotFound
+        true
+      end
+
+      # Get the size of a cached entry in bytes.
+      #
+      # @param key [String] logical cache key
+      # @return [Integer, nil] size in bytes, or nil if entry doesn't exist/expired
+      def size(key)
+        return nil if expired?(key)
+
+        resp = head_object(key)
+        resp.content_length
+      rescue Aws::S3::Errors::NotFound
+        nil
+      end
+
+      # Execute a block with a distributed lock.
+      # Uses conditional PUT for lock acquisition.
+      #
+      # @param key [String] logical cache key
+      # @yield block to execute with lock held
+      # @raise [LockTimeoutError] if lock cannot be acquired within timeout
+      def with_lock(key)
+        lkey = lock_key(key)
+        lock_value = SecureRandom.uuid
+        deadline = Time.now + @lock_timeout
+
+        loop do
+          if try_acquire_lock(lkey, lock_value)
+            logger.debug("Acquired lock", key:)
+            break
+          end
+
+          cleanup_stale_lock(lkey)
+          raise LockTimeoutError, "Failed to acquire lock for key: #{key}" if Time.now > deadline
+
+          sleep 0.1
+        end
+
+        begin
+          yield
+        ensure
+          @client.delete_object(bucket: @bucket, key: lkey)
+          logger.debug("Released lock", key:)
+        end
+      end
+
+      # Enumerate cache entries.
+      #
+      # @yield [key, entry] logical key and Entry object
+      # @yieldparam key [String] logical cache key
+      # @yieldparam entry [Entry] cache entry metadata
+      # @return [Enumerator] if no block given
+      def each
+        return enum_for(__method__) unless block_given?
+
+        list_all_objects do |objects|
+          objects.each do |obj|
+            next if obj.key.end_with?(".lock")
+
+            logical_key, entry = build_entry_with_metadata(obj)
+            next if logical_key.nil? # Skip entries without logical key metadata
+
+            yield logical_key, entry
+          end
+        end
+      end
+
+      # Return backend-specific information.
+      #
+      # @return [Hash] backend configuration
+      def backend_info
+        {
+          type: "s3",
+          bucket: @bucket,
+          prefix: @prefix,
+          lock_timeout: @lock_timeout
+        }
+      end
+
+      # Generate a hashed internal key for the given logical key.
+      # Uses SHA1 to create a unique, deterministic key.
+      # Use Digest(:SHA1) instead of Digest::SHA1 for thread-safety (Ruby 2.2+)
+      #
+      # @param logical_key [String] logical key to hash
+      # @return [String] SHA1 hash of the logical key
+      private def storage_key_for(logical_key) = Digest(:SHA1).hexdigest(logical_key)
+
+      # Generate storage key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] prefixed hashed storage key
+      private def storage_key(logical_key) = "#{@prefix}#{storage_key_for(logical_key)}"
+
+      # Generate lock key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] lock key
+      private def lock_key(logical_key) = "#{@prefix}#{storage_key_for(logical_key)}.lock"
+
+      # Get object metadata.
+      #
+      # @param key [String] logical key
+      # @return [Aws::S3::Types::HeadObjectOutput] object metadata
+      private def head_object(key)
+        @client.head_object(bucket: @bucket, key: storage_key(key))
+      end
+
+      # Check if object exists without expiry check.
+      #
+      # @param key [String] logical key
+      # @return [Boolean] true if exists
+      private def exist_without_expiry_check?(key)
+        head_object(key)
+        true
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      # Try to acquire a distributed lock.
+      #
+      # @param lkey [String] lock key
+      # @param lock_value [String] unique lock value
+      # @return [Boolean] true if lock acquired
+      private def try_acquire_lock(lkey, lock_value)
+        lock_body = "#{lock_value}:#{Time.now.to_i + LOCK_TTL}"
+        @client.put_object(
+          bucket: @bucket,
+          key: lkey,
+          body: lock_body,
+          if_none_match: "*"
+        )
+        true
+      rescue Aws::S3::Errors::PreconditionFailed
+        false
+      end
+
+      # Clean up stale lock if expired.
+      #
+      # @param lkey [String] lock key
+      private def cleanup_stale_lock(lkey)
+        resp = @client.get_object(bucket: @bucket, key: lkey)
+        lock_data = resp.body.read
+        _lock_value, expires_at = lock_data.split(":")
+
+        if expires_at && Time.now.to_i > Integer(expires_at, 10)
+          @client.delete_object(bucket: @bucket, key: lkey)
+          logger.debug("Cleaned up stale lock", key: lkey)
+        end
+      rescue Aws::S3::Errors::NoSuchKey
+        # Lock doesn't exist, nothing to clean up
+      end
+
+      # List all objects in the prefix with pagination.
+      #
+      # @yield [Array<Aws::S3::Types::Object>] batch of objects
+      private def list_all_objects
+        continuation_token = nil
+
+        loop do
+          resp = @client.list_objects_v2(
+            bucket: @bucket,
+            prefix: @prefix,
+            continuation_token:
+          )
+
+          yield resp.contents if resp.contents.any?
+
+          break unless resp.is_truncated
+
+          continuation_token = resp.next_continuation_token
+        end
+      end
+
+      # Build an Entry from an S3 object, fetching metadata to get logical key.
+      #
+      # @param obj [Aws::S3::Types::Object] S3 object
+      # @return [Array(String, Entry), Array(nil, nil)] logical key and entry, or nils if metadata missing
+      private def build_entry_with_metadata(obj)
+        resp = @client.head_object(bucket: @bucket, key: obj.key)
+        logical_key = resp.metadata[LOGICAL_KEY_KEY]
+        return [nil, nil] if logical_key.nil?
+
+        age = Time.now - obj.last_modified
+        expired = check_expired_from_head_response(resp)
+
+        entry = Entry.new(
+          size: obj.size,
+          age:,
+          expired:
+        )
+
+        [logical_key, entry]
+      rescue Aws::S3::Errors::NotFound
+        [nil, nil]
+      end
+
+      # Check if object is expired from head_object response.
+      #
+      # @param resp [Aws::S3::Types::HeadObjectOutput] head_object response
+      # @return [Boolean] true if expired
+      private def check_expired_from_head_response(resp)
+        return false if @ttl.nil?
+
+        value = resp.metadata[EXPIRES_AT_KEY]
+        return false if value.nil?
+
+        Time.now.to_i > Integer(value, 10)
+      end
+    end
+  end
+end

data/lib/factorix/cli/commands/backup_support.rb

@@ -15,7 +15,7 @@ module Factorix
         # @param base [Class] the class prepending this module
         def self.prepended(base)
           base.class_eval do
-            option :backup_extension, type: :string, default: ".bak", desc: "Backup file extension"
+            option :backup_extension, default: ".bak", desc: "Backup file extension"
           end
         end
 

data/lib/factorix/cli/commands/base.rb

@@ -68,8 +68,8 @@ module Factorix
         def self.backup_support! = prepend BackupSupport
 
         # Common options available to all commands
-        option :config_path, type: :string, aliases: ["-c"], desc: "Path to configuration file"
-        option :log_level, type: :string, values: %w[debug info warn error fatal], desc: "Set log level"
+        option :config_path, aliases: ["-c"], desc: "Path to configuration file"
+        option :log_level, values: %w[debug info warn error fatal], desc: "Set log level"
         option :quiet, type: :flag, default: false, aliases: ["-q"], desc: "Suppress non-essential output"
 
         private def say(message, prefix: "")
@@ -78,7 +78,7 @@ module Factorix
           resolved_prefix = EMOJI_PREFIXES.fetch(prefix) { prefix.to_s }
           output = resolved_prefix.empty? ? message : "#{resolved_prefix} #{message}"
           style = STYLES.fetch(prefix, PLAIN)
-          puts style[output]
+          out.puts style[output]
         end
 
         private def quiet?

data/lib/factorix/cli/commands/cache/evict.rb

@@ -36,7 +36,7 @@ module Factorix
 
           option :all, type: :flag, default: false, desc: "Remove all entries"
           option :expired, type: :flag, default: false, desc: "Remove expired entries only"
-          option :older_than, type: :string, default: nil, desc: "Remove entries older than AGE (e.g., 30s, 5m, 2h, 7d)"
+          option :older_than, default: nil, desc: "Remove entries older than AGE (e.g., 30s, 5m, 2h, 7d)"
 
           # Execute the cache evict command
           #
@@ -48,7 +48,6 @@ module Factorix
           def call(caches: nil, all: false, expired: false, older_than: nil, **)
             validate_options!(all, expired, older_than)
 
-            @now = Time.now
             @older_than_seconds = parse_age(older_than) if older_than
 
             cache_names = resolve_cache_names(caches)
@@ -95,7 +94,7 @@ module Factorix
           # @return [Array<Symbol>] resolved cache names
           # @raise [InvalidArgumentError] if unknown cache name specified
           private def resolve_cache_names(caches)
-            all_caches = Application.config.cache.values.keys
+            all_caches = Factorix.config.cache.values.keys
 
             return all_caches if caches.nil? || caches.empty?
 
@@ -114,25 +113,26 @@ module Factorix
           # @param expired [Boolean] remove expired entries only
           # @return [Hash] eviction result with :count and :size
           private def evict_cache(name, all:, expired:)
-            config = Application.config.cache.public_send(name)
-            cache_dir = config.dir
-            ttl = config.ttl
-
-            return {count: 0, size: 0} unless cache_dir.exist?
+            cache = Container.resolve(:"#{name}_cache")
 
             count = 0
             size = 0
 
-            cache_dir.glob("**/*").each do |path|
-              next unless path.file?
-              next if path.extname == ".lock"
+            # Collect keys to evict (we can't modify during iteration)
+            to_evict = []
+            cache.each do |key, entry|
+              next unless should_evict?(entry, all:, expired:)
+
+              to_evict << [key, entry.size]
+            end
 
-              next unless should_evict?(path, ttl, all:, expired:)
+            # Perform eviction
+            to_evict.each do |key, entry_size|
+              next unless cache.delete(key)
 
-              size += path.size
-              path.delete
               count += 1
-              logger.debug("Evicted cache entry", path: path.to_s)
+              size += entry_size
+              logger.debug("Evicted cache entry", key:)
             end
 
             logger.info("Evicted cache entries", cache: name, count:, size:)
@@ -141,23 +141,18 @@ module Factorix
 
           # Determine if a cache entry should be evicted
           #
-          # @param path [Pathname] path to cache entry
-          # @param ttl [Integer, nil] cache TTL
+          # @param entry [Cache::Entry] cache entry
           # @param all [Boolean] remove all entries
           # @param expired [Boolean] remove expired entries only
           # @return [Boolean] true if entry should be evicted
-          private def should_evict?(path, ttl, all:, expired:)
+          private def should_evict?(entry, all:, expired:)
             return true if all
 
-            age_seconds = @now - path.mtime
-
             if expired
-              return false if ttl.nil? # No TTL means never expires
-
-              age_seconds > ttl
+              entry.expired?
             else
               # --older-than
-              age_seconds > @older_than_seconds
+              entry.age > @older_than_seconds
             end
           end