factorix 0.6.0 → 0.8.0

data/lib/factorix/cache/s3.rb

@@ -0,0 +1,388 @@
+# frozen_string_literal: true
+
+begin
+  require "aws-sdk-s3"
+rescue LoadError
+  raise Factorix::Error, "aws-sdk-s3 gem is required for S3 cache backend. Add it to your Gemfile."
+end
+
+require "securerandom"
+
+module Factorix
+  module Cache
+    # S3-based cache storage implementation.
+    #
+    # Stores cache entries in AWS S3 with automatic prefix generation.
+    # TTL is managed via custom metadata on objects.
+    # Supports distributed locking using conditional PUT operations.
+    #
+    # @example Configuration
+    #   Factorix.configure do |config|
+    #     config.cache.download.backend = :s3
+    #     config.cache.download.s3.bucket = "factorix-develop"
+    #     config.cache.download.s3.region = "ap-northeast-1"
+    #     config.cache.download.s3.lock_timeout = 30
+    #   end
+    class S3 < Base
+      # @!parse
+      #   # @return [Dry::Logger::Dispatcher]
+      #   attr_reader :logger
+      include Import[:logger]
+
+      # Default timeout for distributed lock acquisition in seconds.
+      DEFAULT_LOCK_TIMEOUT = 30
+      public_constant :DEFAULT_LOCK_TIMEOUT
+
+      # TTL for distributed locks in seconds.
+      LOCK_TTL = 30
+      private_constant :LOCK_TTL
+
+      # Metadata key for storing expiration timestamp.
+      EXPIRES_AT_KEY = "expires-at"
+      private_constant :EXPIRES_AT_KEY
+
+      # Metadata key for storing logical key.
+      LOGICAL_KEY_KEY = "logical-key"
+      private_constant :LOGICAL_KEY_KEY
+
+      # Initialize a new S3 cache storage.
+      #
+      # @param bucket [String] S3 bucket name (required)
+      # @param region [String, nil] AWS region (defaults to AWS_REGION env or SDK default)
+      # @param cache_type [String, Symbol] Cache type for prefix (e.g., :api, :download)
+      # @param lock_timeout [Integer] Timeout for lock acquisition in seconds
+      # @param ttl [Integer, nil] time-to-live in seconds (nil for unlimited)
+      def initialize(bucket:, cache_type:, region: nil, lock_timeout: DEFAULT_LOCK_TIMEOUT, **)
+        super(**)
+        @client = Aws::S3::Client.new(**{region:}.compact)
+        @bucket = bucket
+        @prefix = "cache/#{cache_type}/"
+        @lock_timeout = lock_timeout
+        logger.info("Initializing S3 cache", bucket: @bucket, prefix: @prefix, ttl: @ttl, lock_timeout: @lock_timeout)
+      end
+
+      # Check if a cache entry exists and is not expired.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if the cache entry exists and is valid
+      def exist?(key)
+        head_object(key)
+        !expired?(key)
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      # Read a cached entry.
+      #
+      # @param key [String] logical cache key
+      # @return [String, nil] cached content or nil if not found/expired
+      def read(key)
+        return nil if expired?(key)
+
+        resp = @client.get_object(bucket: @bucket, key: storage_key(key))
+        resp.body.read
+      rescue Aws::S3::Errors::NoSuchKey
+        nil
+      end
+
+      # Write cached content to a file.
+      #
+      # @param key [String] logical cache key
+      # @param output [Pathname] path to write the cached content
+      # @return [Boolean] true if written successfully, false if not found/expired
+      def write_to(key, output)
+        return false if expired?(key)
+
+        @client.get_object(bucket: @bucket, key: storage_key(key), response_target: output.to_s)
+        logger.debug("Cache hit", key:)
+        true
+      rescue Aws::S3::Errors::NoSuchKey
+        false
+      end
+
+      # Store data in the cache.
+      #
+      # @param key [String] logical cache key
+      # @param src [Pathname] path to the source file
+      # @return [Boolean] true if stored successfully
+      def store(key, src)
+        metadata = {LOGICAL_KEY_KEY => key}
+        metadata[EXPIRES_AT_KEY] = (Time.now.to_i + @ttl).to_s if @ttl
+
+        @client.put_object(
+          bucket: @bucket,
+          key: storage_key(key),
+          body: src.binread,
+          metadata:
+        )
+
+        logger.debug("Stored in cache", key:, size_bytes: src.size)
+        true
+      end
+
+      # Delete a cache entry.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if deleted, false if not found
+      def delete(key)
+        return false unless exist_without_expiry_check?(key)
+
+        @client.delete_object(bucket: @bucket, key: storage_key(key))
+        logger.debug("Deleted from cache", key:)
+        true
+      end
+
+      # Clear all cache entries in this prefix.
+      #
+      # @return [void]
+      def clear
+        logger.info("Clearing S3 cache prefix", bucket: @bucket, prefix: @prefix)
+        count = 0
+
+        list_all_objects do |objects|
+          keys_to_delete = objects.filter_map {|obj| {key: obj.key} unless obj.key.end_with?(".lock") }
+          next if keys_to_delete.empty?
+
+          @client.delete_objects(bucket: @bucket, delete: {objects: keys_to_delete})
+          count += keys_to_delete.size
+        end
+
+        logger.info("Cache cleared", objects_removed: count)
+      end
+
+      # Get the age of a cache entry in seconds.
+      #
+      # @param key [String] logical cache key
+      # @return [Float, nil] age in seconds, or nil if entry doesn't exist
+      def age(key)
+        resp = head_object(key)
+        Time.now - resp.last_modified
+      rescue Aws::S3::Errors::NotFound
+        nil
+      end
+
+      # Check if a cache entry has expired based on TTL.
+      #
+      # @param key [String] logical cache key
+      # @return [Boolean] true if expired, false otherwise
+      def expired?(key)
+        return false if @ttl.nil?
+
+        resp = head_object(key)
+        value = resp.metadata[EXPIRES_AT_KEY]
+        return false if value.nil?
+
+        Time.now.to_i > Integer(value, 10)
+      rescue Aws::S3::Errors::NotFound
+        true
+      end
+
+      # Get the size of a cached entry in bytes.
+      #
+      # @param key [String] logical cache key
+      # @return [Integer, nil] size in bytes, or nil if entry doesn't exist/expired
+      def size(key)
+        return nil if expired?(key)
+
+        resp = head_object(key)
+        resp.content_length
+      rescue Aws::S3::Errors::NotFound
+        nil
+      end
+
+      # Execute a block with a distributed lock.
+      # Uses conditional PUT for lock acquisition.
+      #
+      # @param key [String] logical cache key
+      # @yield block to execute with lock held
+      # @raise [LockTimeoutError] if lock cannot be acquired within timeout
+      def with_lock(key)
+        lkey = lock_key(key)
+        lock_value = SecureRandom.uuid
+        deadline = Time.now + @lock_timeout
+
+        loop do
+          if try_acquire_lock(lkey, lock_value)
+            logger.debug("Acquired lock", key:)
+            break
+          end
+
+          cleanup_stale_lock(lkey)
+          raise LockTimeoutError, "Failed to acquire lock for key: #{key}" if Time.now > deadline
+
+          sleep 0.1
+        end
+
+        begin
+          yield
+        ensure
+          @client.delete_object(bucket: @bucket, key: lkey)
+          logger.debug("Released lock", key:)
+        end
+      end
+
+      # Enumerate cache entries.
+      #
+      # @yield [key, entry] logical key and Entry object
+      # @yieldparam key [String] logical cache key
+      # @yieldparam entry [Entry] cache entry metadata
+      # @return [Enumerator] if no block given
+      def each
+        return enum_for(__method__) unless block_given?
+
+        list_all_objects do |objects|
+          objects.each do |obj|
+            next if obj.key.end_with?(".lock")
+
+            logical_key, entry = build_entry_with_metadata(obj)
+            next if logical_key.nil? # Skip entries without logical key metadata
+
+            yield logical_key, entry
+          end
+        end
+      end
+
+      # Return backend-specific information.
+      #
+      # @return [Hash] backend configuration
+      def backend_info
+        {
+          type: "s3",
+          bucket: @bucket,
+          prefix: @prefix,
+          lock_timeout: @lock_timeout
+        }
+      end
+
+      # Generate a hashed internal key for the given logical key.
+      # Uses SHA1 to create a unique, deterministic key.
+      # Use Digest(:SHA1) instead of Digest::SHA1 for thread-safety (Ruby 2.2+)
+      #
+      # @param logical_key [String] logical key to hash
+      # @return [String] SHA1 hash of the logical key
+      private def storage_key_for(logical_key) = Digest(:SHA1).hexdigest(logical_key)
+
+      # Generate storage key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] prefixed hashed storage key
+      private def storage_key(logical_key) = "#{@prefix}#{storage_key_for(logical_key)}"
+
+      # Generate lock key for the given logical key.
+      #
+      # @param logical_key [String] logical key
+      # @return [String] lock key
+      private def lock_key(logical_key) = "#{@prefix}#{storage_key_for(logical_key)}.lock"
+
+      # Get object metadata.
+      #
+      # @param key [String] logical key
+      # @return [Aws::S3::Types::HeadObjectOutput] object metadata
+      private def head_object(key)
+        @client.head_object(bucket: @bucket, key: storage_key(key))
+      end
+
+      # Check if object exists without expiry check.
+      #
+      # @param key [String] logical key
+      # @return [Boolean] true if exists
+      private def exist_without_expiry_check?(key)
+        head_object(key)
+        true
+      rescue Aws::S3::Errors::NotFound
+        false
+      end
+
+      # Try to acquire a distributed lock.
+      #
+      # @param lkey [String] lock key
+      # @param lock_value [String] unique lock value
+      # @return [Boolean] true if lock acquired
+      private def try_acquire_lock(lkey, lock_value)
+        lock_body = "#{lock_value}:#{Time.now.to_i + LOCK_TTL}"
+        @client.put_object(
+          bucket: @bucket,
+          key: lkey,
+          body: lock_body,
+          if_none_match: "*"
+        )
+        true
+      rescue Aws::S3::Errors::PreconditionFailed
+        false
+      end
+
+      # Clean up stale lock if expired.
+      #
+      # @param lkey [String] lock key
+      private def cleanup_stale_lock(lkey)
+        resp = @client.get_object(bucket: @bucket, key: lkey)
+        lock_data = resp.body.read
+        _lock_value, expires_at = lock_data.split(":")
+
+        if expires_at && Time.now.to_i > Integer(expires_at, 10)
+          @client.delete_object(bucket: @bucket, key: lkey)
+          logger.debug("Cleaned up stale lock", key: lkey)
+        end
+      rescue Aws::S3::Errors::NoSuchKey
+        # Lock doesn't exist, nothing to clean up
+      end
+
+      # List all objects in the prefix with pagination.
+      #
+      # @yield [Array<Aws::S3::Types::Object>] batch of objects
+      private def list_all_objects
+        continuation_token = nil
+
+        loop do
+          resp = @client.list_objects_v2(
+            bucket: @bucket,
+            prefix: @prefix,
+            continuation_token:
+          )
+
+          yield resp.contents if resp.contents.any?
+
+          break unless resp.is_truncated
+
+          continuation_token = resp.next_continuation_token
+        end
+      end
+
+      # Build an Entry from an S3 object, fetching metadata to get logical key.
+      #
+      # @param obj [Aws::S3::Types::Object] S3 object
+      # @return [Array(String, Entry), Array(nil, nil)] logical key and entry, or nils if metadata missing
+      private def build_entry_with_metadata(obj)
+        resp = @client.head_object(bucket: @bucket, key: obj.key)
+        logical_key = resp.metadata[LOGICAL_KEY_KEY]
+        return [nil, nil] if logical_key.nil?
+
+        age = Time.now - obj.last_modified
+        expired = check_expired_from_head_response(resp)
+
+        entry = Entry[
+          size: obj.size,
+          age:,
+          expired:
+        ]
+
+        [logical_key, entry]
+      rescue Aws::S3::Errors::NotFound
+        [nil, nil]
+      end
+
+      # Check if object is expired from head_object response.
+      #
+      # @param resp [Aws::S3::Types::HeadObjectOutput] head_object response
+      # @return [Boolean] true if expired
+      private def check_expired_from_head_response(resp)
+        return false if @ttl.nil?
+
+        value = resp.metadata[EXPIRES_AT_KEY]
+        return false if value.nil?
+
+        Time.now.to_i > Integer(value, 10)
+      end
+    end
+  end
+end

data/lib/factorix/cli/commands/cache/evict.rb

@@ -48,7 +48,6 @@ module Factorix
           def call(caches: nil, all: false, expired: false, older_than: nil, **)
             validate_options!(all, expired, older_than)
 
-            @now = Time.now
             @older_than_seconds = parse_age(older_than) if older_than
 
             cache_names = resolve_cache_names(caches)
@@ -114,25 +113,26 @@ module Factorix
           # @param expired [Boolean] remove expired entries only
           # @return [Hash] eviction result with :count and :size
           private def evict_cache(name, all:, expired:)
-            config = Factorix.config.cache.public_send(name)
-            cache_dir = config.dir
-            ttl = config.ttl
-
-            return {count: 0, size: 0} unless cache_dir.exist?
+            cache = Container.resolve(:"#{name}_cache")
 
             count = 0
             size = 0
 
-            cache_dir.glob("**/*").each do |path|
-              next unless path.file?
-              next if path.extname == ".lock"
+            # Collect keys to evict (we can't modify during iteration)
+            to_evict = []
+            cache.each do |key, entry|
+              next unless should_evict?(entry, all:, expired:)
+
+              to_evict << [key, entry.size]
+            end
 
-              next unless should_evict?(path, ttl, all:, expired:)
+            # Perform eviction
+            to_evict.each do |key, entry_size|
+              next unless cache.delete(key)
 
-              size += path.size
-              path.delete
               count += 1
-              logger.debug("Evicted cache entry", path: path.to_s)
+              size += entry_size
+              logger.debug("Evicted cache entry", key:)
             end
 
             logger.info("Evicted cache entries", cache: name, count:, size:)
@@ -141,23 +141,18 @@ module Factorix
 
           # Determine if a cache entry should be evicted
           #
-          # @param path [Pathname] path to cache entry
-          # @param ttl [Integer, nil] cache TTL
+          # @param entry [Cache::Entry] cache entry
           # @param all [Boolean] remove all entries
           # @param expired [Boolean] remove expired entries only
           # @return [Boolean] true if entry should be evicted
-          private def should_evict?(path, ttl, all:, expired:)
+          private def should_evict?(entry, all:, expired:)
             return true if all
 
-            age_seconds = @now - path.mtime
-
             if expired
-              return false if ttl.nil? # No TTL means never expires
-
-              age_seconds > ttl
+              entry.expired?
             else
               # --older-than
-              age_seconds > @older_than_seconds
+              entry.age > @older_than_seconds
             end
           end
 

data/lib/factorix/cli/commands/cache/stat.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "dry/inflector"
 require "json"
 
 module Factorix
@@ -8,13 +9,12 @@ module Factorix
       module Cache
         # Display cache statistics
         #
-        # This command outout.puts statistics for all cache stores
+        # This command outputs statistics for all cache stores
         # in a human-readable or JSON format.
         #
         # @example
         #   $ factorix cache stat
         #   download:
-        #     Directory:      ~/.cache/factorix/download
         #     TTL:            unlimited
         #     Entries:        42 / 42 (100.0% valid)
         #     ...
@@ -41,7 +41,6 @@ module Factorix
           def call(json:, **)
             logger.debug("Collecting cache statistics")
 
-            @now = Time.now
             cache_names = Factorix.config.cache.values.keys
             stats = cache_names.to_h {|name| [name, collect_stats(name)] }
 
@@ -52,52 +51,42 @@ module Factorix
             end
           end
 
+          # Collect statistics for a cache
+          #
+          # @param name [Symbol] cache name
+          # @return [Hash] cache statistics
           private def collect_stats(name)
+            cache = Container.resolve(:"#{name}_cache")
             config = Factorix.config.cache.public_send(name)
-            cache_dir = config.dir
 
-            entries = scan_entries(cache_dir, config.ttl)
+            entries = scan_entries(cache)
 
             {
-              directory: cache_dir.to_s,
               ttl: config.ttl,
-              max_file_size: config.max_file_size,
-              compression_threshold: config.compression_threshold,
               entries: build_entry_stats(entries),
               size: build_size_stats(entries),
               age: build_age_stats(entries),
-              stale_locks: count_stale_locks(cache_dir)
+              backend_info: cache.backend_info
             }
           end
 
-          # Scan cache directory and collect entry information
+          # Collect cache entries using the cache interface
           #
-          # @param cache_dir [Pathname] cache directory path
-          # @param ttl [Integer, nil] time-to-live in seconds
-          # @return [Array<Hash>] array of entry info hashes
-          private def scan_entries(cache_dir, ttl)
-            return [] unless cache_dir.exist?
-
+          # @param cache [Cache::Base] cache instance
+          # @return [Array<Cache::Entry>] array of cache entries
+          private def scan_entries(cache)
             entries = []
-            cache_dir.glob("**/*").each do |path|
-              next unless path.file?
-              next if path.extname == ".lock"
-
-              age_seconds = @now - path.mtime
-              expired = ttl ? age_seconds > ttl : false
-
-              entries << {size: path.size, age: age_seconds, expired:}
-            end
+            cache.each {|_key, entry| entries << entry }
             entries
           end
 
           # Build entry count statistics
           #
-          # @param entries [Array<Hash>] entry info array
+          # @param entries [Array<Cache::Entry>] entry array
           # @return [Hash] entry statistics
           private def build_entry_stats(entries)
             total = entries.size
-            valid = entries.count {|e| !e[:expired] }
+            valid = entries.count {|e| !e.expired? }
             expired = total - valid
 
             {total:, valid:, expired:}
@@ -105,37 +94,26 @@ module Factorix
 
           # Build size statistics
           #
-          # @param entries [Array<Hash>] entry info array
+          # @param entries [Array<Cache::Entry>] entry array
           # @return [Hash] size statistics
           private def build_size_stats(entries)
             return {total: 0, avg: 0, min: 0, max: 0} if entries.empty?
 
-            sizes = entries.map {|e| e[:size] }
+            sizes = entries.map(&:size)
             {total: sizes.sum, avg: sizes.sum / sizes.size, min: sizes.min, max: sizes.max}
           end
 
           # Build age statistics
           #
-          # @param entries [Array<Hash>] entry info array
+          # @param entries [Array<Cache::Entry>] entry array
           # @return [Hash] age statistics
           private def build_age_stats(entries)
             return {oldest: nil, newest: nil, avg: nil} if entries.empty?
 
-            ages = entries.map {|e| e[:age] }
+            ages = entries.map(&:age)
             {oldest: ages.max, newest: ages.min, avg: ages.sum / ages.size}
           end
 
-          # Count stale lock files
-          #
-          # @param cache_dir [Pathname] cache directory path
-          # @return [Integer] number of stale lock files
-          private def count_stale_locks(cache_dir)
-            return 0 unless cache_dir.exist?
-
-            lock_lifetime = Factorix::Cache::FileSystem::LOCK_FILE_LIFETIME
-            cache_dir.glob("**/*.lock").count {|path| @now - path.mtime > lock_lifetime }
-          end
-
           # Output statistics in text format (ccache-style)
           #
           # @param stats [Hash] statistics for all caches
@@ -153,10 +131,7 @@ module Factorix
           # @param data [Hash] cache statistics
           # @return [void]
           private def output_cache_stats(data)
-            out.puts "  Directory:      #{data[:directory]}"
             out.puts "  TTL:            #{format_ttl(data[:ttl])}"
-            out.puts "  Max file size:  #{format_size(data[:max_file_size])}"
-            out.puts "  Compression:    #{format_compression(data[:compression_threshold])}"
 
             entries = data[:entries]
             valid_pct = entries[:total] > 0 ? (Float(entries[:valid]) / entries[:total] * 100) : 0.0
@@ -172,7 +147,43 @@ module Factorix
               out.puts "  Age:            -"
             end
 
-            out.puts "  Stale locks:    #{data[:stale_locks]}"
+            output_backend_info(data[:backend_info])
+          end
+
+          INFLECTOR = Dry::Inflector.new do |inflections|
+            inflections.acronym("URL")
+          end
+          private_constant :INFLECTOR
+
+          # Output backend-specific information
+          #
+          # @param info [Hash] backend-specific information
+          # @return [void]
+          private def output_backend_info(info)
+            return if info.empty?
+
+            out.puts "  Backend:"
+            info.each do |key, value|
+              label = INFLECTOR.humanize(key)
+              formatted_value = format_backend_value(key, value)
+              out.puts "    %-20s %s" % [label + ":", formatted_value]
+            end
+          end
+
+          # Format a backend info value for display
+          #
+          # @param key [Symbol] the key name
+          # @param value [Object] the value to format
+          # @return [String] formatted value
+          private def format_backend_value(key, value)
+            case key
+            when :max_file_size, :compression_threshold
+              value.nil? ? "unlimited" : format_size(value)
+            when :lock_timeout
+              format_duration(value)
+            else
+              value.to_s
+            end
           end
 
           # Format TTL value for display
@@ -182,18 +193,6 @@ module Factorix
           private def format_ttl(ttl)
             ttl.nil? ? "unlimited" : format_duration(ttl)
           end
-
-          # Format compression threshold for display
-          #
-          # @param threshold [Integer, nil] compression threshold in bytes
-          # @return [String] formatted compression setting
-          private def format_compression(threshold)
-            case threshold
-            when nil then "disabled"
-            when 0 then "enabled (always)"
-            else "enabled (>= #{format_size(threshold)})"
-            end
-          end
         end
       end
     end