langfuse-rb 0.1.0 → 0.3.0

data/lib/langfuse/rails_cache_adapter.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "prompt_cache"
+require_relative "stale_while_revalidate"
+
 module Langfuse
   # Rails.cache adapter for distributed caching with Redis
   #
@@ -12,20 +15,30 @@ module Langfuse
   #   adapter.get("greeting:1") # => prompt_data
   #
   class RailsCacheAdapter
-    attr_reader :ttl, :namespace, :lock_timeout
+    include StaleWhileRevalidate
+
+    attr_reader :ttl, :namespace, :lock_timeout, :stale_ttl, :thread_pool, :logger
 
     # Initialize a new Rails.cache adapter
     #
     # @param ttl [Integer] Time-to-live in seconds (default: 60)
     # @param namespace [String] Cache key namespace (default: "langfuse")
     # @param lock_timeout [Integer] Lock timeout in seconds for stampede protection (default: 10)
+    # @param stale_ttl [Integer] Stale TTL for SWR in seconds (default: 0, SWR disabled).
+    #   Note: :indefinite is normalized to 1000 years by Config before being passed here.
+    # @param refresh_threads [Integer] Number of background refresh threads (default: 5)
+    # @param logger [Logger, nil] Logger instance for error reporting (default: nil, creates new logger)
     # @raise [ConfigurationError] if Rails.cache is not available
-    def initialize(ttl: 60, namespace: "langfuse", lock_timeout: 10)
+    def initialize(ttl: 60, namespace: "langfuse", lock_timeout: 10, stale_ttl: 0, refresh_threads: 5,
+                   logger: default_logger)
       validate_rails_cache!
 
       @ttl = ttl
       @namespace = namespace
       @lock_timeout = lock_timeout
+      @stale_ttl = stale_ttl
+      @logger = logger
+      initialize_swr(refresh_threads: refresh_threads) if swr_enabled?
     end
 
     # Get a value from the cache
@@ -42,14 +55,57 @@ module Langfuse
     # @param value [Object] Value to cache
     # @return [Object] The cached value
     def set(key, value)
-      Rails.cache.write(namespaced_key(key), value, expires_in: ttl)
+      # Calculate expiration: use total_ttl if SWR enabled, otherwise just ttl
+      expires_in = swr_enabled? ? total_ttl : ttl
+      Rails.cache.write(namespaced_key(key), value, expires_in:)
       value
     end
 
-    # Fetch a value from cache with distributed lock for stampede protection
+    # Clear the entire Langfuse cache namespace
+    #
+    # Note: This uses delete_matched which may not be available on all cache stores.
+    # Works with Redis, Memcached, and memory stores. File store support varies.
+    #
+    # @return [void]
+    def clear
+      # Delete all keys matching the namespace pattern
+      Rails.cache.delete_matched("#{namespace}:*")
+    end
+
+    # Get current cache size
+    #
+    # Note: Rails.cache doesn't provide a size method, so we return nil
+    # to indicate this operation is not supported.
+    #
+    # @return [nil]
+    def size
+      nil
+    end
+
+    # Check if cache is empty
+    #
+    # Note: Rails.cache doesn't provide an efficient way to check if empty,
+    # so we return false to indicate this operation is not supported.
+    #
+    # @return [Boolean] Always returns false (unsupported operation)
+    def empty?
+      false
+    end
+
+    # Build a cache key from prompt name and options
+    #
+    # @param name [String] Prompt name
+    # @param version [Integer, nil] Optional version
+    # @param label [String, nil] Optional label
+    # @return [String] Cache key
+    def self.build_key(name, version: nil, label: nil)
+      PromptCache.build_key(name, version: version, label: label)
+    end
+
+    # Fetch a value from cache with lock for stampede protection
     #
     # This method prevents cache stampedes (thundering herd) by ensuring only one
-    # process fetches from the source when the cache is empty. Other processes wait
+    # process/thread fetches from the source when the cache is empty. Others wait
     # for the first one to populate the cache.
     #
     # Uses exponential backoff: 50ms, 100ms, 200ms (3 retries max, ~350ms total).
@@ -60,7 +116,7 @@ module Langfuse
     # @return [Object] Cached or freshly fetched value
     #
     # @example
-    #   adapter.fetch_with_lock("greeting:v1") do
+    #   cache.fetch_with_lock("greeting:v1") do
     #     api_client.get_prompt("greeting")
     #   end
     def fetch_with_lock(key)
@@ -68,8 +124,8 @@ module Langfuse
       cached = get(key)
       return cached if cached
 
-      # 2. Cache miss - try to acquire distributed lock
-      lock_key = "#{namespaced_key(key)}:lock"
+      # 2. Cache miss - try to acquire lock
+      lock_key = build_lock_key(key)
 
       if acquire_lock(lock_key)
         begin
@@ -92,74 +148,57 @@ module Langfuse
       end
     end
 
-    # Clear the entire Langfuse cache namespace
-    #
-    # Note: This uses delete_matched which may not be available on all cache stores.
-    # Works with Redis, Memcached, and memory stores. File store support varies.
-    #
-    # @return [void]
-    def clear
-      # Delete all keys matching the namespace pattern
-      Rails.cache.delete_matched("#{namespace}:*")
-    end
+    private
 
-    # Get current cache size
-    #
-    # Note: Rails.cache doesn't provide a size method, so we return nil
-    # to indicate this operation is not supported.
-    #
-    # @return [nil]
-    def size
-      nil
-    end
+    # Implementation of StaleWhileRevalidate abstract methods
 
-    # Check if cache is empty
-    #
-    # Note: Rails.cache doesn't provide an efficient way to check if empty,
-    # so we return false to indicate this operation is not supported.
+    # Get value from cache (SWR interface)
     #
-    # @return [Boolean] Always returns false (unsupported operation)
-    def empty?
-      false
+    # @param key [String] Cache key
+    # @return [Object, nil] Cached value
+    def cache_get(key)
+      get(key)
     end
 
-    # Build a cache key from prompt name and options
+    # Set value in cache (SWR interface)
     #
-    # @param name [String] Prompt name
-    # @param version [Integer, nil] Optional version
-    # @param label [String, nil] Optional label
-    # @return [String] Cache key
-    def self.build_key(name, version: nil, label: nil)
-      PromptCache.build_key(name, version: version, label: label)
+    # @param key [String] Cache key
+    # @param value [Object] Value to cache (expects CacheEntry)
+    # @return [Object] The cached value
+    def cache_set(key, value)
+      set(key, value)
     end
 
-    private
-
-    # Add namespace prefix to cache key
+    # Build lock key with namespace
     #
-    # @param key [String] Original cache key
-    # @return [String] Namespaced cache key
-    def namespaced_key(key)
-      "#{namespace}:#{key}"
+    # Used for both fetch operations (stampede protection) and refresh operations
+    # (preventing duplicate background refreshes).
+    #
+    # @param key [String] Cache key
+    # @return [String] Namespaced lock key
+    def build_lock_key(key)
+      "#{namespaced_key(key)}:lock"
     end
 
-    # Acquire a distributed lock using Rails.cache
+    # Acquire a lock using Rails.cache
     #
-    # Uses atomic "write if not exists" operation to ensure only one process
-    # can acquire the lock.
+    # Used for both fetch operations and refresh operations.
+    # Uses the configured lock_timeout for all locking scenarios.
     #
     # @param lock_key [String] Full lock key (already namespaced)
-    # @return [Boolean] true if lock was acquired, false if already held by another process
+    # @return [Boolean] true if lock was acquired, false if already held
     def acquire_lock(lock_key)
       Rails.cache.write(
         lock_key,
         true,
         unless_exist: true, # Atomic: only write if key doesn't exist
-        expires_in: lock_timeout # Auto-expire to prevent deadlocks
+        expires_in: lock_timeout # Use configured lock timeout
       )
     end
 
-    # Release a distributed lock
+    # Release a lock
+    #
+    # Used for both fetch and refresh operations.
     #
     # @param lock_key [String] Full lock key (already namespaced)
     # @return [void]
@@ -172,7 +211,7 @@ module Langfuse
     # Uses exponential backoff: 50ms, 100ms, 200ms (3 retries, ~350ms total).
     # This gives the lock holder time to fetch and populate the cache.
     #
-    # @param key [String] Cache key (not namespaced)
+    # @param key [String] Cache key
     # @return [Object, nil] Cached value if found, nil if still empty after waiting
     def wait_for_cache(key)
       intervals = [0.05, 0.1, 0.2] # 50ms, 100ms, 200ms (exponential backoff)
@@ -186,6 +225,16 @@ module Langfuse
       nil # Cache still empty after all retries
     end
 
+    # Rails.cache-specific helper methods
+
+    # Add namespace prefix to cache key
+    #
+    # @param key [String] Original cache key
+    # @return [String] Namespaced cache key
+    def namespaced_key(key)
+      "#{namespace}:#{key}"
+    end
+
     # Validate that Rails.cache is available
     #
     # @raise [ConfigurationError] if Rails.cache is not available
@@ -196,5 +245,16 @@ module Langfuse
       raise ConfigurationError,
             "Rails.cache is not available. Rails cache backend requires Rails with a configured cache store."
     end
+
+    # Create a default logger
+    #
+    # @return [Logger]
+    def default_logger
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger
+      else
+        Logger.new($stdout, level: Logger::WARN)
+      end
+    end
   end
 end

data/lib/langfuse/stale_while_revalidate.rb

@@ -0,0 +1,262 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Langfuse
+  # Stale-While-Revalidate caching pattern module
+  #
+  # Provides SWR functionality for cache implementations. When included,
+  # allows serving stale data immediately while refreshing in the background.
+  #
+  # Including classes must implement:
+  # - cache_get(key) - Read from cache
+  # - cache_set(key, value) - Write to cache
+  # - acquire_lock(lock_key) - Acquire lock for background refresh
+  # - release_lock(lock_key) - Release refresh lock
+  #
+  # @example
+  #   class MyCache
+  #     include Langfuse::StaleWhileRevalidate
+  #
+  #     def initialize(ttl: 60, stale_ttl: 0)
+  #       @ttl = ttl
+  #       @stale_ttl = stale_ttl
+  #       @logger = Logger.new($stdout)
+  #       initialize_swr if stale_ttl.positive?
+  #     end
+  #
+  #     def cache_get(key)
+  #       @storage[key]
+  #     end
+  #
+  #     def cache_set(key, value)
+  #       @storage[key] = value
+  #     end
+  #
+  #     def acquire_lock(lock_key)
+  #       # Implementation-specific lock acquisition
+  #     end
+  #
+  #     def release_lock(lock_key)
+  #       # Implementation-specific lock release
+  #     end
+  #   end
+  module StaleWhileRevalidate
+    # Initialize SWR infrastructure
+    #
+    # Must be called by including class after setting @stale_ttl, @ttl, and @logger.
+    # Typically called in the class's initialize method when stale_ttl is provided.
+    #
+    # @param refresh_threads [Integer] Number of background refresh threads (default: 5)
+    # @return [void]
+    def initialize_swr(refresh_threads: 5)
+      @thread_pool = initialize_thread_pool(refresh_threads)
+    end
+
+    # Fetch a value from cache with Stale-While-Revalidate support
+    #
+    # This method implements SWR caching: serves stale data immediately while
+    # refreshing in the background. Requires SWR to be enabled (stale_ttl must be positive).
+    #
+    # Three cache states:
+    # - FRESH: Return immediately, no action needed
+    # - STALE: Return stale data + trigger background refresh
+    # - EXPIRED: Must fetch fresh data synchronously
+    #
+    # @param key [String] Cache key
+    # @yield Block to execute to fetch fresh data
+    # @return [Object] Cached, stale, or freshly fetched value
+    # @raise [ConfigurationError] if SWR is not enabled (stale_ttl is not positive)
+    #
+    # @example
+    #   cache.fetch_with_stale_while_revalidate("greeting:v1") do
+    #     api_client.get_prompt("greeting")
+    #   end
+    def fetch_with_stale_while_revalidate(key, &)
+      raise ConfigurationError, "fetch_with_stale_while_revalidate requires a positive stale_ttl" unless swr_enabled?
+
+      entry = cache_get(key)
+
+      if entry&.fresh?
+        # FRESH - return immediately
+        logger.debug("CACHE HIT!")
+        entry.data
+      elsif entry&.stale?
+        # REVALIDATE - return stale + refresh in background
+        logger.debug("CACHE STALE!")
+        schedule_refresh(key, &)
+        entry.data # Instant response!
+      else
+        # MISS - must fetch synchronously
+        logger.debug("CACHE MISS!")
+        fetch_and_cache(key, &)
+      end
+    end
+
+    # Check if SWR is enabled
+    #
+    # SWR is enabled when stale_ttl is positive, meaning there's a grace period
+    # where stale data can be served while revalidating in the background.
+    #
+    # @return [Boolean] true if stale_ttl is positive
+    def swr_enabled?
+      stale_ttl.positive?
+    end
+
+    # Shutdown the cache refresh thread pool gracefully
+    #
+    # @return [void]
+    def shutdown
+      return unless @thread_pool
+
+      @thread_pool.shutdown
+      @thread_pool.wait_for_termination(5) # Wait up to 5 seconds
+    end
+
+    private
+
+    # Initialize thread pool for background refresh operations
+    #
+    # @param refresh_threads [Integer] Maximum number of refresh threads
+    # @return [Concurrent::CachedThreadPool]
+    def initialize_thread_pool(refresh_threads)
+      Concurrent::CachedThreadPool.new(
+        max_threads: refresh_threads,
+        min_threads: 0,
+        max_queue: 50,
+        fallback_policy: :discard
+      )
+    end
+
+    # Schedule a background refresh for a cache key
+    #
+    # Prevents duplicate refreshes by using a fetch lock. If another process/thread
+    # is already refreshing this key, this method returns immediately.
+    #
+    # Errors during refresh are caught and logged to prevent thread crashes.
+    #
+    # @param key [String] Cache key
+    # @yield Block to execute to fetch fresh data
+    # @return [void]
+    def schedule_refresh(key, &block)
+      # Prevent duplicate refreshes
+      lock_key = build_lock_key(key)
+      return unless acquire_lock(lock_key)
+
+      @thread_pool.post do
+        value = yield block
+        set_cache_entry(key, value)
+      rescue StandardError => e
+        logger.error("Langfuse cache refresh failed for key '#{key}': #{e.class} - #{e.message}")
+      ensure
+        release_lock(lock_key)
+      end
+    end
+
+    # Fetch data and cache it with SWR metadata
+    #
+    # @param key [String] Cache key
+    # @yield Block to execute to fetch fresh data
+    # @return [Object] Freshly fetched value
+    def fetch_and_cache(key, &block)
+      value = yield block
+      set_cache_entry(key, value)
+    end
+
+    # Set value in cache with SWR metadata (CacheEntry)
+    #
+    # @param key [String] Cache key
+    # @param value [Object] Value to cache
+    # @return [Object] The cached value
+    def set_cache_entry(key, value)
+      now = Time.now
+      fresh_until = now + ttl
+      stale_until = fresh_until + stale_ttl
+      entry = PromptCache::CacheEntry.new(value, fresh_until, stale_until)
+
+      cache_set(key, entry)
+
+      value
+    end
+
+    # Build a lock key for fetch operations
+    #
+    # Can be overridden by including class if custom key format is needed.
+    #
+    # @param key [String] Cache key
+    # @return [String] Lock key
+    def build_lock_key(key)
+      "#{key}:lock"
+    end
+
+    # Calculate total TTL (fresh + stale)
+    #
+    # @return [Integer] Total TTL in seconds
+    def total_ttl
+      ttl + stale_ttl
+    end
+
+    # Abstract methods that must be implemented by including class
+
+    # Get a value from cache
+    #
+    # @param key [String] Cache key
+    # @return [Object, nil] Cached value or nil
+    # @raise [NotImplementedError] if not implemented by including class
+    def cache_get(_key)
+      raise NotImplementedError, "#{self.class} must implement #cache_get"
+    end
+
+    # Set a value in cache
+    #
+    # @param key [String] Cache key
+    # @param value [Object] Value to cache
+    # @return [Object] The cached value
+    # @raise [NotImplementedError] if not implemented by including class
+    def cache_set(_key, _value)
+      raise NotImplementedError, "#{self.class} must implement #cache_set"
+    end
+
+    # Acquire a lock
+    #
+    # @param lock_key [String] Lock key
+    # @return [Boolean] true if lock was acquired
+    # @raise [NotImplementedError] if not implemented by including class
+    def acquire_lock(_lock_key)
+      raise NotImplementedError, "#{self.class} must implement #acquire_lock"
+    end
+
+    # Release a lock
+    #
+    # @param lock_key [String] Lock key
+    # @return [void]
+    # @raise [NotImplementedError] if not implemented by including class
+    def release_lock(_lock_key)
+      raise NotImplementedError, "#{self.class} must implement #release_lock"
+    end
+
+    # Get TTL value
+    #
+    # @return [Integer] TTL in seconds
+    # @raise [NotImplementedError] if not implemented by including class
+    def ttl
+      @ttl || raise(NotImplementedError, "#{self.class} must provide @ttl")
+    end
+
+    # Get stale TTL value
+    #
+    # @return [Integer] Stale TTL in seconds
+    # @raise [NotImplementedError] if not implemented by including class
+    def stale_ttl
+      @stale_ttl || raise(NotImplementedError, "#{self.class} must provide @stale_ttl")
+    end
+
+    # Get logger instance
+    #
+    # @return [Logger] Logger instance
+    # @raise [NotImplementedError] if not implemented by including class
+    def logger
+      @logger || raise(NotImplementedError, "#{self.class} must provide @logger")
+    end
+  end
+end

data/lib/langfuse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langfuse
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: langfuse-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - SimplePractice
@@ -13,30 +13,42 @@ dependencies:
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: faraday-retry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: mustache
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +63,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-api
   requirement: !ruby/object:Gem::Requirement
@@ -146,6 +172,7 @@ files:
 - lib/langfuse/rails_cache_adapter.rb
 - lib/langfuse/score_client.rb
 - lib/langfuse/span_processor.rb
+- lib/langfuse/stale_while_revalidate.rb
 - lib/langfuse/text_prompt_client.rb
 - lib/langfuse/types.rb
 - lib/langfuse/version.rb
@@ -171,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Ruby SDK for Langfuse - LLM observability and prompt management
 test_files: []