familia 1.2.1 → 2.0.0.pre.pre

data/Gemfile.lock

@@ -1,7 +1,11 @@
 PATH
   remote: .
   specs:
-    familia (1.2.1)
+    familia (2.0.0.pre.pre)
+      benchmark
+      connection_pool
+      csv
+      logger
       redis (>= 4.8.1, < 6.0)
       stringio (~> 3.1.1)
       uri-redis (~> 1.3)
@@ -9,76 +13,114 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.2)
+    ast (2.4.3)
+    base64 (0.3.0)
+    benchmark (0.4.1)
     byebug (11.1.3)
     coderay (1.1.3)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    csv (3.3.5)
+    diff-lcs (1.6.2)
     drydock (0.6.9)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
+    json (2.13.0)
+    kramdown (2.5.1)
+      rexml (>= 3.3.9)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
     method_source (1.1.0)
-    parallel (1.25.1)
-    parser (3.3.4.0)
+    minitest (5.25.5)
+    parallel (1.27.0)
+    parser (3.3.8.0)
       ast (~> 2.4.1)
       racc
+    prism (1.4.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
-    racc (1.8.0)
+    racc (1.8.1)
     rainbow (3.1.1)
-    redis (5.2.0)
+    redis (5.4.1)
       redis-client (>= 0.22.0)
-    redis-client (0.22.2)
+    redis-client (0.25.1)
       connection_pool
-    regexp_parser (2.9.2)
-    rexml (3.3.2)
-      strscan
-    rubocop (1.65.0)
+    regexp_parser (2.10.0)
+    rexml (3.4.1)
+    rspec (3.13.1)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.5)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.4)
+    rubocop (1.78.0)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.4, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.45.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.3)
-      parser (>= 3.3.1.0)
-    rubocop-performance (1.21.1)
-      rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-thread_safety (0.5.1)
-      rubocop (>= 0.90.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.46.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+    rubocop-thread_safety (0.7.3)
+      lint_roller (~> 1.1)
+      rubocop (~> 1.72, >= 1.72.1)
+      rubocop-ast (>= 1.44.0, < 2.0)
+    ruby-prof (1.7.2)
+      base64
     ruby-progressbar (1.13.0)
+    stackprof (0.2.27)
     storable (0.10.0)
-    stringio (3.1.1)
-    strscan (3.1.0)
+    stringio (3.1.7)
     sysinfo (0.10.0)
       drydock (< 1.0)
       storable (~> 0.10)
-    tryouts (2.4.1)
-      sysinfo (~> 0.10)
-    unicode-display_width (2.5.0)
+    tryouts (3.1.1)
+      minitest (~> 5.0)
+      rspec (~> 3.0)
+      sysinfo (>= 0.8, < 1.0)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
     uri-redis (1.3.0)
+    yard (0.9.37)
 
 PLATFORMS
-  arm64-darwin-23
   arm64-darwin-24
   ruby
 
 DEPENDENCIES
   byebug (~> 11.0)
+  concurrent-ruby (~> 1.3.5)
   familia!
+  kramdown
   pry-byebug (~> 3.10.1)
   rubocop
   rubocop-performance
   rubocop-thread_safety
-  tryouts (~> 2.4)
+  ruby-prof
+  stackprof
+  tryouts (~> 3.1.1)
+  yard (~> 0.9)
 
 BUNDLED WITH
-   2.5.13
+   2.6.2

data/README.md

@@ -1,8 +1,8 @@
-# Familia - 1.1.0-rc1 (November 2024)
+# Familia - 2.0.0
 
-**Organize and store Ruby objects in Redis. A powerful Ruby ORM (of sorts) for Redis.**
+**Organize and store Ruby objects in Valkey/Redis. A powerful Ruby ORM (of sorts) for Valkey/Redis.**
 
-Familia provides a flexible and feature-rich way to interact with Redis using Ruby objects. It's designed to make working with Redis as natural as working with Ruby classes, while offering advanced features for complex data management.
+Familia provides a flexible and feature-rich way to interact with Valkey using Ruby objects. It's designed to make working with Valkey as natural as working with Ruby classes, while offering advanced features for complex data management.
 
 ## Installation
 
@@ -18,11 +18,11 @@ Get it in one of the following ways:
 
 ### 1. Defining Horreum Classes
 
-Familia uses the concept of "Horreum" classes to represent Redis-backed objects:
+Familia uses the concept of "Horreum" classes to represent Valkey-compatible objects:
 
 ```ruby
 class Flower < Familia::Horreum
-  identifier :token
+  identifier_field :token
   field :name
   list :owners
   set :tags
@@ -38,20 +38,20 @@ You can define identifiers in various ways:
 
 ```ruby
 class User < Familia::Horreum
-  identifier :email
+  identifier_field :email
   # or
-  identifier -> (user) { "user:#{user.email}" }
+  identifier_field -> (user) { "user:#{user.email}" }
   # or
-  identifier [:type, :email]
+  identifier_field [:type, :email]
 
   field :email
   field :type
 end
 ```
 
-### 3. Redis Data Types
+### 3. Data Types
 
-Familia supports various Redis data types:
+Familia supports various Valkey-compatible data types:
 
 ```ruby
 class Product < Familia::Horreum
@@ -63,9 +63,9 @@ class Product < Familia::Horreum
 end
 ```
 
-### 4. Class-level Redis Types
+### 4. Class-level Valkey-compatible Types
 
-You can also define Redis types at the class level:
+You can also define Valkey-compatible types at the class level:
 
 ```ruby
 class Customer < Familia::Horreum
@@ -78,12 +78,12 @@ end
 
 ### 5. Automatic Expiration
 
-Use the expiration feature to set TTL for objects:
+Use the expiration feature to set default TTL for objects:
 
 ```ruby
 class Session < Familia::Horreum
   feature :expiration
-  ttl 180.minutes
+  default_expiration 180.minutes
 end
 ```
 
@@ -110,7 +110,7 @@ Use quantization for time-based metrics:
 ```ruby
 class DailyMetric < Familia::Horreum
   feature :quantization
-  string :counter, ttl: 1.day, quantize: [10.minutes, '%H:%M']
+  string :counter, default_expiration: 1.day, quantize: [10.minutes, '%H:%M']
 end
 ```
 
@@ -140,13 +140,6 @@ class Customer < Familia::Horreum
     verified && !reset_requested
   end
 end
-
-class Session < Familia::Horreum
-  def external_identifier
-    elements = [ipaddress || 'UNKNOWNIP', custid || 'anon']
-    @external_identifier ||= Familia.generate_sha_hash(elements)
-  end
-end
 ```
 ### 10. Open-ended Serialization
 
@@ -171,6 +164,29 @@ user.transaction do |conn|
 end
 ```
 
+### 12. Connection Management and Pooling
+
+Familia supports custom connection providers for advanced scenarios like connection pooling:
+
+```ruby
+# Using connection_pool gem for thread-safe pooling
+require 'connection_pool'
+
+# Create pools for each logical database
+pools = {
+  "redis://localhost:6379/0" => ConnectionPool.new(size: 10) { Redis.new(db: 0) },
+  "redis://localhost:6379/1" => ConnectionPool.new(size: 5) { Redis.new(db: 1) }
+}
+
+# Configure Familia to use the pools
+Familia.connection_provider = lambda do |uri|
+  pool = pools[uri] || pools["redis://localhost:6379/0"]
+  pool.with { |conn| conn }
+end
+```
+
+See the [Connection Pooling Guide](docs/connection_pooling.md) for detailed examples.
+
 
 ## Usage Examples
 
@@ -225,7 +241,7 @@ end
 
 ## Conclusion
 
-Familia provides a powerful and flexible way to work with Redis in Ruby applications. Its features like automatic expiration, safe dumping, and quantization make it suitable for a wide range of use cases, from simple key-value storage to complex time-series data management.
+Familia provides a powerful and flexible way to work with Valkey-compatible in Ruby applications. Its features like automatic expiration, safe dumping, and quantization make it suitable for a wide range of use cases, from simple key-value storage to complex time-series data management.
 
 For more information, visit:
 - [Github Repository](https://github.com/delano/familia)

data/bin/irb

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+irb -ruri/redis -Ilib -r familia -Itry -r helpers/test_helpers

data/docs/connection_pooling.md

@@ -0,0 +1,317 @@
+# Connection Pooling with Familia
+
+Familia uses a flexible connection provider pattern that allows you to implement connection pooling in your application. This guide shows how to configure connection pools for optimal performance with multiple logical databases.
+
+## Key Concepts
+
+1. **Connection Provider Contract**: When you provide a `connection_provider`, it MUST return connections already on the correct logical database. Familia will NOT issue SELECT commands after receiving a connection from the provider.
+
+2. **URI-based Selection**: Familia passes normalized URIs (e.g., `redis://localhost:6379/2`) to your provider, encoding the logical database in the URI.
+
+3. **One Pool Per Database**: Since Familia models can use different logical databases, you typically need one connection pool per unique database.
+
+## Basic Connection Pool Setup
+
+### Example 1: Simple Connection Pool
+
+```ruby
+require 'connection_pool'
+require 'familia'
+
+class MyApp
+  def self.setup_familia_pools
+    @pools = {}
+    
+    Familia.connection_provider = lambda do |uri|
+      parsed = URI.parse(uri)
+      pool_key = "#{parsed.host}:#{parsed.port}/#{parsed.db || 0}"
+      
+      # Create a pool for each unique database
+      @pools[pool_key] ||= ConnectionPool.new(size: 10, timeout: 5) do
+        Redis.new(
+          host: parsed.host,
+          port: parsed.port,
+          db: parsed.db || 0  # Connection created with correct DB
+        )
+      end
+      
+      # Return a connection from the pool
+      @pools[pool_key].with { |conn| conn }
+    end
+  end
+end
+```
+
+### Example 2: Advanced Pooling with Different Configurations
+
+```ruby
+class MyApp
+  # Different pool sizes based on expected traffic
+  POOL_CONFIGS = {
+    0 => { size: 20, timeout: 5 },  # High-traffic main database
+    1 => { size: 5, timeout: 5 },   # Low-traffic analytics database
+    2 => { size: 10, timeout: 5 },  # Medium-traffic cache database
+    3 => { size: 5, timeout: 5 }    # Session database
+  }
+  
+  def self.setup_familia_pools
+    @pools = {}
+    
+    Familia.connection_provider = lambda do |uri|
+      parsed = URI.parse(uri)
+      db = parsed.db || 0
+      pool_key = "#{parsed.host}:#{parsed.port}/#{db}"
+      
+      @pools[pool_key] ||= begin
+        config = POOL_CONFIGS[db] || { size: 5, timeout: 5 }
+        ConnectionPool.new(**config) do
+          Redis.new(host: parsed.host, port: parsed.port, db: db)
+        end
+      end
+      
+      @pools[pool_key].with { |conn| conn }
+    end
+  end
+end
+```
+
+### Example 3: Using with Puma/Multi-threaded Servers
+
+```ruby
+# In config/initializers/familia.rb
+
+# Global connection pools shared across threads
+$redis_pools = {}
+$pool_mutex = Mutex.new
+
+Familia.connection_provider = lambda do |uri|
+  parsed = URI.parse(uri)
+  pool_key = parsed.to_s
+  
+  # Thread-safe pool creation
+  $pool_mutex.synchronize do
+    $redis_pools[pool_key] ||= ConnectionPool.new(
+      size: ENV.fetch('RAILS_MAX_THREADS', 5).to_i,
+      timeout: 5
+    ) do
+      Redis.new(
+        host: parsed.host,
+        port: parsed.port,
+        db: parsed.db || 0,
+        timeout: 1,  # Connection timeout
+        reconnect_attempts: 3
+      )
+    end
+  end
+  
+  $redis_pools[pool_key].with { |conn| conn }
+end
+```
+
+### Example 4: Using with Sidekiq
+
+```ruby
+# Sidekiq has its own connection pool management
+# This example shows how to share pools between Sidekiq and web processes
+
+class RedisConnectionPools
+  include Singleton
+  
+  def initialize
+    @pools = {}
+    @mutex = Mutex.new
+  end
+  
+  def get_connection(uri)
+    parsed = URI.parse(uri)
+    pool_key = "#{parsed.host}:#{parsed.port}/#{parsed.db || 0}"
+    
+    pool = @mutex.synchronize do
+      @pools[pool_key] ||= ConnectionPool.new(
+        size: determine_pool_size,
+        timeout: 5
+      ) do
+        Redis.new(
+          host: parsed.host,
+          port: parsed.port,
+          db: parsed.db || 0
+        )
+      end
+    end
+    
+    pool.with { |conn| conn }
+  end
+  
+  private
+  
+  def determine_pool_size
+    if defined?(Sidekiq)
+      # Sidekiq workers need more connections
+      Sidekiq.options[:concurrency] + 2
+    else
+      # Web processes need fewer connections
+      ENV.fetch('RAILS_MAX_THREADS', 5).to_i
+    end
+  end
+end
+
+# Configure Familia
+Familia.connection_provider = lambda do |uri|
+  RedisConnectionPools.instance.get_connection(uri)
+end
+```
+
+## Model Configuration
+
+Models can specify different logical databases:
+
+```ruby
+# Models using different logical databases
+class Customer < Familia::Horreum
+  self.logical_database = 0  # Main application data
+  field :name
+  field :email
+end
+
+class Analytics < Familia::Horreum
+  self.logical_database = 1  # Analytics data  
+  field :event_type
+  field :timestamp
+end
+
+class Session < Familia::Horreum
+  self.logical_database = 2  # Session/cache data
+  feature :expiration
+  default_expiration 1.hour
+  field :user_id
+  field :data
+end
+
+# Models can share the same logical database
+class Order < Familia::Horreum
+  self.logical_database = 0  # Shares DB with Customer
+  field :customer_id
+  field :total
+end
+```
+
+## Performance Optimization
+
+### Avoiding SELECT Command Overhead
+
+Without proper pooling configuration, each Redis operation might issue a SELECT command:
+
+```
+# Bad: Without connection provider
+SET key value     # Connection on DB 0
+SELECT 2          # Switch to DB 2
+SET key2 value2   # Now on DB 2
+SELECT 0          # Switch back
+GET key           # Now on DB 0
+```
+
+With the connection provider pattern:
+
+```
+# Good: With connection provider  
+SET key value     # Connection already on correct DB
+SET key2 value2   # Different connection, already on correct DB
+GET key           # Original connection, still on correct DB
+```
+
+### Pool Sizing Guidelines
+
+1. **Web Applications**: `pool_size = number_of_threads + buffer`
+   ```ruby
+   size: ENV.fetch('RAILS_MAX_THREADS', 5).to_i + 2
+   ```
+
+2. **Background Jobs**: `pool_size = concurrency + buffer`
+   ```ruby
+   size: Sidekiq.options[:concurrency] + 5
+   ```
+
+3. **Mixed Workloads**: Size based on the logical database's usage pattern
+   ```ruby
+   POOL_CONFIGS = {
+     0 => { size: 20 },  # High-traffic main DB
+     1 => { size: 5 },   # Low-traffic analytics
+     2 => { size: 15 }   # Medium-traffic cache
+   }
+   ```
+
+## Testing Your Configuration
+
+```ruby
+# Test helper to verify pools are working correctly
+class ConnectionPoolTester
+  def self.test_pools
+    # Create test models using different databases
+    class TestModel0 < Familia::Horreum
+      self.logical_database = 0
+      field :value
+    end
+    
+    class TestModel1 < Familia::Horreum  
+      self.logical_database = 1
+      field :value
+    end
+    
+    # Test concurrent access
+    threads = 10.times.map do |i|
+      Thread.new do
+        100.times do |j|
+          # This should use the pool for DB 0
+          TestModel0.create(value: "thread-#{i}-#{j}")
+          
+          # This should use the pool for DB 1
+          TestModel1.create(value: "thread-#{i}-#{j}")
+        end
+      end
+    end
+    
+    threads.each(&:join)
+    puts "Pool test completed successfully"
+  end
+end
+```
+
+## Monitoring and Debugging
+
+Enable debug mode to verify connection providers are returning connections on the correct database:
+
+```ruby
+Familia.debug = true  # Logs warnings if provider returns wrong DB
+```
+
+Monitor pool usage:
+
+```ruby
+# Add monitoring to your connection provider
+Familia.connection_provider = lambda do |uri|
+  pool = get_pool_for(uri)
+  
+  # Log pool statistics
+  Rails.logger.info "Pool stats: #{pool.size} size, #{pool.available} available"
+  
+  pool.with { |conn| conn }
+end
+```
+
+## Common Pitfalls
+
+1. **Not returning DB-ready connections**: Your provider MUST return connections already on the correct database.
+
+2. **Creating too many pools**: Ensure you're keying pools correctly to avoid creating duplicate pools for the same database.
+
+3. **Forgetting thread safety**: Pool creation should be thread-safe in multi-threaded environments.
+
+4. **Incorrect pool sizing**: Monitor your connection usage and adjust pool sizes accordingly.
+
+## Summary
+
+- Familia delegates connection management to your application via `connection_provider`
+- Providers must return connections already on the correct logical database
+- Use the `connection_pool` gem for robust pooling
+- Create one pool per unique logical database
+- Monitor and tune pool sizes based on your workload

data/familia.gemspec

@@ -1,12 +1,12 @@
-# frozen_string_literal: true
+# lib/familia/settings.rb
 
 require_relative 'lib/familia/version'
 
 Gem::Specification.new do |spec|
   spec.name        = 'familia'
-  spec.version     = Familia::VERSION.to_s
-  spec.summary     = 'An ORM for Redis in Ruby.'
-  spec.description = "Familia: #{spec.summary}. Organize and store ruby objects in Redis"
+  spec.version     = Familia::VERSION
+  spec.summary     = 'An ORM for Valkey-compatible databases in Ruby.'
+  spec.description = "Familia: #{spec.summary}. Organize and store ruby objects in Valkey/Redis"
   spec.authors     = ['Delano Mandelbaum']
   spec.email       = 'gems@solutious.com'
   spec.homepage    = 'https://github.com/delano/familia'
@@ -17,8 +17,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.8')
+  spec.required_ruby_version = Gem::Requirement.new('>= 3.4')
 
+  spec.add_dependency 'benchmark'
+  spec.add_dependency 'connection_pool'
+  spec.add_dependency 'csv'
+  spec.add_dependency 'logger'
   spec.add_dependency 'redis', '>= 4.8.1', '< 6.0'
   spec.add_dependency 'stringio', '~> 3.1.1'
   spec.add_dependency 'uri-redis', '~> 1.3'