familia 2.0.0.pre18 → 2.0.0.pre21

data/try/integration/transaction_safety_core_try.rb

@@ -0,0 +1,176 @@
+# try/integration/transaction_safety_core_try.rb
+#
+# frozen_string_literal: true
+
+# Core Transaction Safety Tests
+#
+# Tests the fundamental transaction safety rules from docs/transaction_safety.md
+# Focuses on the most critical safety mechanisms.
+#
+
+require_relative '../support/helpers/test_helpers'
+
+# Simple test model for transaction safety
+class ::SafetyTestCustomer < Familia::Horreum
+  identifier_field :email
+  field :email
+  field :login_count
+  field :status
+
+  list :orders
+end
+
+# Helper for unique test IDs
+def safety_test_id(prefix = 'test')
+  "#{prefix}_#{Time.now.to_i}_#{rand(1000000)}"
+end
+
+## save raises OperationModeError inside transaction
+error_raised = false
+SafetyTestCustomer.transaction do
+  customer = SafetyTestCustomer.new(email: "#{safety_test_id}@example.com")
+  begin
+    customer.save
+  rescue Familia::OperationModeError => e
+    error_raised = true
+  end
+end
+error_raised
+#=> true
+
+## save_if_not_exists! raises OperationModeError inside transaction
+error_raised_conditional = false
+SafetyTestCustomer.transaction do
+  customer = SafetyTestCustomer.new(email: "#{safety_test_id}@example.com")
+  begin
+    customer.save_if_not_exists!
+  rescue Familia::OperationModeError => e
+    error_raised_conditional = true
+  end
+end
+error_raised_conditional
+#=> true
+
+## create! raises OperationModeError inside transaction
+error_raised_create = false
+SafetyTestCustomer.transaction do
+  begin
+    SafetyTestCustomer.create!(email: "#{safety_test_id}@example.com")
+  rescue Familia::OperationModeError => e
+    error_raised_create = true
+  end
+end
+error_raised_create
+#=> true
+
+## correct pattern save before transaction works
+@correct_customer = SafetyTestCustomer.new(email: "#{safety_test_id('correct')}@example.com")
+@correct_customer.save
+#=> true
+
+## operations work inside transaction after save
+@correct_customer.transaction do
+  @correct_customer.hset(:login_count, '1')
+  @correct_customer.hset(:status, 'active')
+end
+@correct_customer.hget(:login_count).to_i >= 1
+#=> true
+
+## write-only operations work inside transactions
+@write_customer = SafetyTestCustomer.new(email: "#{safety_test_id('write')}@example.com")
+@write_customer.save
+
+@write_result = @write_customer.transaction do |conn|
+  conn.hset(@write_customer.dbkey, 'status', 'premium')
+  conn.hset(@write_customer.dbkey, 'login_count', '5')
+  conn.expire(@write_customer.dbkey, 3600)
+end
+@write_result.class.name
+#=> "MultiResult"
+
+## nested transactions reuse same connection
+@nested_customer = SafetyTestCustomer.new(email: "#{safety_test_id('nested')}@example.com")
+@nested_customer.save
+
+@outer_conn_id = nil
+@inner_conn_id = nil
+
+SafetyTestCustomer.transaction do |outer_conn|
+  @outer_conn_id = outer_conn.object_id
+  @nested_customer.hset(:login_count, '1')
+
+  @nested_customer.transaction do |inner_conn|
+    @inner_conn_id = inner_conn.object_id
+    @nested_customer.hset(:status, 'nested')
+  end
+end
+
+@outer_conn_id == @inner_conn_id
+#=> true
+
+## read operations return Future objects inside transaction
+@read_customer = SafetyTestCustomer.new(email: "#{safety_test_id('read')}@example.com")
+@read_customer.save
+
+@future_object = nil
+@read_customer.transaction do |conn|
+  @future_object = conn.hget(@read_customer.dbkey, 'email')
+end
+@future_object.class.name.include?('Future')
+#=> true
+
+## exists check returns Future inside transaction always truthy
+@pitfall_customer = SafetyTestCustomer.new(email: "#{safety_test_id('pitfall')}@example.com")
+
+@wrong_result = nil
+SafetyTestCustomer.transaction do |conn|
+  existence = conn.exists?(@pitfall_customer.dbkey)
+  @wrong_result = if existence
+    'always_executed'
+  else
+    'never_executed'
+  end
+end
+@wrong_result
+#=> 'always_executed'
+
+## create with success callback works
+@callback_executed = false
+SafetyTestCustomer.create!(email: "#{safety_test_id('callback')}@example.com") do |customer|
+  @callback_executed = true
+  customer.hset(:login_count, '1')
+end
+@callback_executed
+#=> true
+
+## multi-object atomic updates work
+@order_customer = SafetyTestCustomer.new(email: "#{safety_test_id('order')}@example.com")
+@order_customer.save
+
+@multi_result = SafetyTestCustomer.transaction do
+  @order_customer.hset(:status, 'confirmed')
+  @order_customer.orders.push('order123')
+  @order_customer.hset(:login_count, '1')
+end
+@multi_result.successful?
+#=> true
+
+## watch pattern for optimistic locking works
+@watch_customer = SafetyTestCustomer.new(email: "#{safety_test_id('watch')}@example.com")
+@watch_customer.save
+@watch_customer.hset(:balance, '1000')
+
+@success = @watch_customer.watch do
+  current_balance = @watch_customer.hget(:balance).to_i
+
+  if current_balance >= 100
+    @watch_customer.transaction do
+      @watch_customer.hset(:balance, (current_balance - 100).to_s)
+      @watch_customer.hset(:purchases, '1')
+    end
+  end
+end
+
+@new_balance = @watch_customer.hget(:balance).to_i
+@new_balance <= 900
+#=> true

data/try/integration/transaction_safety_workflow_try.rb

@@ -0,0 +1,291 @@
+# try/integration/transaction_safety_workflow_try.rb
+#
+# frozen_string_literal: true
+
+# Transaction Safety Workflow Integration Test
+#
+# Demonstrates the complete transaction safety workflow with realistic
+# business scenarios that show correct usage patterns.
+#
+
+require_relative '../support/helpers/test_helpers'
+
+# Business models for realistic workflow testing
+class ::WorkflowCustomer < Familia::Horreum
+  identifier_field :customer_id
+  field :customer_id
+  field :email
+  field :balance
+  field :status
+  field :login_count
+  field :last_login
+
+  list :orders
+  set :preferences
+end
+
+class ::WorkflowOrder < Familia::Horreum
+  identifier_field :order_id
+  field :order_id
+  field :customer_id
+  field :amount
+  field :status
+  field :created_at
+end
+
+class ::WorkflowInventory < Familia::Horreum
+  identifier_field :product_id
+  field :product_id
+  field :quantity
+  field :reserved
+end
+
+# Helper for unique IDs
+def workflow_id(prefix = 'wf')
+  "#{prefix}_#{Time.now.to_i}_#{rand(100000)}"
+end
+
+## Complete customer registration workflow
+@customer_email = "#{workflow_id('customer')}@example.com"
+@customer_id = workflow_id('cust')
+
+# Step 1: Create customer (validates uniqueness outside transaction)
+@customer = WorkflowCustomer.new(
+  customer_id: @customer_id,
+  email: @customer_email,
+  balance: 1000,
+  status: 'pending',
+  login_count: 0
+)
+@customer.save
+#=> true
+
+## Customer exists after registration
+@customer.exists?
+#=> true
+
+## E-commerce order processing workflow
+@product_id = workflow_id('prod')
+@order_id = workflow_id('order')
+
+# Setup inventory
+@inventory = WorkflowInventory.new(
+  product_id: @product_id,
+  quantity: 50,
+  reserved: 0
+)
+@inventory.save
+
+# Step 1: Create order (outside transaction for validation)
+@order = WorkflowOrder.new(
+  order_id: @order_id,
+  customer_id: @customer_id,
+  amount: 99,
+  status: 'pending',
+  created_at: Time.now.to_i
+)
+@order.save
+#=> true
+
+## Atomic order processing with inventory update
+@processing_result = WorkflowCustomer.transaction do |conn|
+  # Update customer
+  conn.hset(@customer.dbkey, 'balance', '901')  # 1000 - 99
+  conn.hset(@customer.dbkey, 'last_login', Time.now.to_i.to_s)
+  @customer.orders.push(@order_id)
+
+  # Update order
+  conn.hset(@order.dbkey, 'status', 'confirmed')
+
+  # Update inventory
+  conn.hset(@inventory.dbkey, 'quantity', '49')  # 50 - 1
+  conn.hset(@inventory.dbkey, 'reserved', '1')
+
+  # Add customer preference
+  @customer.preferences.add('email_notifications')
+end
+@processing_result.class.name
+#=> "MultiResult"
+
+## All updates applied atomically
+[@customer.hget('balance').to_i, @order.hget('status'), @inventory.hget('quantity').to_i]
+#=> [901, "confirmed", 49]
+
+## Customer login tracking workflow with nested transactions
+@login_start = Time.now
+
+# Read current count outside transaction
+@current_count = @customer.hget('login_count').to_i
+
+@customer.transaction do |outer_conn|
+  # Outer transaction: main login processing
+  outer_conn.hset(@customer.dbkey, 'last_login', @login_start.to_i.to_s)
+
+  # Nested transaction: increment login count (reentrant)
+  @customer.transaction do |inner_conn|
+    inner_conn.hset(@customer.dbkey, 'login_count', (@current_count + 1).to_s)
+
+    # Add login preference tracking
+    @customer.preferences.add('frequent_user') if @current_count > 5
+  end
+
+  # Continue outer transaction
+  outer_conn.hset(@customer.dbkey, 'status', 'active')
+end
+
+@customer.hget('login_count').to_i >= 1
+#=> true
+
+## Bulk order fulfillment workflow
+@order_ids = 3.times.map { |i| workflow_id("bulk_#{i}") }
+
+# Step 1: Create all orders outside transaction
+@bulk_orders = @order_ids.map do |order_id|
+  order = WorkflowOrder.new(
+    order_id: order_id,
+    customer_id: @customer_id,
+    amount: 25,
+    status: 'pending',
+    created_at: Time.now.to_i
+  )
+  order.save
+  order
+end
+@bulk_orders.all?(&:exists?)
+#=> true
+
+## Bulk fulfillment in single transaction
+# Read balance outside transaction
+@current_balance = @customer.hget('balance').to_i
+@bulk_result = WorkflowOrder.transaction do |conn|
+  @bulk_orders.each do |order|
+    conn.hset(order.dbkey, 'status', 'fulfilled')
+    conn.hset(order.dbkey, 'fulfilled_at', Time.now.to_i.to_s)
+    @customer.orders.push(order.order_id)
+  end
+
+  # Update customer balance for all orders
+  new_balance = @current_balance - (25 * 3)
+  conn.hset(@customer.dbkey, 'balance', new_balance.to_s)
+end
+@bulk_result.class.name
+#=> "MultiResult"
+
+## Error handling in transaction workflow
+@error_order_id = workflow_id('error')
+@error_handled = false
+
+begin
+  WorkflowOrder.transaction do |conn|
+    # Valid operation
+    conn.hset("test:#{@error_order_id}", 'status', 'processing')
+
+    # Simulate error during processing
+    raise StandardError, 'Payment processing failed'
+
+    # This would not execute due to error
+    conn.hset("test:#{@error_order_id}", 'status', 'completed')
+  end
+rescue StandardError => e
+  @error_handled = e.message.include?('Payment processing failed')
+end
+
+@error_handled
+#=> true
+
+## Transaction safety violation detection
+@safety_violation_detected = false
+
+WorkflowCustomer.transaction do
+  test_customer = WorkflowCustomer.new(
+    customer_id: workflow_id('safety'),
+    email: "#{workflow_id('safety')}@test.com"
+  )
+
+  begin
+    # This should raise OperationModeError
+    test_customer.save
+  rescue Familia::OperationModeError => e
+    @safety_violation_detected = e.message.include?('Cannot call save within a transaction')
+  end
+end
+
+@safety_violation_detected
+#=> true
+
+## Performance comparison: individual vs batch operations
+@perf_customers = 5.times.map do |i|
+  customer = WorkflowCustomer.new(
+    customer_id: workflow_id("perf_#{i}"),
+    email: "perf#{i}@example.com",
+    balance: 1000
+  )
+  customer.save
+  customer
+end
+
+# Individual transactions
+@individual_start = Time.now
+@perf_customers.each do |customer|
+  customer.transaction do |conn|
+    conn.hset(customer.dbkey, 'status', 'updated_individual')
+  end
+end
+@individual_duration = ((Time.now - @individual_start) * 1000).round(2)
+
+# Single batch transaction
+@batch_start = Time.now
+WorkflowCustomer.transaction do |conn|
+  @perf_customers.each do |customer|
+    conn.hset(customer.dbkey, 'status', 'updated_batch')
+  end
+end
+@batch_duration = ((Time.now - @batch_start) * 1000).round(2)
+
+# Batch should be faster or at least not significantly slower
+@efficiency_ratio = @individual_duration / @batch_duration
+@efficiency_ratio >= 0.1  # Batch should be reasonably fast
+#=> true
+
+## Watch pattern for optimistic concurrency control
+@concurrent_customer = WorkflowCustomer.new(
+  customer_id: workflow_id('concurrent'),
+  email: 'concurrent@test.com',
+  balance: 500
+)
+@concurrent_customer.save
+@concurrent_customer.hset(:version, '1')
+
+@watch_success = @concurrent_customer.watch do
+  current_version = @concurrent_customer.hget(:version).to_i
+  current_balance = @concurrent_customer.hget(:balance).to_i
+
+  # Only proceed if version hasn't changed and sufficient balance
+  if current_version == 1 && current_balance >= 100
+    @concurrent_customer.transaction do |conn|
+      conn.hset(@concurrent_customer.dbkey, 'balance', (current_balance - 100).to_s)
+      conn.hset(@concurrent_customer.dbkey, 'version', '2')
+      conn.hset(@concurrent_customer.dbkey, 'last_transaction', Time.now.to_i.to_s)
+    end
+    true
+  else
+    false
+  end
+end
+
+@concurrent_customer.hget(:balance).to_i == 400
+#=> true
+
+## Workflow completed successfully with all safety checks
+@workflow_summary = {
+  customer_created: @customer.exists?,
+  order_processed: @order.hget('status') == 'confirmed',
+  bulk_fulfilled: @bulk_orders.all? { |o| o.hget('status') == 'fulfilled' },
+  error_handled: @error_handled,
+  safety_enforced: @safety_violation_detected,
+  performance_acceptable: @efficiency_ratio >= 0.1,
+  concurrency_controlled: @concurrent_customer.hget(:version).to_i >= 2
+}
+
+@workflow_summary.values.all?
+#=> true

data/try/integration/verifiable_identifier_try.rb

@@ -1,3 +1,7 @@
+# try/integration/verifiable_identifier_try.rb
+#
+# frozen_string_literal: true
+
 # try/core/verifiable_identifier_try.rb
 
 require_relative '../support/helpers/test_helpers'

data/try/investigation/pipeline_routing/README.md

@@ -0,0 +1,228 @@
+# Pipeline Routing Investigation (ARCHIVED)
+
+**Status**: Investigation complete - NO BUG FOUND
+**Conclusion**: See `CONCLUSION.md` for full analysis
+**Test Files**: Renamed to `.rb.txt` to preserve as documentation without running in CI
+
+## Problem Statement (Original)
+
+In a concurrent test with 20 threads executing pipelined operations, we observed:
+- All 20 threads successfully complete
+- All 20 commands are captured in `DatabaseLogger.commands`
+- BUT: Only 16 commands contain the pipeline separator `' | '`
+- This appeared to mean 4 pipeline operations were logged via `call()` instead of `call_pipelined()`
+
+**Investigation Result**: Single-command pipelines don't have `' | '` separator (expected `Array#join` behavior). This is NOT a bug.
+
+## Investigation Goals
+
+1. Determine if this is a RedisClient middleware dispatch issue
+2. Determine if this is a Familia connection chain issue
+3. Determine if this is timing-dependent (race condition)
+4. Determine if connection reuse vs. fresh connections affects behavior
+
+## Test Suite Structure
+
+### 01_single_thread_baseline_try.rb
+**Purpose**: Establish baseline behavior in single-threaded environment
+
+**Tests**:
+- 10 simple pipeline operations
+- 25 pipeline operations with varying sizes (1-5 commands)
+- Mixed single and pipelined operations (10 each)
+
+**Expected**: 100% of pipeline operations route to `call_pipelined()`
+
+**Key Question**: Does single-threaded execution work correctly?
+
+### 02_small_concurrency_try.rb
+**Purpose**: Test with minimal thread contention (5 threads)
+
+**Tests**:
+- 5 threads with CyclicBarrier (synchronized start)
+- 5 threads without barrier (natural timing)
+- 5 threads with varying pipeline sizes (1-5 commands)
+
+**Expected**: 100% of pipeline operations route to `call_pipelined()`
+
+**Key Question**: Does adding concurrency break routing?
+
+### 03_reproduce_issue_try.rb
+**Purpose**: Reproduce the exact scenario from the original failing test
+
+**Tests**:
+- 20 threads with CyclicBarrier (exact reproduction)
+- 10 repeated trials to check for intermittent behavior
+
+**Expected**: May reproduce the routing issue
+
+**Key Questions**:
+- Is the issue reproducible?
+- Is it deterministic or intermittent?
+- What's the failure rate?
+
+### 04_high_contention_try.rb
+**Purpose**: Test under high thread contention (50+ threads)
+
+**Tests**:
+- 50 threads with synchronized start
+- 50 threads with varying pipeline sizes (1-10 commands)
+- 10 threads × 5 rapid pipelines each (50 total)
+
+**Expected**: If timing-related, higher contention should increase failure rate
+
+**Key Question**: Does the problem scale with thread count?
+
+### 05_connection_isolation_try.rb
+**Purpose**: Test whether the issue is connection-specific
+
+**Tests**:
+- Fresh connection per thread (each thread calls `Familia.dbclient`)
+- Shared connection from main thread (original pattern)
+- Isolated connections via `create_dbclient` (no pooling)
+- Connection via chain per thread (uses connection handlers)
+
+**Expected**: Identifies which connection pattern is problematic
+
+**Key Questions**:
+- Does connection reuse cause the issue?
+- Does the connection chain have a bug?
+- Is middleware registration timing-dependent?
+
+### 06_fiber_state_inspection_try.rb
+**Purpose**: Inspect Fiber-local state during pipeline operations
+
+**Tests**:
+- Single-threaded Fiber state tracking (before/inside/after pipeline)
+- Multi-threaded Fiber isolation verification (10 threads)
+- Middleware call context inspection (capture what middleware sees)
+- Pipeline routing verification (ensure single `call_pipelined` per pipeline)
+
+**Expected**: Fiber-local state should be isolated per thread
+
+**Key Questions**:
+- Is `Fiber[:familia_pipeline]` being set correctly?
+- Is cleanup happening in ensure blocks?
+- Do threads share Fiber state incorrectly?
+- Does middleware receive correct context?
+
+## Running the Tests
+
+### Run all investigation tests
+```bash
+FAMILIA_DEBUG=0 bundle exec try --agent try/investigation/pipeline_routing/
+```
+
+### Run individual tests
+```bash
+# Baseline
+bundle exec try --agent try/investigation/pipeline_routing/01_single_thread_baseline_try.rb
+
+# Small concurrency
+bundle exec try --agent try/investigation/pipeline_routing/02_small_concurrency_try.rb
+
+# Reproduce issue
+bundle exec try --agent try/investigation/pipeline_routing/03_reproduce_issue_try.rb
+
+# High contention
+bundle exec try --agent try/investigation/pipeline_routing/04_high_contention_try.rb
+
+# Connection isolation
+bundle exec try --agent try/investigation/pipeline_routing/05_connection_isolation_try.rb
+
+# Fiber state
+bundle exec try --agent try/investigation/pipeline_routing/06_fiber_state_inspection_try.rb
+```
+
+### Run with verbose output (for failures)
+```bash
+bundle exec try --verbose --fails --stack try/investigation/pipeline_routing/03_reproduce_issue_try.rb
+```
+
+## What to Look For
+
+### Success Indicators
+- All pipeline operations contain `' | '` separator
+- Command counts match expected values
+- No "ROUTING ANOMALY" messages in output
+
+### Failure Indicators
+- Pipeline operations logged without `' | '` separator
+- Fewer `call_pipelined()` invocations than expected
+- "ROUTING ANOMALY DETECTED" in output
+- Mismatched command counts
+
+### Diagnostic Output
+Each test prints detailed analysis including:
+- Total commands captured
+- Pipeline commands (with separator)
+- Single commands (without separator)
+- Percentage breakdown
+- Specific examples of misrouted commands
+
+## Hypothesis Checklist
+
+After running the tests, we should be able to answer:
+
+- [ ] Does single-threaded execution work correctly?
+- [ ] Does small concurrency (5 threads) work correctly?
+- [ ] Can we reproduce the issue with 20 threads?
+- [ ] Is the issue deterministic or intermittent?
+- [ ] Does the problem scale with thread count?
+- [ ] Is it related to connection reuse vs. fresh connections?
+- [ ] Is it related to the connection chain implementation?
+- [ ] Is Fiber-local state being managed correctly?
+- [ ] Does middleware receive the correct context?
+
+## Next Steps
+
+Based on results:
+
+1. **If baseline fails**: RedisClient middleware routing is broken in general
+2. **If only concurrent tests fail**: Thread safety issue in middleware dispatch
+3. **If only shared connection fails**: Connection chain or pooling issue
+4. **If Fiber state leaks**: Cleanup logic in `PipelineCore` is broken
+5. **If intermittent**: Race condition requiring deeper investigation
+
+## Background: How Pipeline Routing Should Work
+
+```ruby
+# RedisClient source (redis-client-0.25.1/lib/redis_client.rb:446)
+def pipelined(exception: true)
+  pipeline = Pipeline.new(@command_builder)
+  yield pipeline
+
+  if pipeline._size == 0
+    []
+  else
+    results = ensure_connected(retryable: pipeline._retryable?) do |connection|
+      commands = pipeline._commands
+      @middlewares.call_pipelined(commands, config) do  # <-- Should call this
+        connection.call_pipelined(commands, pipeline._timeouts, exception: exception)
+      end
+    end
+
+    pipeline._coerce!(results)
+  end
+end
+```
+
+**Expected flow**:
+1. User calls `client.pipelined { |p| p.set(...) }`
+2. RedisClient builds pipeline commands
+3. RedisClient calls `@middlewares.call_pipelined(commands, config)`
+4. DatabaseLogger.call_pipelined receives commands array
+5. Logs with `' | '` separator joining commands
+
+**Anomalous flow** (what we're seeing):
+1. User calls `client.pipelined { |p| p.set(...) }`
+2. ??? Something goes wrong ???
+3. DatabaseLogger.call receives individual command
+4. Logs without `' | '` separator
+
+## Files Involved
+
+- `/Users/d/Projects/opensource/d/familia/lib/middleware/database_logger.rb` - Middleware implementation
+- `/Users/d/Projects/opensource/d/familia/lib/familia/connection.rb` - Connection management
+- `/Users/d/Projects/opensource/d/familia/lib/familia/connection/pipelined_core.rb` - Pipeline execution
+- `/Users/d/Projects/opensource/d/familia/try/unit/core/middleware_thread_safety_try.rb` - Original test that exposed this

data/try/performance/benchmarks_try.rb

@@ -1,3 +1,7 @@
+# try/performance/benchmarks_try.rb
+#
+# frozen_string_literal: true
+
 # Performance benchmarks separate from stress tests
 
 require_relative '../support/helpers/test_helpers'