async-redis 0.11.2 → 0.13.0

data/context/index.yaml

@@ -0,0 +1,36 @@
+# Automatically generated context index for Utopia::Project guides.
+# Do not edit then files in this directory directly, instead edit the guides and then run `bake utopia:project:agent:context:update`.
+---
+description: A Redis client library.
+metadata:
+  documentation_uri: https://socketry.github.io/async-redis/
+  source_code_uri: https://github.com/socketry/async-redis.git
+files:
+- path: getting-started.md
+  title: Getting Started
+  description: This guide explains how to use the `async-redis` gem to connect to
+    a Redis server and perform basic operations.
+- path: transactions-and-pipelines.md
+  title: Transactions and Pipelines
+  description: This guide explains how to use Redis transactions and pipelines with
+    `async-redis` for atomic operations and improved performance.
+- path: subscriptions.md
+  title: Subscriptions
+  description: This guide explains how to use Redis pub/sub functionality with `async-redis`
+    to publish and subscribe to messages.
+- path: data-structures.md
+  title: Data Structures and Operations
+  description: This guide explains how to work with Redis data types and operations
+    using `async-redis`.
+- path: streams.md
+  title: Streams
+  description: This guide explains how to use Redis streams with `async-redis` for
+    reliable message processing and event sourcing.
+- path: scripting.md
+  title: Scripting
+  description: This guide explains how to use Redis Lua scripting with `async-redis`
+    for atomic operations and advanced data processing.
+- path: client-architecture.md
+  title: Client Architecture
+  description: This guide explains the different client types available in `async-redis`
+    and when to use each one.

data/context/scripting.md

@@ -0,0 +1,243 @@
+# Scripting
+
+This guide explains how to use Redis Lua scripting with `async-redis` for atomic operations and advanced data processing.
+
+Lua scripting moves complex logic to the Redis server, ensuring atomicity and reducing network round trips. This is essential for operations that need to read, compute, and write data atomically.
+
+Critical for:
+- **Atomic business logic**: Complex operations that must be consistent.
+- **Performance optimization**: Reduce network calls for multi-step operations.
+- **Race condition prevention**: Ensure operations complete without interference.
+- **Custom data structures**: Implement specialized behaviors not available in standard Redis commands.
+
+## Basic Script Loading and Execution
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Simple Lua script:
+		increment_script = <<~LUA
+			local key = KEYS[1]
+			local increment = tonumber(ARGV[1])
+			local current = redis.call('GET', key) or 0
+			local new_value = tonumber(current) + increment
+			redis.call('SET', key, new_value)
+			return new_value
+		LUA
+		
+		# Load and execute script:
+		script_sha = client.script("LOAD", increment_script)
+		puts "Script loaded with SHA: #{script_sha}"
+		
+		# Execute script by SHA:
+		result = client.evalsha(script_sha, 1, "counter", 5)
+		puts "Counter incremented to: #{result}"
+		
+		# Execute script directly:
+		result = client.eval(increment_script, 1, "counter", 3)
+		puts "Counter incremented to: #{result}"
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Parameter Passing and Return Values
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Script with multiple parameters and complex return:
+		session_update_script = <<~LUA
+			local session_key = KEYS[1]
+			local user_id = ARGV[1]
+			local username = ARGV[2]
+			local increment_activity = ARGV[3] == 'true'
+			
+			-- Update session fields:
+			redis.call('HSET', session_key, 'user_id', user_id, 'username', username)
+			
+			-- Conditionally increment activity count:
+			local activity_count = 0
+			if increment_activity then
+				activity_count = redis.call('HINCRBY', session_key, 'activity_count', 1)
+			else
+				activity_count = tonumber(redis.call('HGET', session_key, 'activity_count') or 0)
+			end
+			
+			-- Update last activity timestamp:
+			redis.call('HSET', session_key, 'last_activity', redis.call('TIME')[1])
+			
+			-- Return session data:
+			return {
+				user_id,
+				username,
+				activity_count,
+				redis.call('HGET', session_key, 'last_activity')
+			}
+		LUA
+		
+		# Execute with parameters:
+		session_id = "session:" + SecureRandom.hex(16)
+		result = client.eval(session_update_script, 1, session_id, "12345", "alice", "true")
+		user_id, username, activity_count, last_activity = result
+		
+		puts "Updated session for #{username} (ID: #{user_id})"
+		puts "Activity count: #{activity_count}, Last activity: #{last_activity}"
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Script Caching Pattern
+
+Instead of complex script managers, use a simple caching pattern with fallback:
+
+``` ruby
+require "async/redis"
+
+class JobQueue
+	# Define scripts as class constants:
+	DEQUEUE_SCRIPT = <<~LUA
+		local queue_key = KEYS[1]
+		local processing_key = KEYS[2]
+		local current_time = ARGV[1]
+		
+		-- Get job from queue:
+		local job = redis.call('LPOP', queue_key)
+		if not job then
+			return nil
+		end
+		
+		-- Add to processing set with timestamp:
+		redis.call('ZADD', processing_key, current_time, job)
+		
+		return job
+	LUA
+	
+	COMPLETE_SCRIPT = <<~LUA
+		local processing_key = KEYS[1]
+		local job_data = ARGV[1]
+		
+		-- Remove job from processing set:
+		local removed = redis.call('ZREM', processing_key, job_data)
+		
+		return removed
+	LUA
+	
+	def initialize(client, queue_name)
+		@client = client
+		@queue_name = queue_name
+		@processing_name = "#{queue_name}:processing"
+		
+		# Load all scripts at initialization:
+		@dequeue_sha = @client.script("LOAD", DEQUEUE_SCRIPT)
+		@complete_sha = @client.script("LOAD", COMPLETE_SCRIPT)
+	end
+	
+	def dequeue_job
+		@client.evalsha(@dequeue_sha, 2, @queue_name, @processing_name, Time.now.to_i)
+	end
+	
+	def complete_job(job_data)
+		@client.evalsha(@complete_sha, 1, @processing_name, job_data)
+	end
+	
+	def enqueue_job(job_data)
+		@client.rpush(@queue_name, job_data)
+	end
+	
+	def cleanup_stale_jobs(timeout_seconds = 300)
+		cutoff_time = Time.now.to_i - timeout_seconds
+		stale_jobs = @client.zrangebyscore(@processing_name, "-inf", cutoff_time)
+		
+		if stale_jobs.any?
+			puts "Found #{stale_jobs.length} stale jobs, requeueing..."
+			
+			# Move stale jobs back to queue:
+			stale_jobs.each do |job|
+				@client.lpush(@queue_name, job)
+			end
+			
+			# Remove from processing set:
+			@client.zremrangebyscore(@processing_name, "-inf", cutoff_time)
+		end
+	end
+end
+
+# Usage:
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		queue = JobQueue.new(client, "work_queue")
+		
+		# Add some jobs:
+		5.times do |i|
+			queue.enqueue_job("job_#{i}")
+			puts "Enqueued job_#{i}"
+		end
+		
+		# Process jobs:
+		3.times do |i|
+			job = queue.dequeue_job
+			if job
+				puts "Processing: #{job}"
+				# Simulate work...
+				sleep 0.1
+				puts "Completed: #{job}"
+			else
+				puts "No jobs available"
+			end
+		end
+		
+		# Check for stale jobs:
+		queue.cleanup_stale_jobs(60)
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Best Practices
+
+### When to Use Scripts
+
+Use Lua scripts when you need:
+- **Atomic multi-step operations**: Multiple Redis commands that must succeed or fail together.
+- **Complex conditional logic**: Operations that depend on current Redis state.
+- **Performance optimization**: Reduce network round trips for complex operations.
+- **Race condition prevention**: Ensure operations complete without interference.
+
+### When Not to Use Scripts
+
+Avoid scripts for:
+- **Simple operations**: Single Redis commands don't need scripting.
+- **Long-running operations**: Scripts block the Redis server.
+- **Operations with external dependencies**: Scripts can't make network calls.
+- **Frequently changing logic**: Scripts are cached and harder to update.
+
+### Script Performance Tips
+
+- **Keep scripts short**: Long scripts block other operations.
+- **Use local variables**: Avoid repeated Redis calls for the same data.
+- **Cache scripts**: Use `EVALSHA` instead of `EVAL` for better performance.
+- **Handle script cache misses**: Implement fallback logic for `NOSCRIPT` errors.
+- **Validate inputs early**: Check parameters before performing operations.
+
+Scripts provide powerful atomic operations but should be used judiciously to maintain Redis performance and simplicity.

data/context/streams.md

@@ -0,0 +1,317 @@
+# Streams
+
+This guide explains how to use Redis streams with `async-redis` for reliable message processing and event sourcing.
+
+Streams are designed for high-throughput message processing and event sourcing. They provide durability, consumer groups for load balancing, and automatic message acknowledgment.
+
+Use streams when you need:
+- **Event sourcing**: Capture all changes to application state.
+- **Message queues**: Reliable message delivery with consumer groups.
+- **Audit logs**: Immutable record of system events.
+- **Real-time analytics**: Process streams of user events or metrics.
+
+## Stream Creation and Consumption
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Add entries to stream:
+		events = [
+			{ "type" => "user_signup", "user_id" => "123", "email" => "alice@example.com" },
+			{ "type" => "purchase", "user_id" => "123", "amount" => "29.99" },
+			{ "type" => "user_signup", "user_id" => "456", "email" => "bob@example.com" }
+		]
+		
+		events.each do |event|
+			entry_id = client.xadd("user_events", "*", event)
+			puts "Added event with ID: #{entry_id}"
+		end
+		
+		# Read from stream:
+		entries = client.xrange("user_events", "-", "+")
+		puts "Stream entries:"
+		entries.each do |entry_id, fields|
+			puts "  #{entry_id}: #{fields}"
+		end
+		
+		# Read latest entries:
+		latest = client.xrevrange("user_events", "+", "-", count: 2)
+		puts "Latest 2 entries: #{latest}"
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Reading New Messages
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Add some initial events:
+		client.xadd("notifications", "*", "type" => "welcome", "user_id" => "123")
+		client.xadd("notifications", "*", "type" => "reminder", "user_id" => "456")
+		
+		# Read only new messages (blocking):
+		puts "Waiting for new messages..."
+		
+		# This will block until new messages arrive:
+		messages = client.xread("BLOCK", 5000, "STREAMS", "notifications", "$")
+		
+		if messages && !messages.empty?
+			stream_name, entries = messages[0]
+			puts "Received #{entries.length} new messages:"
+			entries.each do |entry_id, fields|
+				puts "  #{entry_id}: #{fields}"
+			end
+		else
+			puts "No new messages received within timeout"
+		end
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Consumer Groups
+
+Consumer groups enable multiple workers to process messages in parallel while ensuring each message is processed exactly once:
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Create consumer group:
+		begin
+			client.xgroup("CREATE", "user_events", "processors", "0", "MKSTREAM")
+			puts "Created consumer group 'processors'"
+		rescue Protocol::Redis::ServerError => e
+			puts "Consumer group already exists: #{e.message}"
+		end
+		
+		# Add some test events:
+		3.times do |i|
+			client.xadd("user_events", "*", "event" => "test_#{i}", "timestamp" => Time.now.to_f)
+		end
+		
+		# Consume messages:
+		consumer_name = "worker_1"
+		messages = client.xreadgroup("GROUP", "processors", consumer_name, "COUNT", 2, "STREAMS", "user_events", ">")
+		
+		if messages && !messages.empty?
+			stream_name, entries = messages[0]
+			puts "Consumer #{consumer_name} received #{entries.length} messages:"
+			
+			entries.each do |entry_id, fields|
+				puts "  Processing #{entry_id}: #{fields}"
+				
+				# Simulate message processing:
+				sleep 0.1
+				
+				# Acknowledge message processing:
+				client.xack("user_events", "processors", entry_id)
+				puts "  Acknowledged #{entry_id}"
+			end
+		else
+			puts "No new messages for consumer #{consumer_name}"
+		end
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Multiple Consumers
+
+Demonstrate load balancing across multiple consumers:
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do |task|
+	begin
+		# Create consumer group:
+		begin
+			client.xgroup("CREATE", "work_queue", "workers", "0", "MKSTREAM")
+		rescue Protocol::Redis::ServerError
+			# Group already exists
+		end
+		
+		# Producer task - add work items:
+		producer = task.async do
+			10.times do |i|
+				client.xadd("work_queue", "*", 
+					"task_id" => i, 
+					"data" => "work_item_#{i}",
+					"priority" => rand(1..5)
+				)
+				puts "Added work item #{i}"
+				sleep 0.5
+			end
+		end
+		
+		# Consumer tasks - process work items:
+		consumers = 3.times.map do |worker_id|
+			task.async do
+				consumer_name = "worker_#{worker_id}"
+				
+				loop do
+					messages = client.xreadgroup(
+						"GROUP", "workers", consumer_name,
+						"COUNT", 1,
+						"BLOCK", 1000,
+						"STREAMS", "work_queue", ">"
+					)
+					
+					if messages && !messages.empty?
+						stream_name, entries = messages[0]
+						
+						entries.each do |entry_id, fields|
+							puts "#{consumer_name} processing: #{fields}"
+							
+							# Simulate work:
+							sleep rand(0.1..0.5)
+							
+							# Acknowledge completion:
+							client.xack("work_queue", "workers", entry_id)
+							puts "#{consumer_name} completed: #{entry_id}"
+						end
+					end
+				end
+			end
+		end
+		
+		# Wait for producer to finish:
+		producer.wait
+		
+		# Let consumers process remaining work:
+		sleep 3
+		
+		# Stop all consumers:
+		consumers.each(&:stop)
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Message Acknowledgment and Recovery
+
+Handle message acknowledgment and recover from failures:
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Check for pending messages:
+		pending_info = client.xpending("user_events", "processors")
+		if pending_info && pending_info[0] > 0
+			puts "Found #{pending_info[0]} pending messages"
+			
+			# Get detailed pending information:
+			pending_details = client.xpending("user_events", "processors", "-", "+", 10)
+			pending_details.each do |entry_id, consumer, idle_time, delivery_count|
+				puts "Message #{entry_id} pending for #{idle_time}ms (delivered #{delivery_count} times to #{consumer})"
+				
+				# Claim long-pending messages for reprocessing:
+				if idle_time > 60000  # 1 minute
+					claimed = client.xclaim("user_events", "processors", "recovery_worker", 60000, entry_id)
+					if claimed && !claimed.empty?
+						puts "Claimed message #{entry_id} for reprocessing"
+						
+						# Process the claimed message:
+						claimed.each do |claimed_id, fields|
+							puts "Reprocessing: #{fields}"
+							# ... process message ...
+						end
+						
+						# Acknowledge after successful processing:
+						client.xack("user_events", "processors", entry_id)
+						puts "Acknowledged recovered message #{entry_id}"
+					end
+				end
+			end
+		else
+			puts "No pending messages found"
+		end
+		
+	ensure
+		client.close
+	end
+end
+```
+
+## Stream Information and Management
+
+Monitor and manage stream health:
+
+``` ruby
+require "async/redis"
+
+endpoint = Async::Redis.local_endpoint
+client = Async::Redis::Client.new(endpoint)
+
+Async do
+	begin
+		# Get stream information:
+		stream_info = client.xinfo("STREAM", "user_events")
+		puts "Stream info: #{stream_info}"
+		
+		# Get consumer group information:
+		begin
+			groups_info = client.xinfo("GROUPS", "user_events")
+			puts "Consumer groups:"
+			groups_info.each do |group|
+				group_data = Hash[*group]
+				puts "  Group: #{group_data['name']}, Consumers: #{group_data['consumers']}, Pending: #{group_data['pending']}"
+			end
+		rescue Protocol::Redis::ServerError
+			puts "No consumer groups exist for this stream"
+		end
+		
+		# Get consumers in a group:
+		begin
+			consumers_info = client.xinfo("CONSUMERS", "user_events", "processors")
+			puts "Consumers in 'processors' group:"
+			consumers_info.each do |consumer|
+				consumer_data = Hash[*consumer]
+				puts "  Consumer: #{consumer_data['name']}, Pending: #{consumer_data['pending']}, Idle: #{consumer_data['idle']}ms"
+			end
+		rescue Protocol::Redis::ServerError
+			puts "Consumer group 'processors' does not exist"
+		end
+		
+		# Trim stream to keep only recent messages:
+		trimmed = client.xtrim("user_events", "MAXLEN", "~", 1000)
+		puts "Trimmed #{trimmed} messages from stream"
+		
+	ensure
+		client.close
+	end
+end
+```