anima-core 0.2.1 → 1.0.0

data/skills/activerecord/examples/querying/batch_processing.rb

@@ -0,0 +1,226 @@
+# Batch Processing Examples
+# Demonstrates find_each, find_in_batches, in_batches for large datasets
+
+# ============================================
+# The Problem: Loading All Records
+# ============================================
+
+# BAD - loads ALL records into memory at once
+User.all.each do |user|
+  NewsMailer.weekly_digest(user).deliver_later
+end
+# With 1 million users = 1 million User objects in memory!
+
+# GOOD - loads in batches of 1000, GC can clean up between batches
+User.find_each do |user|
+  NewsMailer.weekly_digest(user).deliver_later
+end
+# Memory stays constant regardless of total count
+
+# ============================================
+# find_each - Process Individual Records
+# ============================================
+
+# Basic usage - yields one record at a time
+User.find_each do |user|
+  user.calculate_monthly_stats
+end
+
+# With batch size
+User.find_each(batch_size: 500) do |user|
+  # Process in batches of 500
+end
+
+# With start/finish - process subset by primary key
+User.find_each(start: 1000, finish: 5000) do |user|
+  # Only users with id between 1000 and 5000
+end
+
+# With scope
+User.active.find_each do |user|
+  # Only active users
+end
+
+# Enumerator form (for chaining)
+User.find_each.map(&:email)
+User.find_each.with_index { |user, i| puts "#{i}: #{user.name}" }
+
+# ============================================
+# find_in_batches - Process Batches of Records
+# ============================================
+
+# Yields arrays of records
+User.find_in_batches(batch_size: 100) do |users|
+  # users is Array<User> with up to 100 elements
+  ExternalApi.bulk_sync(users.map(&:external_id))
+end
+
+# Practical: Bulk API calls
+Product.find_in_batches(batch_size: 50) do |products|
+  SearchIndex.bulk_update(products)
+  sleep(0.5)  # Rate limiting
+end
+
+# With eager loading
+User.includes(:profile, :preferences).find_in_batches do |users|
+  users.each do |user|
+    user.profile  # No N+1
+  end
+end
+
+# ============================================
+# in_batches - Process Batches as Relations
+# ============================================
+
+# Yields ActiveRecord::Relation objects
+User.in_batches do |batch|
+  # batch is a Relation, not Array
+  batch.update_all(newsletter_sent_at: Time.current)
+end
+
+# Bulk update pattern
+User.where(legacy: true).in_batches.update_all(migrated: true)
+
+# Bulk delete with throttling
+User.where("created_at < ?", 5.years.ago).in_batches do |batch|
+  batch.delete_all
+  sleep(0.1)  # Reduce database load
+end
+
+# Bulk operations via relation
+Order.where(status: "pending").in_batches do |batch|
+  batch.update_all(status: "cancelled", cancelled_at: Time.current)
+end
+
+# With load: true to also get records
+User.in_batches(load: true) do |batch|
+  batch.each { |user| user.some_instance_method }
+end
+
+# ============================================
+# Comparison: When to Use Which
+# ============================================
+
+# find_each - Individual record processing
+# - Sending emails one by one
+# - Complex per-record logic
+# - Instance methods needed
+User.find_each { |u| u.send_notification }
+
+# find_in_batches - Batch operations on loaded records
+# - Bulk API calls with arrays
+# - Batch exports
+# - When you need the actual objects in groups
+User.find_in_batches { |users| CsvExporter.export(users) }
+
+# in_batches - SQL-level bulk operations
+# - update_all, delete_all
+# - Bulk SQL operations
+# - Maximum efficiency, no Ruby object overhead
+User.in_batches.update_all(processed: true)
+
+# ============================================
+# Important Caveats
+# ============================================
+
+# ORDERING IS IGNORED
+# Batch processing always orders by primary key
+User.order(:name).find_each { |u| }
+# WARNING: Scoped order is ignored, will use primary key order
+
+# RESULTS MAY BE INCONSISTENT
+# If records are added/deleted during iteration, may skip/duplicate
+# Solution: Use in_batches with explicit locking for critical operations
+
+# CUSTOM ORDER WITH cursor (Rails 7.1+)
+User.find_each(cursor: [:created_at, :id]) { |u| }
+# Orders by created_at, then id for stability
+
+# ============================================
+# Practical Examples
+# ============================================
+
+# Example 1: Data migration
+class BackfillUserSettings < ActiveRecord::Migration[7.1]
+  def up
+    User.in_batches do |batch|
+      batch.update_all(settings: { notifications: true }.to_json)
+    end
+  end
+end
+
+# Example 2: Export to CSV
+require "csv"
+
+def export_users_to_csv(file_path)
+  CSV.open(file_path, "w") do |csv|
+    csv << ["ID", "Name", "Email", "Created At"]
+
+    User.find_each do |user|
+      csv << [user.id, user.name, user.email, user.created_at]
+    end
+  end
+end
+
+# Example 3: Background job processing
+class ProcessAllOrdersJob < ApplicationJob
+  def perform
+    Order.pending.find_each do |order|
+      ProcessOrderJob.perform_later(order.id)
+    end
+  end
+end
+
+# Example 4: Batch API sync with rate limiting
+def sync_products_to_external_service
+  Product.active.find_in_batches(batch_size: 25) do |products|
+    ExternalService.bulk_upsert(
+      products.map { |p| p.as_external_format }
+    )
+    sleep(1)  # Respect rate limits
+  end
+end
+
+# Example 5: Data cleanup with progress tracking
+def cleanup_old_sessions
+  total = Session.where("created_at < ?", 30.days.ago).count
+  deleted = 0
+
+  Session.where("created_at < ?", 30.days.ago).in_batches do |batch|
+    count = batch.delete_all
+    deleted += count
+    Rails.logger.info "Deleted #{deleted}/#{total} old sessions"
+    sleep(0.05)
+  end
+end
+
+# Example 6: Memory-efficient aggregation
+def calculate_total_balance
+  total = 0
+
+  Account.active.find_each do |account|
+    total += account.calculated_balance  # Complex calculation
+  end
+
+  total
+end
+
+# Better: Use database when possible
+Account.active.sum(:balance)  # Single SQL query
+
+# ============================================
+# Batch Processing with Transactions
+# ============================================
+
+# Each batch in its own transaction
+User.in_batches do |batch|
+  batch.transaction do
+    batch.update_all(processed: true)
+    AuditLog.create!(action: "batch_processed", count: batch.count)
+  end
+end
+
+# Whole operation in one transaction (careful with large datasets!)
+User.transaction do
+  User.in_batches.update_all(processed: true)
+end

data/skills/activerecord/examples/querying/eager_loading.rb

@@ -0,0 +1,259 @@
+# Eager Loading Examples
+# Demonstrates includes, preload, eager_load, joins and N+1 prevention
+
+# ============================================
+# Sample Models for Examples
+# ============================================
+
+class Author < ApplicationRecord
+  has_many :posts
+  has_many :comments, through: :posts
+end
+
+class Post < ApplicationRecord
+  belongs_to :author
+  has_many :comments
+  has_many :tags, through: :taggings
+  has_many :taggings
+end
+
+class Comment < ApplicationRecord
+  belongs_to :post
+  belongs_to :user
+end
+
+# ============================================
+# The N+1 Problem
+# ============================================
+
+# BAD - N+1 queries (1 query + N additional queries)
+posts = Post.limit(10)
+posts.each do |post|
+  puts post.author.name  # Each iteration triggers a query!
+end
+# Query 1: SELECT * FROM posts LIMIT 10
+# Query 2: SELECT * FROM authors WHERE id = 1
+# Query 3: SELECT * FROM authors WHERE id = 2
+# ... 10 more queries!
+
+# GOOD - Eager loading (2 queries total)
+posts = Post.includes(:author).limit(10)
+posts.each do |post|
+  puts post.author.name  # No additional queries!
+end
+# Query 1: SELECT * FROM posts LIMIT 10
+# Query 2: SELECT * FROM authors WHERE id IN (1, 2, 3, ...)
+
+# ============================================
+# includes - Smart Default (Auto-chooses strategy)
+# ============================================
+
+# Separate queries (preload strategy) - when just accessing data
+Post.includes(:author)
+# SELECT * FROM posts
+# SELECT * FROM authors WHERE id IN (1, 2, 3, ...)
+
+# Single JOIN (eager_load strategy) - when filtering by association
+Post.includes(:author).where(authors: { verified: true })
+# SELECT posts.*, authors.*
+#   FROM posts LEFT OUTER JOIN authors ON authors.id = posts.author_id
+#   WHERE authors.verified = true
+
+# Multiple associations
+Post.includes(:author, :comments)
+Post.includes(:author, comments: :user)  # Nested
+
+# ============================================
+# references - Required for String Conditions
+# ============================================
+
+# ERROR - Rails doesn't know to JOIN
+Post.includes(:author).where("authors.created_at > ?", 1.week.ago)
+# PG::UndefinedTable: ERROR: missing FROM-clause entry for table "authors"
+
+# CORRECT - explicitly reference the table
+Post.includes(:author)
+    .where("authors.created_at > ?", 1.week.ago)
+    .references(:authors)
+# SELECT posts.*, authors.*
+#   FROM posts LEFT OUTER JOIN authors ON ...
+#   WHERE authors.created_at > '2024-01-23'
+
+# Hash conditions auto-reference
+Post.includes(:author).where(authors: { verified: true })  # Works!
+
+# ============================================
+# preload - Always Separate Queries
+# ============================================
+
+# Forces separate queries regardless of conditions
+Author.preload(:posts, :comments)
+# SELECT * FROM authors
+# SELECT * FROM posts WHERE author_id IN (1, 2, 3)
+# SELECT * FROM comments WHERE author_id IN (1, 2, 3)
+
+# CANNOT filter by preloaded association
+Author.preload(:posts).where(posts: { published: true })
+# ERROR! posts is not in the FROM clause
+
+# Use When:
+# - Large datasets (JOINs cause row multiplication)
+# - Not filtering by associated data
+# - Want predictable query behavior
+
+# Example: preload is better here
+authors = Author.preload(:posts)  # 2 queries
+# vs eager_load with 1000 posts per author = massive result set
+
+# ============================================
+# eager_load - Always LEFT OUTER JOIN
+# ============================================
+
+# Forces single query with LEFT OUTER JOIN
+Author.eager_load(:posts)
+# SELECT authors.*, posts.*
+#   FROM authors
+#   LEFT OUTER JOIN posts ON posts.author_id = authors.id
+
+# Use When:
+# - Filtering by association attributes
+# - Sorting by association attributes
+# - Need records even without associations (LEFT join includes NULLs)
+
+# Example: authors sorted by latest post
+Author.eager_load(:posts)
+      .order("posts.created_at DESC NULLS LAST")
+      .distinct
+
+# ============================================
+# joins - INNER JOIN (Filtering Only)
+# ============================================
+
+# Creates JOIN but does NOT load associated records
+Author.joins(:posts).where(posts: { published: true }).distinct
+# SELECT DISTINCT authors.*
+#   FROM authors
+#   INNER JOIN posts ON posts.author_id = authors.id
+#   WHERE posts.published = true
+
+# WARNING: Accessing association still causes N+1!
+Author.joins(:posts).each do |author|
+  author.posts  # N+1 query here!
+end
+
+# Use When:
+# - Only need to filter, not access associated data
+# - Combine with includes for filtering + loading
+
+# Pattern: joins + includes together
+Author.joins(:posts)
+      .includes(:posts)
+      .where(posts: { published: true })
+      .distinct
+
+# ============================================
+# left_outer_joins - Include Records Without Association
+# ============================================
+
+# Authors including those without posts
+Author.left_outer_joins(:posts)
+      .select("authors.*, COUNT(posts.id) AS posts_count")
+      .group("authors.id")
+
+# Find authors with no posts
+Author.left_outer_joins(:posts).where(posts: { id: nil })
+
+# ============================================
+# Nested Eager Loading
+# ============================================
+
+# One level
+Post.includes(:author)
+
+# Two levels
+Post.includes(author: :profile)
+
+# Multiple at same level
+Post.includes(:author, :comments)
+
+# Multiple nested
+Post.includes(author: [:profile, :posts])
+
+# Deep nesting
+Post.includes(comments: { user: :profile })
+
+# Mixed
+Post.includes(:author, comments: { user: :profile }, tags: :category)
+
+# ============================================
+# Eager Loading Decision Tree (Applied)
+# ============================================
+
+# Scenario 1: Display posts with author names
+# Need: Access author → Use includes
+posts = Post.includes(:author).limit(20)
+posts.each { |p| "#{p.title} by #{p.author.name}" }
+
+# Scenario 2: Find posts by verified authors only
+# Need: Filter by association → Use eager_load or includes with references
+posts = Post.eager_load(:author).where(authors: { verified: true })
+
+# Scenario 3: Count posts per author
+# Need: Filter only, no data access → Use joins
+Post.joins(:author)
+    .group("authors.id")
+    .count
+
+# Scenario 4: Load authors with their many posts (large dataset)
+# Need: Access data, large join explosion risk → Use preload
+authors = Author.preload(:posts).limit(10)
+
+# Scenario 5: Load posts with multiple associations
+# Need: Mixed access patterns
+Post.includes(:author)           # Always accessed
+    .includes(:comments)         # Sometimes accessed
+    .preload(:tags)              # Large count, rarely accessed
+
+# ============================================
+# Performance Comparison
+# ============================================
+
+# Setup: 100 posts, each with 1 author and 50 comments
+
+# N+1 (worst)
+Post.all.each { |p| p.author; p.comments.to_a }
+# 1 + 100 + 100 = 201 queries
+
+# includes (better)
+Post.includes(:author, :comments).each { |p| p.author; p.comments.to_a }
+# 3 queries (posts, authors, comments)
+
+# eager_load with many comments (can be slow)
+Post.eager_load(:author, :comments).each { |p| p.author; p.comments.to_a }
+# 1 query, but returns 100 * 50 = 5000 rows!
+
+# preload is safer for large associations
+Post.preload(:author, :comments).each { |p| p.author; p.comments.to_a }
+# 3 queries, 100 + 100 + 5000 rows loaded separately
+
+# ============================================
+# Strict Loading - Prevent N+1 at Runtime
+# ============================================
+
+# Relation level - raises if lazy loading attempted
+posts = Post.strict_loading
+posts.first.author  # ActiveRecord::StrictLoadingViolationError!
+
+# Must eager load to access
+posts = Post.strict_loading.includes(:author)
+posts.first.author  # Works!
+
+# Model level - all queries are strict
+class Post < ApplicationRecord
+  self.strict_loading_by_default = true
+end
+
+# Association level
+class Author < ApplicationRecord
+  has_many :posts, strict_loading: true
+end

data/skills/activerecord/examples/querying/finder_methods.rb

@@ -0,0 +1,170 @@
+# Finder Methods Examples
+# Demonstrates find, find_by, where, and find_or_create_by
+
+# ============================================
+# Sample Models for Examples
+# ============================================
+
+class User < ApplicationRecord
+  has_many :posts
+  has_many :orders
+
+  scope :active, -> { where(active: true) }
+  scope :admins, -> { where(role: "admin") }
+end
+
+# ============================================
+# find - Retrieve by Primary Key
+# ============================================
+
+# Single record - raises RecordNotFound if not found
+user = User.find(1)
+# SELECT * FROM users WHERE id = 1
+
+# Multiple records - returns array, raises if ANY not found
+users = User.find([1, 2, 3])
+users = User.find(1, 2, 3)  # Same result
+# SELECT * FROM users WHERE id IN (1, 2, 3)
+
+# Use when record MUST exist (will crash otherwise)
+def show
+  @user = User.find(params[:id])  # 404 if not found
+end
+
+# ============================================
+# find_by - Retrieve First Match
+# ============================================
+
+# Returns nil if not found
+user = User.find_by(email: "test@example.com")
+# SELECT * FROM users WHERE email = 'test@example.com' LIMIT 1
+
+# Multiple conditions
+user = User.find_by(email: "test@example.com", active: true)
+
+# With string conditions (use placeholders!)
+user = User.find_by("email LIKE ?", "%@company.com")
+
+# find_by! raises RecordNotFound
+user = User.find_by!(email: "test@example.com")
+
+# Use when absence is acceptable
+def authenticate(email, password)
+  user = User.find_by(email:)
+  return nil unless user
+  user if user.authenticate(password)
+end
+
+# ============================================
+# where - Build Conditions (Returns Relation)
+# ============================================
+
+# Hash conditions - safest, auto-escaped
+User.where(active: true)
+User.where(role: ["admin", "moderator"])  # IN clause
+User.where(age: 18..65)                   # BETWEEN
+User.where(deleted_at: nil)               # IS NULL
+
+# Chaining - lazy evaluation
+users = User.where(active: true)
+            .where(role: "admin")
+            .order(created_at: :desc)
+# Query not executed until iteration
+
+# String conditions - ALWAYS use placeholders
+User.where("age > ?", 18)
+User.where("name LIKE ?", "%#{User.sanitize_sql_like(query)}%")
+User.where("created_at > :date", date: 1.week.ago)
+
+# DANGER - SQL injection!
+# User.where("name = '#{params[:name]}'")  # NEVER do this!
+
+# ============================================
+# where.not - Negation
+# ============================================
+
+User.where.not(role: "admin")
+# WHERE role != 'admin'
+
+User.where.not(deleted_at: nil)
+# WHERE deleted_at IS NOT NULL
+
+User.where.not(status: ["banned", "suspended"])
+# WHERE status NOT IN ('banned', 'suspended')
+
+# ============================================
+# where.associated / where.missing (Rails 7+)
+# ============================================
+
+# Users who have at least one post
+User.where.associated(:posts)
+# SELECT users.* FROM users
+#   INNER JOIN posts ON posts.user_id = users.id
+
+# Users with no posts
+User.where.missing(:posts)
+# SELECT users.* FROM users
+#   LEFT OUTER JOIN posts ON posts.user_id = users.id
+#   WHERE posts.id IS NULL
+
+# ============================================
+# find_or_create_by / find_or_initialize_by
+# ============================================
+
+# Finds existing or creates new record
+user = User.find_or_create_by(email: "new@example.com") do |u|
+  u.name = "New User"  # Only set for new records
+  u.role = "member"
+end
+
+# Finds or builds (doesn't save)
+user = User.find_or_initialize_by(email: "maybe@example.com")
+user.new_record?  # true if not found, false if found
+user.save if user.new_record?
+
+# With scoped relation
+admin = User.admins.find_or_create_by(email: "admin@example.com")
+
+# ============================================
+# Handling Race Conditions
+# ============================================
+
+# find_or_create_by can fail under concurrent access
+# when two requests try to create the same unique record
+
+# Solution 1: Rescue and retry
+def find_or_create_user(email)
+  User.find_or_create_by(email:)
+rescue ActiveRecord::RecordNotUnique
+  retry
+end
+
+# Solution 2: Use create_or_find_by (Rails 6+)
+# Creates first, finds on unique constraint violation
+User.create_or_find_by(email: "user@example.com")
+
+# ============================================
+# Practical Examples
+# ============================================
+
+# API controller filtering
+class UsersController < ApplicationController
+  def index
+    @users = User.all
+
+    @users = @users.where(role: params[:role]) if params[:role].present?
+    @users = @users.where(active: true) if params[:active] == "true"
+    @users = @users.where("created_at >= ?", params[:since]) if params[:since]
+
+    @users = @users.order(params[:sort] || :created_at)
+    @users = @users.limit(params[:limit] || 20)
+  end
+end
+
+# Search with sanitization
+def search_users(query)
+  return User.none if query.blank?
+
+  sanitized = User.sanitize_sql_like(query)
+  User.where("name ILIKE :q OR email ILIKE :q", q: "%#{sanitized}%")
+end