familia 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a908aad71096ba6a650fa67545aad0a2f891be6472fc6cb4ed067977c4edd706
-  data.tar.gz: 01d676ccdf7a783d585053be2861319785a8aa5c33298848607f589b3dfa6aaf
+  metadata.gz: e83013bddd8ff985ad1cd9ef73421146b5fea6c35cbb77aa0cf12656dd270f68
+  data.tar.gz: ba7ebef4af806d823317fd41e1875733b12971b3c6321e841c9ecbf0c9d727a0
 SHA512:
-  metadata.gz: 6eb5d61cb2255e5901969ae607b324a1b396f819562815d9f7f57a851b433b6cd4c695d339978de0a3341f92ddb909c8c4878a3580d442cf651449b6ac6d3be8
-  data.tar.gz: 96f2e0a1bdefbd12e6332aefcdf189ae8279d07adaa05348d839ebccf40a9e306bdd18fc21a5a9af478dab94e6b5de71fd6e86ac7cbdd264a99a26f502d598d8
+  metadata.gz: 126d79d60cad6c4ed4ed03a557f4435aec82d03ef2230ad64e4d2205f8c8856cc74b140c18fb5f1d567631f203f6b259e7d19d0bed8f36e09b5540fc004f4e6d
+  data.tar.gz: cb5a52b4622c9f193d3858b0f2766795f49000f56fccbffdc630dc5b05e68db0a8c200ad25137bf543d9fc64ee2f53cab891517a5d9f91c2dd8debeb2119b7e7

data/CHANGELOG.rst

@@ -7,6 +7,51 @@ The format is based on `Keep a Changelog <https://keepachangelog.com/en/1.1.0/>`
 
    <!--scriv-insert-here-->
 
+.. _changelog-2.1.0:
+
+2.1.0 — 2026-02-01
+==================
+
+Added
+-----
+
+- Redis-native migration system with three patterns: Base (abstract foundation),
+  Model (record-by-record iteration via SCAN), and Pipeline (bulk updates with
+  Redis pipelining). Includes dependency resolution using topological sort,
+  dry-run mode, CLI support, and comprehensive Rake tasks.
+
+- Migration registry for tracking applied migrations in Redis with rollback
+  support and schema drift detection.
+
+- Lua script framework with atomic operations: rename_field, copy_field,
+  delete_field, rename_key_preserve_ttl, and backup_and_modify_field.
+
+- Optional JSON Schema validation for Horreum models via ``feature :schema_validation``
+  with centralized SchemaRegistry supporting convention-based and explicit schema
+  discovery using the json_schemer gem.
+
+- V1 to V2 serialization migration example at ``examples/migrations/v1_to_v2_serialization_migration.rb``
+  demonstrating how to upgrade Horreum objects from v1.x format (selective serialization
+  with type information loss) to v2.0 format (universal JSON encoding with type preservation).
+  Includes type detection heuristics, field type declarations, and batch processing.
+
+Documentation
+-------------
+
+- Added comprehensive migration writing guide at ``docs/guides/writing-migrations.md``
+  covering all three migration patterns, CLI usage, dependencies, and best practices.
+
+AI Assistance
+-------------
+
+- Claude Code assisted with test coverage analysis, identifying gaps in Model and
+  Pipeline test coverage. Implemented 67 new tests covering CLI entry points,
+  circular dependency detection, and comprehensive Model/Pipeline scenarios.
+
+- Claude Code identified and fixed a bug where schema validation hooks were never
+  triggered in Model migrations, and optimized N+1 query patterns in Registry and
+  Runner classes.
+
 .. _changelog-2.0.0:
 
 2.0.0 — 2026-01-19

data/Gemfile

@@ -15,6 +15,8 @@ end
 group :development, :test do
   gem 'benchmark', '~> 0.4', require: false
   gem 'debug', require: false
+  gem 'json_schemer', '~> 2.0', require: false
+  gem 'rake', '~> 13.0', require: false
   gem 'irb', '~> 1.15.2', require: false
   gem 'redcarpet', require: false
   gem 'reek', require: false

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    familia (2.0.0)
+    familia (2.1.0)
       concurrent-ruby (~> 1.3)
       connection_pool (~> 2.5)
       csv (~> 3.3)
@@ -58,12 +58,18 @@ GEM
     erb (5.1.3)
     ffi (1.17.2)
     ffi (1.17.2-arm64-darwin)
+    hana (1.3.7)
     io-console (0.8.1)
     irb (1.15.3)
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     json (2.15.1)
+    json_schemer (2.5.0)
+      bigdecimal
+      hana (~> 1.3)
+      regexp_parser (~> 2.0)
+      simpleidn (~> 0.2)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     logger (1.7.0)
@@ -87,6 +93,7 @@ GEM
       stringio
     racc (1.8.1)
     rainbow (3.1.1)
+    rake (13.3.1)
     rbnacl (7.1.2)
       ffi (~> 1)
     rbs (3.9.5)
@@ -152,6 +159,7 @@ GEM
     ruby-prof (1.7.2)
       base64
     ruby-progressbar (1.13.0)
+    simpleidn (0.2.3)
     stackprof (0.2.27)
     stringio (3.1.9)
     timecop (0.9.10)
@@ -185,6 +193,8 @@ DEPENDENCIES
   debug
   familia!
   irb (~> 1.15.2)
+  json_schemer (~> 2.0)
+  rake (~> 13.0)
   rbnacl (~> 7.1, >= 7.1.1)
   redcarpet
   reek

data/docs/guides/writing-migrations.md

@@ -0,0 +1,345 @@
+# Writing Migrations
+
+## Quick Start
+
+Minimal working migration:
+
+```ruby
+class NormalizeEmails < Familia::Migration::Base
+  self.migration_id = '20260131_120000_normalize_emails'
+  self.description = 'Lowercase all email addresses'
+
+  def migration_needed?
+    redis.exists('needs:normalization') > 0
+  end
+
+  def migrate
+    redis.scan_each(match: 'user:*:object') do |key|
+      for_realsies_this_time? do
+        # perform changes
+      end
+      track_stat(:processed)
+    end
+  end
+end
+```
+
+Run it:
+
+```bash
+bundle exec rake familia:migrate:dry_run    # Preview
+bundle exec rake familia:migrate            # Apply
+```
+
+## Migration Types
+
+| Type | Use When | Key Method |
+|------|----------|------------|
+| `Base` | Raw Redis operations, key patterns, config changes | `migrate` |
+| `Model` | Iterating over Horreum objects with per-record logic | `process_record(obj, key)` |
+| `Pipeline` | Bulk updates with Redis pipelining (1000+ records) | `should_process?(obj)`, `build_update_fields(obj)` |
+
+### Base
+
+Direct Redis access. Use for key renames, TTL changes, config migrations:
+
+```ruby
+class AddTTLToSessions < Familia::Migration::Base
+  self.migration_id = '20260131_add_ttl'
+
+  def migration_needed?
+    redis.exists('legacy:session:*') > 0
+  end
+
+  def migrate
+    cursor = '0'
+    loop do
+      cursor, keys = redis.scan(cursor, match: 'legacy:session:*', count: 1000)
+      keys.each do |key|
+        for_realsies_this_time? { redis.expire(key, 3600) }
+        track_stat(:keys_expired)
+      end
+      break if cursor == '0'
+    end
+  end
+end
+```
+
+### Model
+
+SCAN-based iteration over Horreum objects. Use for per-record transformations with error handling:
+
+```ruby
+class CustomerEmailMigration < Familia::Migration::Model
+  self.migration_id = '20260131_customer_emails'
+
+  def prepare
+    @model_class = Customer
+    @batch_size = 500  # optional, default: 1000
+  end
+
+  def process_record(customer, key)
+    return unless customer.email =~ /[A-Z]/
+
+    for_realsies_this_time? do
+      customer.email = customer.email.downcase
+      customer.save
+    end
+    track_stat(:records_updated)
+  end
+end
+```
+
+### Pipeline
+
+Batched updates using Redis pipelining. Use for high-volume simple field updates:
+
+```ruby
+class AddDefaultSettings < Familia::Migration::Pipeline
+  self.migration_id = '20260131_default_settings'
+
+  def prepare
+    @model_class = User
+    @batch_size = 100  # smaller batches for pipelines
+  end
+
+  def should_process?(user)
+    user.settings.nil?
+  end
+
+  def build_update_fields(user)
+    { 'settings' => '{}' }
+  end
+end
+```
+
+Override `execute_update` for custom pipeline operations:
+
+```ruby
+def execute_update(pipe, obj, fields, original_key)
+  dbkey = original_key || obj.dbkey
+  pipe.hmset(dbkey, *fields.flatten)
+  pipe.expire(dbkey, 86400)  # also set TTL
+end
+```
+
+## Class Attributes
+
+| Attribute | Required | Description |
+|-----------|----------|-------------|
+| `migration_id` | Yes | Unique identifier. Format: `YYYYMMDD_HHMMSS_snake_case_name` |
+| `description` | No | Human-readable summary for status output |
+| `dependencies` | No | Array of migration IDs that must run first |
+
+```ruby
+class BuildEmailIndex < Familia::Migration::Base
+  self.migration_id = '20260131_150000_build_index'
+  self.description = 'Create secondary index for email lookups'
+  self.dependencies = ['20260131_120000_normalize_emails']
+  # ...
+end
+```
+
+## Lifecycle Methods
+
+| Method | Purpose | Required |
+|--------|---------|----------|
+| `prepare` | Initialize config, set `@model_class` | Model/Pipeline only |
+| `migration_needed?` | Idempotency check. Return `false` to skip. | Yes |
+| `migrate` | Core migration logic | Base only |
+| `process_record(obj, key)` | Per-record logic | Model only |
+| `should_process?(obj)` | Filter predicate | Pipeline only |
+| `build_update_fields(obj)` | Return Hash of field updates | Pipeline only |
+| `down` | Rollback logic | No (enables rollback) |
+
+## Dry Run vs Live
+
+Wrap destructive operations with `for_realsies_this_time?`:
+
+```ruby
+def migrate
+  redis.scan_each(match: 'session:*') do |key|
+    for_realsies_this_time? do
+      redis.del(key)  # only executes with --run
+    end
+    track_stat(:deleted)
+  end
+end
+```
+
+| Mode | `for_realsies_this_time?` | Registry Updated |
+|------|---------------------------|------------------|
+| Dry run (`:dry_run` task) | Block skipped | No |
+| Live (`:run` task) | Block executes | Yes |
+
+## Dependencies
+
+Dependencies ensure execution order. The runner uses topological sort (Kahn's algorithm).
+
+```ruby
+class MigrationA < Familia::Migration::Base
+  self.migration_id = 'step_a'
+  self.dependencies = []
+end
+
+class MigrationB < Familia::Migration::Base
+  self.migration_id = 'step_b'
+  self.dependencies = ['step_a']  # runs after step_a
+end
+```
+
+Rollback is blocked if dependents are still applied:
+
+```ruby
+runner.rollback('step_a')
+# => Errors::HasDependents if step_b is applied
+```
+
+## Rollback
+
+Implement `down` to enable rollback:
+
+```ruby
+class AddFeatureFlag < Familia::Migration::Base
+  self.migration_id = '20260131_feature_flag'
+
+  def migration_needed?
+    !redis.exists?('config:feature:enabled')
+  end
+
+  def migrate
+    for_realsies_this_time? do
+      redis.set('config:feature:enabled', 'true')
+    end
+  end
+
+  def down
+    redis.del('config:feature:enabled')
+  end
+end
+```
+
+Check reversibility:
+
+```ruby
+instance = AddFeatureFlag.new
+instance.reversible?  # => true
+```
+
+## Lua Scripts
+
+Use `Familia::Migration::Script` for atomic operations:
+
+```ruby
+# Rename hash field atomically
+Familia::Migration::Script.execute(
+  redis,
+  :rename_field,
+  keys: ['user:123:object'],
+  argv: ['old_name', 'new_name']
+)
+```
+
+Built-in scripts:
+
+| Script | Purpose | KEYS | ARGV |
+|--------|---------|------|------|
+| `:rename_field` | Rename hash field | `[hash_key]` | `[old, new]` |
+| `:copy_field` | Copy field within hash | `[hash_key]` | `[src, dst]` |
+| `:delete_field` | Delete hash field | `[hash_key]` | `[field]` |
+| `:rename_key_preserve_ttl` | Rename key, keep TTL | `[src, dst]` | `[]` |
+| `:backup_and_modify_field` | Backup old value, set new | `[hash, backup]` | `[field, value, ttl]` |
+
+Register custom scripts:
+
+```ruby
+Familia::Migration::Script.register(:my_script, <<~LUA)
+  local key = KEYS[1]
+  return redis.call('GET', key)
+LUA
+```
+
+## CLI Reference
+
+```bash
+# Status
+bundle exec rake familia:migrate:status       # Show applied/pending
+bundle exec rake familia:migrate:validate     # Check dependency issues
+bundle exec rake familia:migrate:schema_drift # Models with changed schemas
+
+# Execution
+bundle exec rake familia:migrate:dry_run      # Preview (no changes)
+bundle exec rake familia:migrate              # Apply all pending
+bundle exec rake familia:migrate:run          # Same as above
+
+# Rollback
+bundle exec rake "familia:migrate:rollback[20260131_120000_migration_id]"
+```
+
+## Statistics
+
+Track operations with `track_stat`:
+
+```ruby
+def process_record(obj, key)
+  if obj.email.blank?
+    track_stat(:skipped_blank)
+    return
+  end
+
+  for_realsies_this_time? do
+    obj.email = obj.email.downcase
+    obj.save
+  end
+  track_stat(:records_updated)
+end
+```
+
+Access stats:
+
+```ruby
+instance.stats[:records_updated]  # => 42
+instance.stats[:skipped_blank]    # => 7
+```
+
+## Configuration
+
+```ruby
+Familia::Migration.configure do |config|
+  config.migrations_key = 'familia:migrations'  # Registry key prefix
+  config.backup_ttl = 86_400                    # Backup expiration (24h)
+  config.batch_size = 1000                      # Default SCAN batch
+end
+```
+
+## Best Practices
+
+1. **Test locally first.** Run dry run, verify stats, then run live on staging before production.
+
+2. **Deploy schema changes separately.** Avoid updating model definitions and running migrations in the same deploy. New model logic can break migration code.
+
+3. **Keep migrations idempotent.** `migration_needed?` should return `false` after successful execution.
+
+4. **Use descriptive IDs.** `20260131_120000_normalize_customer_emails` beats `20260131_fix_stuff`.
+
+5. **Backup critical data.** Use `:backup_and_modify_field` or `registry.backup_field` before destructive changes.
+
+## Error Reference
+
+| Error | Cause |
+|-------|-------|
+| `NotReversible` | `down` not implemented |
+| `NotApplied` | Rollback of unapplied migration |
+| `DependencyNotMet` | Dependency not yet applied |
+| `HasDependents` | Rollback blocked by dependents |
+| `CircularDependency` | Dependency cycle detected |
+| `PreconditionFailed` | `@model_class` not set in `prepare` |
+
+## Source Files
+
+- [`lib/familia/migration/base.rb`](../../lib/familia/migration/base.rb)
+- [`lib/familia/migration/model.rb`](../../lib/familia/migration/model.rb)
+- [`lib/familia/migration/pipeline.rb`](../../lib/familia/migration/pipeline.rb)
+- [`lib/familia/migration/registry.rb`](../../lib/familia/migration/registry.rb)
+- [`lib/familia/migration/runner.rb`](../../lib/familia/migration/runner.rb)
+- [`lib/familia/migration/script.rb`](../../lib/familia/migration/script.rb)