familia 2.0.0.pre26 → 2.1.0

data/try/migration/pipeline_try.rb

@@ -0,0 +1,460 @@
+# try/migration/pipeline_try.rb
+#
+# frozen_string_literal: true
+
+require_relative '../support/helpers/test_helpers'
+require_relative '../../lib/familia/migration'
+
+Familia.debug = false
+
+@redis = Familia.dbclient
+@test_id = "#{Process.pid}_#{Time.now.to_i}"
+@prefix = "familia:test:pipeline:#{@test_id}"
+
+@initial_migrations = Familia::Migration.migrations.dup
+
+# Test model class for pipeline migrations
+class PipelineTestRecord < Familia::Horreum
+  identifier_field :record_id
+  field :record_id
+  field :name
+  field :status
+  field :old_field
+  field :new_field
+  field :migrated_at
+end
+
+# Simple Pipeline migration
+class SimplePipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_simple'
+  self.description = 'Simple pipeline migration test'
+
+  class << self
+    attr_accessor :processed_count
+  end
+  self.processed_count = 0
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    true
+  end
+
+  def build_update_fields(obj)
+    self.class.processed_count += 1
+    { new_field: 'pipeline_updated' }
+  end
+end
+
+# Pipeline migration that filters records
+class FilteringPipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_filtering'
+
+  class << self
+    attr_accessor :skipped_count, :processed_count
+  end
+  self.skipped_count = 0
+  self.processed_count = 0
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    if obj.status == 'skip_me'
+      self.class.skipped_count += 1
+      track_stat(:skipped)
+      return false
+    end
+    self.class.processed_count += 1
+    true
+  end
+
+  def build_update_fields(obj)
+    { new_field: 'filtered_update', migrated_at: Time.now.to_i.to_s }
+  end
+end
+
+# Pipeline migration with empty fields (should skip HMSET)
+class EmptyFieldsPipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_empty'
+
+  class << self
+    attr_accessor :build_called_count
+  end
+  self.build_called_count = 0
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    true
+  end
+
+  def build_update_fields(obj)
+    self.class.build_called_count += 1
+    {} # Return empty hash
+  end
+end
+
+# Pipeline migration with nil fields
+class NilFieldsPipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_nil'
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    true
+  end
+
+  def build_update_fields(obj)
+    nil # Return nil
+  end
+end
+
+# Pipeline migration with custom execute_update
+class CustomExecutePipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_custom_execute'
+
+  class << self
+    attr_accessor :custom_execute_called, :original_keys
+  end
+  self.custom_execute_called = 0
+  self.original_keys = []
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    true
+  end
+
+  def build_update_fields(obj)
+    { custom_field: 'custom_value' }
+  end
+
+  def execute_update(pipe, obj, fields, original_key = nil)
+    self.class.custom_execute_called += 1
+    self.class.original_keys << original_key
+    super(pipe, obj, fields, original_key)
+  end
+end
+
+# Pipeline migration that errors during batch
+class ErrorPipelineMigration < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_error'
+
+  class << self
+    attr_accessor :error_triggered
+  end
+  self.error_triggered = false
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    if obj.name == 'trigger_error'
+      self.class.error_triggered = true
+      raise 'Intentional pipeline error'
+    end
+    true
+  end
+
+  def build_update_fields(obj)
+    { new_field: 'updated' }
+  end
+end
+
+# Pipeline without should_process? - should raise
+class MissingShouldProcessPipeline < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_missing_should'
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def build_update_fields(obj)
+    { field: 'value' }
+  end
+end
+
+# Pipeline without build_update_fields - should raise
+class MissingBuildFieldsPipeline < Familia::Migration::Pipeline
+  self.migration_id = 'pipeline_test_missing_build'
+
+  def prepare
+    @model_class = PipelineTestRecord
+    @batch_size = 10
+  end
+
+  def should_process?(obj)
+    true
+  end
+end
+
+# Helper to create test records with unique prefix
+def create_pipeline_record(suffix, name: 'Test', status: 'active')
+  id = "#{@test_id}_#{suffix}"
+  record = PipelineTestRecord.new(record_id: id, name: name, status: status)
+  record.save
+  record
+end
+
+# Helper to cleanup all test records
+def cleanup_records
+  pattern = "pipeline_test_record:#{@test_id}_*"
+  @redis.keys(pattern).each { |k| @redis.del(k) }
+end
+
+cleanup_records
+
+## Pipeline is a subclass of Model
+Familia::Migration::Pipeline < Familia::Migration::Model
+#=> true
+
+## Pipeline initializes correctly
+migration = SimplePipelineMigration.new
+migration.is_a?(Familia::Migration::Pipeline)
+#=> true
+
+## Pipeline prepare sets model_class
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new
+migration.prepare
+migration.model_class == PipelineTestRecord
+#=> true
+
+## Pipeline should_process? raises NotImplementedError for base class
+begin
+  Familia::Migration::Pipeline.new.send(:should_process?, nil)
+  false
+rescue NotImplementedError
+  true
+end
+#=> true
+
+## Pipeline build_update_fields raises NotImplementedError for base class
+begin
+  Familia::Migration::Pipeline.new.send(:build_update_fields, nil)
+  false
+rescue NotImplementedError
+  true
+end
+#=> true
+
+## Pipeline process_record is a no-op
+migration = SimplePipelineMigration.new
+migration.send(:process_record, nil, 'key')
+true
+#=> true
+
+## Pipeline processes records in batches
+cleanup_records
+create_pipeline_record('p1', name: 'P1')
+create_pipeline_record('p2', name: 'P2')
+create_pipeline_record('p3', name: 'P3')
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+SimplePipelineMigration.processed_count >= 3
+#=> true
+
+## Pipeline updates are applied in actual_run mode
+cleanup_records
+record = create_pipeline_record('q1', name: 'Q1')
+dbkey = record.dbkey
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+@redis.hget(dbkey, 'new_field')
+#=> "pipeline_updated"
+
+## Pipeline respects dry_run mode
+cleanup_records
+record = create_pipeline_record('r1', name: 'R1')
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new(run: false)
+migration.prepare
+migration.migrate
+reloaded = PipelineTestRecord.find_by_key(record.dbkey)
+reloaded.new_field.nil?
+#=> true
+
+## Pipeline filtering works correctly
+cleanup_records
+create_pipeline_record('s1', status: 'active')
+create_pipeline_record('s2', status: 'skip_me')
+create_pipeline_record('s3', status: 'active')
+FilteringPipelineMigration.skipped_count = 0
+FilteringPipelineMigration.processed_count = 0
+migration = FilteringPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+FilteringPipelineMigration.skipped_count >= 1 && FilteringPipelineMigration.processed_count >= 2
+#=> true
+
+## Pipeline tracks records_updated correctly
+cleanup_records
+create_pipeline_record('t1')
+create_pipeline_record('t2')
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+migration.records_updated >= 2
+#=> true
+
+## Pipeline with empty fields skips HMSET
+cleanup_records
+record = create_pipeline_record('u1', name: 'Original')
+EmptyFieldsPipelineMigration.build_called_count = 0
+migration = EmptyFieldsPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+EmptyFieldsPipelineMigration.build_called_count >= 1
+#=> true
+
+## Pipeline with nil fields skips HMSET
+cleanup_records
+record = create_pipeline_record('v1', name: 'Original')
+migration = NilFieldsPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+record.name
+#=> 'Original'
+
+## Pipeline custom execute_update is called
+cleanup_records
+record = create_pipeline_record('w1')
+CustomExecutePipelineMigration.custom_execute_called = 0
+CustomExecutePipelineMigration.original_keys = []
+migration = CustomExecutePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+CustomExecutePipelineMigration.custom_execute_called >= 1
+#=> true
+
+## Pipeline custom execute_update receives original_key
+cleanup_records
+create_pipeline_record('x1')
+CustomExecutePipelineMigration.custom_execute_called = 0
+CustomExecutePipelineMigration.original_keys = []
+migration = CustomExecutePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+CustomExecutePipelineMigration.original_keys.first.include?('object')
+#=> true
+
+## Pipeline handles batch errors gracefully
+cleanup_records
+create_pipeline_record('y1', name: 'trigger_error')
+ErrorPipelineMigration.error_triggered = false
+migration = ErrorPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+ErrorPipelineMigration.error_triggered && migration.error_count >= 1
+#=> true
+
+## Pipeline tracks errors per batch size
+cleanup_records
+create_pipeline_record('z1', name: 'trigger_error')
+create_pipeline_record('z2', name: 'normal')
+ErrorPipelineMigration.error_triggered = false
+migration = ErrorPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+migration.error_count >= 1
+#=> true
+
+## process_batch calls should_process? and build_update_fields
+cleanup_records
+create_pipeline_record('aa1')
+SimplePipelineMigration.processed_count = 0
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+SimplePipelineMigration.processed_count >= 1
+#=> true
+
+## Pipeline tracks total_scanned
+cleanup_records
+create_pipeline_record('bb1')
+create_pipeline_record('bb2')
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+migration.total_scanned >= 2
+#=> true
+
+## Pipeline tracks records_needing_update
+cleanup_records
+create_pipeline_record('cc1')
+create_pipeline_record('cc2')
+create_pipeline_record('cc3')
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+migration.records_needing_update >= 3
+#=> true
+
+## Pipeline returns true on success
+cleanup_records
+create_pipeline_record('dd1')
+migration = SimplePipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+#=> true
+
+## Pipeline returns false when errors
+cleanup_records
+create_pipeline_record('ee1', name: 'trigger_error')
+ErrorPipelineMigration.error_triggered = false
+migration = ErrorPipelineMigration.new(run: true)
+migration.prepare
+migration.migrate
+#=> false
+
+## MissingShouldProcessPipeline raises NotImplementedError
+cleanup_records
+create_pipeline_record('ff1')
+migration = MissingShouldProcessPipeline.new(run: true)
+migration.prepare
+begin
+  migration.migrate
+  false
+rescue NotImplementedError => e
+  e.message.include?('should_process?')
+end
+#=> true
+
+## MissingBuildFieldsPipeline raises NotImplementedError
+cleanup_records
+create_pipeline_record('gg1')
+migration = MissingBuildFieldsPipeline.new(run: true)
+migration.prepare
+begin
+  migration.migrate
+  false
+rescue NotImplementedError => e
+  e.message.include?('build_update_fields')
+end
+#=> true
+
+cleanup_records
+Familia::Migration.migrations.replace(@initial_migrations)

data/try/migration/rake_tasks_try.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'rake'
+require_relative '../../lib/familia/migration/rake_tasks'
+
+# Enable metadata recording so descriptions are available
+Rake::TaskManager.record_task_metadata = true
+
+# Clear existing tasks and reload to ensure fresh state
+Rake::Task.clear
+Familia::Migration::RakeTasks.new
+
+## All expected tasks are defined
+expected_tasks = %w[
+  familia:migrate
+  familia:migrate:run
+  familia:migrate:status
+  familia:migrate:dry_run
+  familia:migrate:rollback
+  familia:migrate:validate
+  familia:migrate:schema_drift
+]
+expected_tasks.all? { |name| Rake::Task.task_defined?(name) }
+#=> true
+
+## familia:migrate:run has correct description
+Rake::Task['familia:migrate:run'].comment
+#=> 'Run all pending migrations'
+
+## familia:migrate:status has correct description
+Rake::Task['familia:migrate:status'].comment
+#=> 'Show migration status table'
+
+## familia:migrate:dry_run has correct description
+Rake::Task['familia:migrate:dry_run'].comment
+#=> 'Preview pending migrations (dry run)'
+
+## familia:migrate:rollback has correct description
+Rake::Task['familia:migrate:rollback'].comment
+#=> 'Rollback a specific migration'
+
+## familia:migrate:validate has correct description
+Rake::Task['familia:migrate:validate'].comment
+#=> 'Validate migration dependencies'
+
+## familia:migrate:schema_drift has correct description
+Rake::Task['familia:migrate:schema_drift'].comment
+#=> 'List models with schema drift'
+
+## familia:migrate shortcut task has correct description
+Rake::Task['familia:migrate'].comment
+#=> 'Run all pending migrations'
+
+## familia:migrate shortcut has familia:migrate:run as prerequisite
+Rake::Task['familia:migrate'].prerequisites
+#=> ['migrate:run']
+
+## familia:migrate:rollback accepts an argument
+task = Rake::Task['familia:migrate:rollback']
+task.arg_names
+#=> [:id]

data/try/migration/registry_try.rb

@@ -0,0 +1,199 @@
+# try/migration/registry_try.rb
+#
+# frozen_string_literal: true
+
+require_relative '../support/helpers/test_helpers'
+require_relative '../../lib/familia/migration'
+
+Familia.debug = false
+
+@redis = Familia.dbclient
+@prefix = "familia:test:registry:#{Process.pid}:#{Time.now.to_i}"
+@registry = Familia::Migration::Registry.new(redis: @redis, prefix: @prefix)
+
+# Clean any existing test keys
+@redis.keys("#{@prefix}:*").each { |k| @redis.del(k) }
+
+# Mock migration classes for pending and status tests
+# Using instance variables so they persist across test cases
+@applied_migration = Class.new do
+  def self.migration_id; 'test_migration_1'; end
+end
+@pending_migration = Class.new do
+  def self.migration_id; 'test_migration_2'; end
+end
+@also_pending_migration = Class.new do
+  def self.migration_id; 'test_migration_3'; end
+end
+
+## Registry class exists
+Familia::Migration::Registry.is_a?(Class)
+#=> true
+
+## Registry initializes with correct prefix
+@registry.prefix
+#=> @prefix
+
+## Registry initializes with provided redis client
+@registry.redis == @redis
+#=> true
+
+## client method returns the redis client
+@registry.client == @redis
+#=> true
+
+## applied? returns false for new migration
+@registry.applied?('test_migration_1')
+#=> false
+
+## record_applied adds migration to applied set
+@registry.record_applied('test_migration_1', { records_updated: 100 })
+@registry.applied?('test_migration_1')
+#=> true
+
+## applied_at returns Time when applied
+result = @registry.applied_at('test_migration_1')
+result.is_a?(Time) && (Time.now - result) < 5
+#=> true
+
+## applied_at returns nil when not applied
+@registry.applied_at('nonexistent_migration')
+#=> nil
+
+## all_applied returns array with applied migrations
+applied = @registry.all_applied
+applied.is_a?(Array) && applied.any? { |h| h[:migration_id] == 'test_migration_1' }
+#=> true
+
+## all_applied entry contains applied_at timestamp
+applied = @registry.all_applied
+entry = applied.find { |h| h[:migration_id] == 'test_migration_1' }
+entry[:applied_at].is_a?(Time)
+#=> true
+
+## metadata returns hash with correct status
+meta = @registry.metadata('test_migration_1')
+meta.is_a?(Hash) && meta[:status] == 'applied'
+#=> true
+
+## metadata contains keys_scanned from stats
+meta = @registry.metadata('test_migration_1')
+meta.key?(:keys_scanned)
+#=> true
+
+## metadata returns nil for unapplied migration
+@registry.metadata('nonexistent_migration')
+#=> nil
+
+## pending filters out applied migrations
+pending = @registry.pending([@applied_migration, @pending_migration])
+pending.map(&:migration_id)
+#=> ['test_migration_2']
+
+## pending returns empty array when all are applied
+@registry.record_applied('test_migration_2', {})
+pending = @registry.pending([@applied_migration, @pending_migration])
+pending
+#=> []
+
+## pending returns empty array for nil input
+@registry.pending(nil)
+#=> []
+
+## pending returns empty array for empty input
+@registry.pending([])
+#=> []
+
+## status returns combined info for all migrations
+status = @registry.status([@applied_migration, @pending_migration, @also_pending_migration])
+statuses = status.map { |s| [s[:migration_id], s[:status]] }.to_h
+[statuses['test_migration_1'], statuses['test_migration_3']]
+#=> [:applied, :pending]
+
+## status entry includes applied_at for applied migrations
+status = @registry.status([@applied_migration])
+entry = status.first
+entry[:applied_at].is_a?(Time)
+#=> true
+
+## status entry has nil applied_at for pending migrations
+status = @registry.status([@also_pending_migration])
+entry = status.first
+entry[:applied_at]
+#=> nil
+
+## status returns empty array for nil input
+@registry.status(nil)
+#=> []
+
+## record_rollback removes from applied set
+@registry.record_rollback('test_migration_1')
+@registry.applied?('test_migration_1')
+#=> false
+
+## record_rollback updates metadata to rolled_back status
+meta = @registry.metadata('test_migration_1')
+meta[:status]
+#=> 'rolled_back'
+
+## record_rollback adds rolled_back_at timestamp
+meta = @registry.metadata('test_migration_1')
+meta.key?(:rolled_back_at)
+#=> true
+
+## backup_field stores value in backup hash
+@registry.backup_field('backup_test', 'some:key', 'field1', 'original_value')
+backup_key = "#{@prefix}:backup:backup_test"
+@redis.hget(backup_key, 'some:key:field1')
+#=> 'original_value'
+
+## backup key has TTL set
+ttl = @redis.ttl("#{@prefix}:backup:backup_test")
+ttl > 0 && ttl <= Familia::Migration.config.backup_ttl
+#=> true
+
+## restore_backup returns count of restored fields
+@redis.del('some:key')
+@redis.hset('some:key', 'field1', 'modified_value')
+count = @registry.restore_backup('backup_test')
+count >= 1
+#=> true
+
+## restore_backup restores correct value
+@redis.hget('some:key', 'field1')
+#=> 'original_value'
+
+## restore_backup returns 0 when no backup exists
+@registry.restore_backup('nonexistent_backup')
+#=> 0
+
+## clear_backup removes backup data
+@registry.clear_backup('backup_test')
+@redis.exists("#{@prefix}:backup:backup_test")
+#=> 0
+
+## record_applied works with class that has migration_id
+@class_migration = Class.new do
+  def self.migration_id; 'class_based_migration'; end
+end
+@registry.record_applied(@class_migration, {})
+@registry.applied?('class_based_migration')
+#=> true
+
+## record_applied works with instance whose class has migration_id
+@instance_migration_class = Class.new do
+  def self.migration_id; 'instance_based_migration'; end
+end
+instance = @instance_migration_class.new
+@registry.record_applied(instance, { duration_ms: 150 })
+@registry.applied?('instance_based_migration')
+#=> true
+
+## record_applied stores duration_ms in metadata
+meta = @registry.metadata('instance_based_migration')
+meta[:duration_ms]
+#=> 150
+
+# Teardown
+@redis.keys("#{@prefix}:*").each { |k| @redis.del(k) }
+@redis.del('some:key')