familia 2.0.0.pre3 → 2.0.0.pre5

data/try/memory/memory_detailed_test.rb

@@ -0,0 +1,121 @@
+# try/edge_cases/memory_detailed_test_try.rb
+
+require 'objspace'
+require 'json'
+
+require_relative '../helpers/test_helpers'
+
+class DetailedMemoryTester
+  def self.test_with_details
+    ObjectSpace.trace_object_allocations_start
+
+    secret = "SENSITIVE_#{rand(999999)}_DATA"
+    puts "Testing with secret: #{secret}"
+    puts "Secret object_id: #{secret.object_id}"
+    puts "Secret frozen?: #{secret.frozen?}\n\n"
+
+    # Track all string copies
+    tracker = {}
+
+    # Before creating RedactedString
+    find_secret_copies(secret, "BEFORE RedactedString creation", tracker)
+
+    # Create RedactedString
+    redacted = RedactedString.new(secret)
+    find_secret_copies(secret, "AFTER RedactedString creation", tracker)
+
+    # Use expose block
+    exposed_value = nil
+    redacted.expose do |plain|
+      exposed_value = plain.object_id
+      find_secret_copies(secret, "DURING expose block", tracker)
+    end
+    find_secret_copies(secret, "AFTER expose block", tracker)
+
+    # Clear and GC
+    redacted.clear!
+    original_secret = secret
+    secret = nil  # Remove our reference
+    GC.start(full_mark: true, immediate_sweep: true)
+
+    find_secret_copies(original_secret, "AFTER clear! and GC", tracker)
+
+    # Final report
+    puts "\n" + "="*60
+    puts "FINAL ANALYSIS"
+    puts "="*60
+
+    remaining_copies = []
+    ObjectSpace.each_object(String) do |str|
+      begin
+        if str.include?(original_secret)
+          remaining_copies << {
+            object_id: str.object_id,
+            size: str.bytesize,
+            encoding: str.encoding.name,
+            frozen: str.frozen?,
+            tainted: (str.tainted? rescue "N/A"),
+            value_preview: str[0..50]
+          }
+        end
+      rescue => e
+        # Skip strings that can't be accessed
+      end
+    end
+
+    if remaining_copies.empty?
+      puts "✅ SUCCESS: No copies found in memory!"
+    else
+      puts "❌ FAILURE: #{remaining_copies.size} copies still in memory:"
+      remaining_copies.each do |copy|
+        puts "\n  Object ID: #{copy[:object_id]}"
+        puts "  Size: #{copy[:size]} bytes"
+        puts "  Frozen: #{copy[:frozen]}"
+        puts "  Encoding: #{copy[:encoding]}"
+      end
+    end
+
+    # Show memory stats
+    puts "\n" + "="*60
+    puts "MEMORY STATISTICS"
+    puts "="*60
+    puts "Total strings in ObjectSpace: #{ObjectSpace.each_object(String).count}"
+    puts "GC count: #{GC.count}"
+    puts "GC stat: #{GC.stat[:heap_live_slots]} live slots"
+
+    tracker
+  end
+
+  private
+
+  def self.find_secret_copies(secret, phase, tracker)
+    copies = []
+
+    ObjectSpace.each_object(String) do |str|
+      begin
+        if str.include?(secret)
+          copies << {
+            object_id: str.object_id,
+            frozen: str.frozen?,
+            source: ObjectSpace.allocation_sourcefile(str),
+            line: ObjectSpace.allocation_sourceline(str)
+          }
+        end
+      rescue => e
+        # Some strings might not be accessible
+      end
+    end
+
+    tracker[phase] = copies
+
+    puts "#{phase}: Found #{copies.size} copies"
+    copies.each do |copy|
+      source_info = copy[:source] ? "#{copy[:source]}:#{copy[:line]}" : "unknown source"
+      puts "  - Object #{copy[:object_id]} (frozen: #{copy[:frozen]}) from #{source_info}"
+    end
+    puts ""
+  end
+end
+
+# Run the detailed test
+DetailedMemoryTester.test_with_details

data/try/memory/memory_docker_ruby_dump.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# try/edge_cases/docker_dump.sh
+
+# Usage: bash $0 <container_id>
+#
+# See example output at end.
+
+# Set CONTAINER_ID to $CONTAINER_ID or the first argument
+CONTAINER_ID=${CONTAINER_ID:-$1}
+
+if [ -z "$CONTAINER_ID" ]; then
+  echo "Usage: $0 <container_id>"
+  echo "Or set CONTAINER_ID environment variable"
+  exit 1
+fi
+
+# Create a script to dump all string-like patterns
+docker exec $CONTAINER_ID bash -c '
+  # Install required packages
+  apt-get update -qq && apt-get install -y -qq procps binutils
+
+  PID=$(pgrep -f ruby)
+
+  if [ -z "$PID" ]; then
+    echo "No Ruby process found"
+    exit 1
+  fi
+
+  echo "Dumping memory for Ruby process $PID"
+
+  # Check if maps file exists
+  if [ ! -f "/proc/$PID/maps" ]; then
+    echo "Cannot access memory maps for process $PID"
+    exit 1
+  fi
+
+  # Get memory regions
+  grep -E "rw-p|r--p" /proc/$PID/maps | while read line; do
+    start=$(echo $line | cut -d"-" -f1)
+    end=$(echo $line | cut -d" " -f1 | cut -d"-" -f2)
+
+    # Convert hex to decimal and dump
+    start_dec=$((16#$start))
+    end_dec=$((16#$end))
+    size=$((end_dec - start_dec))
+
+    # Skip if size is too large (> 10MB) to avoid hanging
+    if [ $size -gt 10485760 ]; then
+      continue
+    fi
+
+    dd if=/proc/$PID/mem bs=1 skip=$start_dec count=$size 2>/dev/null
+  done | strings | grep -i "secret\|api\|key\|token" | head -20
+'
+
+# Example Output:
+#
+#   $ SECRET=august7th2025
+#   $
+#   $ docker run --rm -d -p 3000:3000 \
+#     -e SECRET=$SECRET \
+#     -e REDIS_URL=redis://host.docker.internal:6379/0 \
+#     ghcr.io/onetimesecret/devtimesecret-lite:latest
+#
+#     abcd1234
+#
+#   $ bash try/edge_cases/docker_ruby_dump.sh abcd1234
+#   ...
+#   Dumping memory for Ruby process 60
+#   SECRET
+#   SECRET
+#   SECRET=august6th2025
+#     done | strings | grep -i "secret...
+#   SECRET=august6th2025
+#     done | strings | grep -i "secret...
+#   grep -i "secret\|api\|key|token"
+#     done | strings | grep -i "secret...
+#   SECRET=august6th2025
+#
+#   $ docker kill abcd1234

data/try/memory/memory_search_for_string.rb

@@ -0,0 +1,83 @@
+# try/edge_cases/search_memory_for_string_try.rb
+
+require 'objspace'
+
+require_relative '../helpers/test_helpers'
+
+# Enable object space tracking
+ObjectSpace.trace_object_allocations_start
+
+def search_memory_for_string(target)
+  found_locations = []
+
+  ObjectSpace.each_object(String) do |str|
+    begin
+      if str.include?(target)
+        found_locations << {
+          value: str[0..100], # First 100 chars
+          object_id: str.object_id,
+          source: ObjectSpace.allocation_sourcefile(str),
+          line: ObjectSpace.allocation_sourceline(str),
+          frozen: str.frozen?
+        }
+      end
+    rescue => e
+      # Some strings might not be accessible
+    end
+  end
+
+  found_locations
+end
+
+# Test scenario
+secret = "SUPER_SECRET_API_KEY_12345"
+puts "Testing with secret: #{secret}"
+
+# Create RedactedString
+redacted = RedactedString.new(secret)
+puts "Created RedactedString"
+
+# Force GC to see if copies persist
+GC.start(full_mark: true, immediate_sweep: true)
+
+# Search memory
+puts "\n=== Memory search BEFORE expose ==="
+found = search_memory_for_string("SUPER_SECRET_API_KEY")
+found.each do |location|
+  puts "Found at object_id: #{location[:object_id]}"
+  puts "  Value: #{location[:value]}"
+  puts "  Source: #{location[:source]}:#{location[:line]}"
+  puts "  Frozen: #{location[:frozen]}"
+end
+
+# Use expose
+redacted.expose do |plain|
+  puts "\nInside expose block, plain = [REDACTED for display]"
+
+  # Search during expose
+  puts "\n=== Memory search DURING expose ==="
+  found = search_memory_for_string("SUPER_SECRET_API_KEY")
+  puts "Found #{found.size} instances"
+end
+
+# After expose
+GC.start(full_mark: true, immediate_sweep: true)
+puts "\n=== Memory search AFTER expose ==="
+found = search_memory_for_string("SUPER_SECRET_API_KEY")
+found.each do |location|
+  puts "Found at object_id: #{location[:object_id]}"
+  puts "  Value: #{location[:value]}"
+end
+
+# Also check with marshal dump
+puts "\n=== Checking Marshal dump ==="
+begin
+  marshaled = Marshal.dump(ObjectSpace.each_object.to_a)
+  if marshaled.include?("SUPER_SECRET_API_KEY")
+    puts "❌ Secret found in Marshal dump!"
+  else
+    puts "✅ Secret not found in Marshal dump"
+  end
+rescue => e
+  puts "Marshal failed: #{e}"
+end

data/try/memory/test_actual_redactedstring_protection.rb

@@ -0,0 +1,38 @@
+# try/memory/test_actual_redactedstring_protection.rb
+
+require_relative '../helpers/test_helpers'
+
+# Test 1: Does it prevent logging leaks?
+secret = "API_KEY_12345"
+redacted = RedactedString.new(secret)
+
+puts "Logging test:"
+puts "Normal string logs as: #{secret}"  # Shows: API_KEY_12345
+puts "Redacted string logs as: #{redacted}"  # Shows: [REDACTED]
+puts "✅ Logging protection works!\n\n"
+
+# Test 2: Does it prevent exception leaks?
+begin
+  raise StandardError, "Error with secret: #{redacted}"
+rescue => e
+  puts "Exception message: #{e.message}"
+  puts "✅ Exception protection works!\n\n" if e.message.include?("[REDACTED]")
+end
+
+# Test 3: Does it prevent debug leaks?
+require 'pp'
+data = {
+  user: "john",
+  token: redacted
+}
+puts "Debug output:"
+pp data  # Will show token: [REDACTED]
+puts "✅ Debug protection works!\n\n"
+
+# Test 4: Real-world usage pattern
+redacted.expose do |token|
+  # Simulate API call
+  puts "Making API call with token (simulated)"
+  # HTTParty.get("https://api.example.com", headers: { "Authorization" => token })
+end
+puts "After API call, trying to access: #{redacted}"  # Still shows [REDACTED]

data/try/models/customer_safe_dump_try.rb

@@ -1,6 +1,5 @@
 # try/models/customer_safedump_try.rb
 
-require_relative '../../lib/familia'
 require_relative '../helpers/test_helpers'
 
 # Setup

data/try/models/customer_try.rb

@@ -1,7 +1,6 @@
 # try/models/customer_try.rb
 
 # Customer Tryouts
-require_relative '../../lib/familia'
 require_relative '../helpers/test_helpers'
 
 # Setup

data/try/models/datatype_base_try.rb

@@ -1,8 +1,7 @@
-# try/datatypes/datatype_base_try.rb
+# try/data_types/data_type_base_try.rb
 
 # Test DataType base functionality
 
-require_relative '../../lib/familia'
 require_relative '../helpers/test_helpers'
 
 Familia.debug = false

data/try/models/familia_object_try.rb

@@ -1,6 +1,5 @@
 # try/models/familia_object_try.rb
 
-require_relative '../../lib/familia'
 require_relative '../helpers/test_helpers'
 
 Familia.debug = false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: familia
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre3
+  version: 2.0.0.pre5
 platform: ruby
 authors:
 - Delano Mandelbaum
@@ -132,31 +132,54 @@ files:
 - Gemfile.lock
 - LICENSE.txt
 - README.md
+- TEST_COVERAGE.md
 - bin/irb
 - docs/connection_pooling.md
+- docs/overview.md
+- docs/wiki/API-Reference.md
+- docs/wiki/Encrypted-Fields-Overview.md
+- docs/wiki/Home.md
+- docs/wiki/Implementation-Guide.md
+- docs/wiki/Security-Model.md
 - familia.gemspec
 - lib/familia.rb
 - lib/familia/base.rb
 - lib/familia/connection.rb
 - lib/familia/core_ext.rb
-- lib/familia/datatype.rb
-- lib/familia/datatype/commands.rb
-- lib/familia/datatype/serialization.rb
-- lib/familia/datatype/types/hashkey.rb
-- lib/familia/datatype/types/list.rb
-- lib/familia/datatype/types/sorted_set.rb
-- lib/familia/datatype/types/string.rb
-- lib/familia/datatype/types/unsorted_set.rb
+- lib/familia/data_type.rb
+- lib/familia/data_type/commands.rb
+- lib/familia/data_type/serialization.rb
+- lib/familia/data_type/types/hashkey.rb
+- lib/familia/data_type/types/list.rb
+- lib/familia/data_type/types/sorted_set.rb
+- lib/familia/data_type/types/string.rb
+- lib/familia/data_type/types/unsorted_set.rb
+- lib/familia/encryption.rb
+- lib/familia/encryption/manager.rb
+- lib/familia/encryption/provider.rb
+- lib/familia/encryption/providers/aes_gcm_provider.rb
+- lib/familia/encryption/providers/secure_xchacha20_poly1305_provider.rb
+- lib/familia/encryption/providers/xchacha20_poly1305_provider.rb
+- lib/familia/encryption/registry.rb
+- lib/familia/encryption_request_cache.rb
 - lib/familia/errors.rb
 - lib/familia/features.rb
+- lib/familia/features/encrypted_fields.rb
+- lib/familia/features/encrypted_fields/encrypted_field_type.rb
 - lib/familia/features/expiration.rb
 - lib/familia/features/quantization.rb
 - lib/familia/features/relatable_objects.rb
 - lib/familia/features/safe_dump.rb
+- lib/familia/features/transient_fields.rb
+- lib/familia/features/transient_fields/redacted_string.rb
+- lib/familia/features/transient_fields/single_use_redacted_string.rb
+- lib/familia/features/transient_fields/transient_field_type.rb
+- lib/familia/field_type.rb
 - lib/familia/horreum.rb
-- lib/familia/horreum/class_methods.rb
-- lib/familia/horreum/commands.rb
 - lib/familia/horreum/connection.rb
+- lib/familia/horreum/database_commands.rb
+- lib/familia/horreum/definition_methods.rb
+- lib/familia/horreum/management_methods.rb
 - lib/familia/horreum/related_fields_management.rb
 - lib/familia/horreum/serialization.rb
 - lib/familia/horreum/settings.rb
@@ -170,6 +193,7 @@ files:
 - lib/familia/version.rb
 - lib/middleware/database_middleware.rb
 - try/configuration/scenarios_try.rb
+- try/core/base_enhancements_try.rb
 - try/core/connection_try.rb
 - try/core/errors_try.rb
 - try/core/extensions_try.rb
@@ -181,13 +205,17 @@ files:
 - try/core/settings_try.rb
 - try/core/tools_try.rb
 - try/core/utils_try.rb
-- try/datatypes/boolean_try.rb
-- try/datatypes/datatype_base_try.rb
-- try/datatypes/hash_try.rb
-- try/datatypes/list_try.rb
-- try/datatypes/set_try.rb
-- try/datatypes/sorted_set_try.rb
-- try/datatypes/string_try.rb
+- try/data_types/boolean_try.rb
+- try/data_types/datatype_base_try.rb
+- try/data_types/hash_try.rb
+- try/data_types/list_try.rb
+- try/data_types/set_try.rb
+- try/data_types/sorted_set_try.rb
+- try/data_types/string_try.rb
+- try/debugging/README.md
+- try/debugging/cache_behavior_tracer.rb
+- try/debugging/encryption_method_tracer.rb
+- try/debugging/provider_diagnostics.rb
 - try/edge_cases/empty_identifiers_try.rb
 - try/edge_cases/hash_symbolization_try.rb
 - try/edge_cases/json_serialization_try.rb
@@ -195,20 +223,59 @@ files:
 - try/edge_cases/reserved_keywords_try.rb
 - try/edge_cases/string_coercion_try.rb
 - try/edge_cases/ttl_side_effects_try.rb
+- try/encryption/config_persistence_try.rb
+- try/encryption/encryption_core_try.rb
+- try/encryption/instance_variable_scope_try.rb
+- try/encryption/module_loading_try.rb
+- try/encryption/providers/aes_gcm_provider_try.rb
+- try/encryption/providers/xchacha20_poly1305_provider_try.rb
+- try/encryption/roundtrip_validation_try.rb
+- try/encryption/secure_memory_handling_try.rb
+- try/features/encrypted_fields_core_try.rb
+- try/features/encrypted_fields_integration_try.rb
+- try/features/encrypted_fields_no_cache_security_try.rb
+- try/features/encrypted_fields_security_try.rb
+- try/features/encryption_fields/aad_protection_try.rb
+- try/features/encryption_fields/context_isolation_try.rb
+- try/features/encryption_fields/error_conditions_try.rb
+- try/features/encryption_fields/fresh_key_derivation_try.rb
+- try/features/encryption_fields/fresh_key_try.rb
+- try/features/encryption_fields/key_rotation_try.rb
+- try/features/encryption_fields/memory_security_try.rb
+- try/features/encryption_fields/missing_current_key_version_try.rb
+- try/features/encryption_fields/nonce_uniqueness_try.rb
+- try/features/encryption_fields/thread_safety_try.rb
 - try/features/expiration_try.rb
+- try/features/feature_dependencies_try.rb
 - try/features/quantization_try.rb
+- try/features/real_feature_integration_try.rb
 - try/features/relatable_objects_try.rb
 - try/features/safe_dump_advanced_try.rb
 - try/features/safe_dump_try.rb
+- try/features/transient_fields/redacted_string_try.rb
+- try/features/transient_fields/refresh_reset_try.rb
+- try/features/transient_fields/simple_refresh_test.rb
+- try/features/transient_fields/single_use_redacted_string_try.rb
+- try/features/transient_fields_core_try.rb
+- try/features/transient_fields_integration_try.rb
 - try/helpers/test_helpers.rb
 - try/horreum/base_try.rb
 - try/horreum/class_methods_try.rb
 - try/horreum/commands_try.rb
+- try/horreum/enhanced_conflict_handling_try.rb
+- try/horreum/field_categories_try.rb
+- try/horreum/field_definition_try.rb
 - try/horreum/initialization_try.rb
 - try/horreum/relations_try.rb
+- try/horreum/serialization_persistent_fields_try.rb
 - try/horreum/serialization_try.rb
 - try/horreum/settings_try.rb
 - try/integration/cross_component_try.rb
+- try/memory/memory_basic_test.rb
+- try/memory/memory_detailed_test.rb
+- try/memory/memory_docker_ruby_dump.sh
+- try/memory/memory_search_for_string.rb
+- try/memory/test_actual_redactedstring_protection.rb
 - try/models/customer_safe_dump_try.rb
 - try/models/customer_try.rb
 - try/models/datatype_base_try.rb