vibe_zstd 1.0.0

data/benchmark/dictionary_training.rb

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "helpers"
+
+# Benchmark: Dictionary Training Algorithms
+# Compares train_dict, train_dict_cover, and train_dict_fast_cover
+
+BenchmarkHelpers.run_comparison(title: "Dictionary Training Algorithm Comparison") do |results|
+  # Generate training samples
+  puts "Generating training samples..."
+  samples = 200.times.map do |i|
+    {
+      id: i,
+      name: "User #{i}",
+      email: "user#{i}@example.com",
+      created_at: Time.now.to_i - rand(100000),
+      status: %w[active pending inactive verified][rand(4)],
+      preferences: {
+        theme: %w[light dark auto][rand(3)],
+        notifications: rand(2) == 1
+      }
+    }.to_json
+  end
+
+  puts "Training samples: #{samples.size}"
+  puts "Average sample size: #{Formatter.format_bytes(samples.map(&:bytesize).sum / samples.size)}"
+  puts "Total training data: #{Formatter.format_bytes(samples.map(&:bytesize).sum)}\n\n"
+
+  dict_size = 16 * 1024 # 16KB
+
+  # Benchmark 1: train_dict (default algorithm - fast)
+  Formatter.section("Testing: train_dict (default/fast algorithm)")
+  dict_fast = nil
+  fast_time = Benchmark.measure do
+    dict_fast = VibeZstd.train_dict(samples, max_dict_size: dict_size)
+  end
+
+  puts "Training time: #{fast_time.real.round(3)}s"
+  puts "Dictionary size: #{Formatter.format_bytes(dict_fast.bytesize)}"
+  puts "Dictionary ID: #{VibeZstd.get_dict_id(dict_fast)}"
+
+  # Test compression effectiveness
+  test_sample = samples[100]
+  cdict_fast = VibeZstd::CDict.new(dict_fast)
+  compressed_fast = VibeZstd.compress(test_sample, dict: cdict_fast)
+  ratio_fast = test_sample.bytesize.to_f / compressed_fast.bytesize
+
+  results << BenchmarkResult.new(
+    :name => "train_dict",
+    :compression_ratio => ratio_fast,
+    :memory_bytes => dict_fast.bytesize,
+    "Training time" => "#{fast_time.real.round(3)}s",
+    "Compressed" => Formatter.format_bytes(compressed_fast.bytesize)
+  )
+
+  # Benchmark 2: train_dict_cover (COVER algorithm - better quality)
+  Formatter.section("Testing: train_dict_cover (COVER algorithm)")
+  dict_cover = nil
+  cover_time = Benchmark.measure do
+    dict_cover = VibeZstd.train_dict_cover(
+      samples,
+      max_dict_size: dict_size,
+      k: 200,  # Segment size
+      d: 6     # Dmer size
+    )
+  end
+
+  puts "Training time: #{cover_time.real.round(3)}s"
+  puts "Dictionary size: #{Formatter.format_bytes(dict_cover.bytesize)}"
+  puts "Dictionary ID: #{VibeZstd.get_dict_id(dict_cover)}"
+
+  cdict_cover = VibeZstd::CDict.new(dict_cover)
+  compressed_cover = VibeZstd.compress(test_sample, dict: cdict_cover)
+  ratio_cover = test_sample.bytesize.to_f / compressed_cover.bytesize
+
+  results << BenchmarkResult.new(
+    :name => "train_dict_cover",
+    :compression_ratio => ratio_cover,
+    :memory_bytes => dict_cover.bytesize,
+    "Training time" => "#{cover_time.real.round(3)}s",
+    "Compressed" => Formatter.format_bytes(compressed_cover.bytesize)
+  )
+
+  # Benchmark 3: train_dict_fast_cover (fast COVER - balanced)
+  Formatter.section("Testing: train_dict_fast_cover (fast COVER)")
+  dict_fast_cover = nil
+  fast_cover_time = Benchmark.measure do
+    dict_fast_cover = VibeZstd.train_dict_fast_cover(
+      samples,
+      max_dict_size: dict_size,
+      k: 200,
+      d: 6,
+      f: 20,   # Frequency array size
+      accel: 5 # Acceleration (1-10, higher = faster)
+    )
+  end
+
+  puts "Training time: #{fast_cover_time.real.round(3)}s"
+  puts "Dictionary size: #{Formatter.format_bytes(dict_fast_cover.bytesize)}"
+  puts "Dictionary ID: #{VibeZstd.get_dict_id(dict_fast_cover)}"
+
+  cdict_fast_cover = VibeZstd::CDict.new(dict_fast_cover)
+  compressed_fast_cover = VibeZstd.compress(test_sample, dict: cdict_fast_cover)
+  ratio_fast_cover = test_sample.bytesize.to_f / compressed_fast_cover.bytesize
+
+  results << BenchmarkResult.new(
+    :name => "train_dict_fast_cover",
+    :compression_ratio => ratio_fast_cover,
+    :memory_bytes => dict_fast_cover.bytesize,
+    "Training time" => "#{fast_cover_time.real.round(3)}s",
+    "Compressed" => Formatter.format_bytes(compressed_fast_cover.bytesize)
+  )
+
+  # Compare compression across multiple samples
+  puts "\n"
+  Formatter.section("Compression effectiveness across test samples")
+
+  test_samples = (101..110).map { |i| samples[i] }
+
+  [:fast, :cover, :fast_cover].each do |dict_type|
+    dict_data = case dict_type
+    when :fast then dict_fast
+    when :cover then dict_cover
+    when :fast_cover then dict_fast_cover
+    end
+
+    cdict = VibeZstd::CDict.new(dict_data)
+    total_original = 0
+    total_compressed = 0
+
+    test_samples.each do |sample|
+      total_original += sample.bytesize
+      compressed = VibeZstd.compress(sample, dict: cdict)
+      total_compressed += compressed.bytesize
+    end
+
+    avg_ratio = total_original.to_f / total_compressed
+    puts "  #{dict_type}: #{Formatter.format_ratio(avg_ratio)} average ratio"
+  end
+
+  # Test dictionary sizes
+  puts "\n"
+  Formatter.section("Dictionary size impact")
+
+  sizes = [4096, 8192, 16384, 32768]
+  size_results = []
+
+  sizes.each do |size|
+    dict = VibeZstd.train_dict(samples, max_dict_size: size)
+    cdict = VibeZstd::CDict.new(dict)
+    compressed = VibeZstd.compress(test_sample, dict: cdict)
+    ratio = test_sample.bytesize.to_f / compressed.bytesize
+
+    size_results << {
+      "Dict Size" => Formatter.format_bytes(size),
+      "Actual Size" => Formatter.format_bytes(dict.bytesize),
+      "Ratio" => Formatter.format_ratio(ratio),
+      "Compressed" => Formatter.format_bytes(compressed.bytesize)
+    }
+  end
+
+  Formatter.table(size_results)
+end
+
+puts "\n💡 Dictionary Training Recommendations:"
+puts "  train_dict (default):"
+puts "    ✓ Fastest training"
+puts "    ✓ Good enough for most use cases"
+puts "    ✓ Use when training time matters"
+puts "\n  train_dict_cover:"
+puts "    ✓ Best compression ratios"
+puts "    ✓ Slower training (2-10x slower)"
+puts "    ✓ Use for production dictionaries"
+puts "\n  train_dict_fast_cover:"
+puts "    ✓ Balanced speed/quality"
+puts "    ✓ Configurable with accel parameter"
+puts "    ✓ Good default for most users"
+puts "\n  Dictionary size:"
+puts "    - Larger = better compression (diminishing returns > 64KB)"
+puts "    - Typical: 16KB-64KB for small messages"
+puts "    - Memory overhead: ~2x dictionary size in memory"

data/benchmark/dictionary_usage.rb

@@ -0,0 +1,121 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "helpers"
+
+# Benchmark: Dictionary Usage Performance
+# Demonstrates compression ratio and speed improvements when using trained dictionaries
+
+BenchmarkHelpers.run_comparison(title: "Dictionary Usage Performance Comparison") do |results|
+  # Load the trained dictionary
+  dict_path = File.join(__dir__, "..", "test", "fixtures", "sample.dict")
+  unless File.exist?(dict_path)
+    puts "⚠️  Dictionary fixture not found. Run: ruby benchmark/generate_fixture.rb"
+    exit 1
+  end
+
+  dict_data = File.binread(dict_path)
+  cdict = VibeZstd::CDict.new(dict_data)
+  ddict = VibeZstd::DDict.new(dict_data)
+
+  puts "Dictionary size: #{Formatter.format_bytes(dict_data.bytesize)}"
+  puts "Dictionary ID: #{VibeZstd.get_dict_id(dict_data)}\n\n"
+
+  # Generate test data (similar to training data for best dictionary performance)
+  test_samples = 100.times.map do |i|
+    {
+      id: i + 1000,
+      name: "User #{i + 1000}",
+      email: "user#{i + 1000}@example.com",
+      created_at: Time.now.to_i,
+      status: %w[active pending inactive verified][rand(4)],
+      preferences: {
+        theme: %w[light dark auto][rand(3)],
+        notifications: rand(2) == 1,
+        language: %w[en es fr de][rand(4)]
+      },
+      metadata: {
+        login_count: rand(1000),
+        last_ip: "10.0.#{rand(255)}.#{rand(255)}",
+        session_duration: rand(3600)
+      }
+    }.to_json
+  end
+
+  # Benchmark without dictionary
+  Formatter.section("Testing: Compression without dictionary")
+  cctx = VibeZstd::CCtx.new
+  dctx = VibeZstd::DCtx.new
+
+  compressed_sizes_no_dict = []
+  no_dict_time = Benchmark.measure do
+    test_samples.each do |sample|
+      compressed = cctx.compress(sample)
+      compressed_sizes_no_dict << compressed.bytesize
+      dctx.decompress(compressed)
+    end
+  end
+
+  no_dict_ops_per_sec = test_samples.size / no_dict_time.real
+  avg_compressed_no_dict = compressed_sizes_no_dict.sum / compressed_sizes_no_dict.size.to_f
+  puts "Completed #{test_samples.size} operations in #{no_dict_time.real.round(3)}s"
+
+  # Benchmark with dictionary
+  Formatter.section("Testing: Compression with dictionary")
+
+  compressed_sizes_with_dict = []
+  with_dict_time = Benchmark.measure do
+    test_samples.each do |sample|
+      compressed = cctx.compress(sample, dict: cdict)
+      compressed_sizes_with_dict << compressed.bytesize
+      dctx.decompress(compressed, dict: ddict)
+    end
+  end
+
+  with_dict_ops_per_sec = test_samples.size / with_dict_time.real
+  avg_compressed_with_dict = compressed_sizes_with_dict.sum / compressed_sizes_with_dict.size.to_f
+  puts "Completed #{test_samples.size} operations in #{with_dict_time.real.round(3)}s"
+
+  # Calculate compression ratios
+  avg_original_size = test_samples.map(&:bytesize).sum / test_samples.size.to_f
+  compression_ratio_no_dict = avg_original_size / avg_compressed_no_dict
+  compression_ratio_with_dict = avg_original_size / avg_compressed_with_dict
+
+  # Memory estimates
+  cdict_memory = Memory.estimate_cdict(dict_data.bytesize)
+  ddict_memory = Memory.estimate_ddict(dict_data.bytesize)
+  dict_memory_overhead = cdict_memory + ddict_memory
+
+  # Collect results
+  results << BenchmarkResult.new(
+    :name => "Without dictionary",
+    :iterations_per_sec => no_dict_ops_per_sec,
+    :compression_ratio => compression_ratio_no_dict,
+    :memory_bytes => Memory.estimate_cctx + Memory.estimate_dctx,
+    "Avg compressed size" => Formatter.format_bytes(avg_compressed_no_dict.to_i)
+  )
+
+  results << BenchmarkResult.new(
+    :name => "With dictionary",
+    :iterations_per_sec => with_dict_ops_per_sec,
+    :compression_ratio => compression_ratio_with_dict,
+    :memory_bytes => Memory.estimate_cctx + Memory.estimate_dctx + dict_memory_overhead,
+    "Avg compressed size" => Formatter.format_bytes(avg_compressed_with_dict.to_i)
+  )
+
+  puts "\n📊 Detailed Statistics:"
+  puts "  Average original size: #{Formatter.format_bytes(avg_original_size.to_i)}"
+  puts "  Average compressed (no dict): #{Formatter.format_bytes(avg_compressed_no_dict.to_i)}"
+  puts "  Average compressed (with dict): #{Formatter.format_bytes(avg_compressed_with_dict.to_i)}"
+  puts "  Compression improvement: #{((avg_compressed_no_dict - avg_compressed_with_dict) / avg_compressed_no_dict * 100).round(1)}% smaller"
+  puts "\n💾 Memory Overhead:"
+  puts "  Dictionary in memory: #{Formatter.format_bytes(dict_memory_overhead)}"
+  puts "  Dictionary on disk: #{Formatter.format_bytes(dict_data.bytesize)}"
+end
+
+puts "\n💡 When to use dictionaries:"
+puts "  ✓ Small, similar data (JSON, logs, etc.)"
+puts "  ✓ Many small messages with repeated patterns"
+puts "  ✓ When compression ratio is more important than speed"
+puts "  ✗ Large files (> 1MB each)"
+puts "  ✗ Highly variable data with no patterns"

data/benchmark/for_readme.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Quick benchmark script to generate results for README
+
+require_relative "helpers"
+
+puts "# Performance Benchmarks\n\n"
+puts "Results from Ruby #{RUBY_VERSION} on #{RUBY_PLATFORM}, Zstd #{VibeZstd.version_string}\n\n"
+
+# 1. Context Reuse
+puts "## Context Reuse Performance\n\n"
+puts "Reusing compression/decompression contexts vs creating new ones (5000 iterations each):\n\n"
+
+# Test with different data sizes
+test_cases = {
+  "1KB" => DataGenerator.json_data(count: 5),
+  "10KB" => DataGenerator.json_data(count: 50),
+  "100KB" => DataGenerator.json_data(count: 500)
+}
+
+puts "| Data Size | New Context | Reused Context | Speedup |"
+puts "|-----------|-------------|----------------|---------|"
+
+test_cases.each do |size_label, test_data|
+  iterations = 5000
+
+  new_ctx_time = Benchmark.measure do
+    iterations.times do
+      cctx = VibeZstd::CCtx.new
+      dctx = VibeZstd::DCtx.new
+      compressed = cctx.compress(test_data)
+      dctx.decompress(compressed)
+    end
+  end
+
+  reused_time = Benchmark.measure do
+    cctx = VibeZstd::CCtx.new
+    dctx = VibeZstd::DCtx.new
+    iterations.times do
+      compressed = cctx.compress(test_data)
+      dctx.decompress(compressed)
+    end
+  end
+
+  new_ops = iterations / new_ctx_time.real
+  reused_ops = iterations / reused_time.real
+  speedup = reused_ops / new_ops
+
+  puts "| #{size_label} | #{Formatter.format_number(new_ops.to_i)} ops/s | #{Formatter.format_number(reused_ops.to_i)} ops/s | #{speedup.round(2)}x |"
+end
+
+cctx_mem = Memory.estimate_cctx(3)
+dctx_mem = Memory.estimate_dctx
+total_mem = cctx_mem + dctx_mem
+
+puts "\n**Memory savings:** Reusing contexts saves #{Formatter.format_bytes(total_mem * 4999)} for 5000 operations (99.98% reduction)\n"
+puts "**Recommendation:** Always reuse CCtx/DCtx instances for multiple operations.\n\n"
+
+# 2. Dictionary Performance
+puts "## Dictionary Compression\n\n"
+puts "Compression with vs without trained dictionaries (100 JSON samples):\n\n"
+
+dict_path = File.join(__dir__, "..", "test", "fixtures", "sample.dict")
+dict_data = File.binread(dict_path)
+cdict = VibeZstd::CDict.new(dict_data)
+VibeZstd::DDict.new(dict_data)
+
+test_sample = {
+  id: 999,
+  name: "Test User",
+  email: "test@example.com",
+  status: "active",
+  preferences: {theme: "dark", notifications: true}
+}.to_json
+
+compressed_no_dict = VibeZstd.compress(test_sample)
+compressed_with_dict = VibeZstd.compress(test_sample, dict: cdict)
+
+ratio_no_dict = test_sample.bytesize.to_f / compressed_no_dict.bytesize
+ratio_with_dict = test_sample.bytesize.to_f / compressed_with_dict.bytesize
+
+puts "| Method | Compressed Size | Ratio | Improvement |"
+puts "|--------|----------------|-------|-------------|"
+puts "| Without dictionary | #{compressed_no_dict.bytesize}B | #{ratio_no_dict.round(2)}x | - |"
+puts "| With dictionary (16KB) | #{compressed_with_dict.bytesize}B | #{ratio_with_dict.round(2)}x | #{((compressed_no_dict.bytesize - compressed_with_dict.bytesize).to_f / compressed_no_dict.bytesize * 100).round(1)}% smaller |"
+puts "\nOriginal size: #{test_sample.bytesize} bytes\n\n"
+
+# 3. Compression Levels
+puts "## Compression Levels\n\n"
+puts "Speed vs compression ratio trade-offs:\n\n"
+
+large_data = DataGenerator.mixed_data(size: 50_000)
+levels = [-1, 1, 3, 9, 19]
+
+puts "| Level | Ratio | Speed (ops/sec) | Memory | Use Case |"
+puts "|-------|-------|-----------------|--------|----------|"
+
+levels.each do |level|
+  cctx = VibeZstd::CCtx.new
+
+  compressed = nil
+  time = Benchmark.measure do
+    10.times { compressed = cctx.compress(large_data, level: level) }
+  end
+
+  ops_per_sec = 10 / time.real
+  ratio = large_data.bytesize.to_f / compressed.bytesize
+  memory = Memory.estimate_cctx(level)
+
+  use_case = case level
+  when -1 then "Ultra-fast, real-time"
+  when 1 then "Fast, high-throughput"
+  when 3 then "Balanced (default)"
+  when 9 then "Better compression"
+  when 19 then "Maximum compression"
+  end
+
+  puts "| #{level} | #{ratio.round(2)}x | #{Formatter.format_number(ops_per_sec.to_i)} | #{Formatter.format_bytes(memory)} | #{use_case} |"
+end
+
+puts "\n"
+
+# 4. Multi-threading
+puts "## Multi-threading Performance\n\n"
+puts "Compression speedup with multiple workers (500KB data):\n\n"
+
+mt_data = DataGenerator.mixed_data(size: 500_000)
+
+puts "| Workers | Throughput | Speedup | Efficiency |"
+puts "|---------|------------|---------|------------|"
+
+baseline_throughput = nil
+
+[0, 2, 4].each do |workers|
+  cctx = VibeZstd::CCtx.new
+  cctx.nb_workers = workers if workers > 0
+
+  cctx.compress(mt_data) # warmup
+
+  time = Benchmark.measure do
+    5.times { cctx.compress(mt_data) }
+  end
+
+  throughput = (mt_data.bytesize * 5 / time.real)
+
+  if workers == 0
+    baseline_throughput = throughput
+    puts "| #{workers} (single) | #{Formatter.format_bytes(throughput.to_i)}/s | 1.0x | 100% |"
+  else
+    speedup = throughput / baseline_throughput
+    efficiency = (speedup / workers * 100).round(0)
+    puts "| #{workers} | #{Formatter.format_bytes(throughput.to_i)}/s | #{speedup.round(2)}x | #{efficiency}% |"
+  end
+end
+
+puts "\n**Note:** Multi-threading is most effective for data > 256KB. Diminishing returns after 4 workers.\n"

data/benchmark/generate_fixture.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Script to generate a trained dictionary fixture for testing and benchmarking
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "vibe_zstd"
+require "json"
+require "fileutils"
+
+# Create fixtures directory
+FIXTURES_DIR = File.join(__dir__, "..", "test", "fixtures")
+FileUtils.mkdir_p(FIXTURES_DIR)
+
+# Generate training samples (JSON-like data similar to web application data)
+puts "Generating training samples..."
+samples = 200.times.map do |i|
+  {
+    id: i,
+    name: "User #{i}",
+    email: "user#{i}@example.com",
+    created_at: Time.now.to_i - rand(100000),
+    status: %w[active pending inactive verified][rand(4)],
+    preferences: {
+      theme: %w[light dark auto][rand(3)],
+      notifications: rand(2) == 1,
+      language: %w[en es fr de][rand(4)]
+    },
+    metadata: {
+      login_count: rand(1000),
+      last_ip: "192.168.#{rand(255)}.#{rand(255)}",
+      user_agent: "Mozilla/5.0 (compatible; Bot/1.0)",
+      session_duration: rand(3600)
+    },
+    tags: rand(5).times.map { |t| "tag#{t}" }
+  }.to_json
+end
+
+puts "Training dictionary with #{samples.size} samples..."
+dict_data = VibeZstd.train_dict(samples, max_dict_size: 16 * 1024) # 16KB dictionary
+
+# Save the dictionary
+dict_path = File.join(FIXTURES_DIR, "sample.dict")
+File.binwrite(dict_path, dict_data)
+
+puts "\nDictionary saved to: #{dict_path}"
+puts "Dictionary size: #{dict_data.bytesize} bytes"
+puts "Dictionary ID: #{VibeZstd.get_dict_id(dict_data)}"
+
+# Test the dictionary
+puts "\nTesting dictionary effectiveness..."
+test_data = {
+  id: 999,
+  name: "Test User",
+  email: "test@example.com",
+  status: "active",
+  preferences: {theme: "dark", notifications: true, language: "en"},
+  metadata: {login_count: 42, last_ip: "192.168.1.1"}
+}.to_json
+
+# Compress without dictionary
+compressed_no_dict = VibeZstd.compress(test_data)
+
+# Compress with dictionary
+cdict = VibeZstd::CDict.new(dict_data)
+compressed_with_dict = VibeZstd.compress(test_data, dict: cdict)
+
+puts "\nCompression comparison:"
+puts "  Original size: #{test_data.bytesize} bytes"
+puts "  Without dictionary: #{compressed_no_dict.bytesize} bytes (#{(test_data.bytesize.to_f / compressed_no_dict.bytesize).round(2)}x)"
+puts "  With dictionary: #{compressed_with_dict.bytesize} bytes (#{(test_data.bytesize.to_f / compressed_with_dict.bytesize).round(2)}x)"
+puts "  Dictionary improvement: #{((compressed_no_dict.bytesize - compressed_with_dict.bytesize).to_f / compressed_no_dict.bytesize * 100).round(1)}% smaller"
+
+# Verify decompression works
+ddict = VibeZstd::DDict.new(dict_data)
+decompressed = VibeZstd.decompress(compressed_with_dict, dict: ddict)
+if decompressed == test_data
+  puts "\n✓ Dictionary verification successful!"
+else
+  puts "\n✗ Dictionary verification FAILED!"
+  exit 1
+end