vibe_zstd 1.0.0

data/benchmark/helpers.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "vibe_zstd"
+require "json"
+require "benchmark" # Built-in Ruby module
+
+# Try to load optional dependencies (gracefully handle missing gems)
+HAS_BENCHMARK_DEPS = begin
+  # Don't use bundler/setup - it's too strict. Just try to require the gems.
+  require "benchmark/ips"
+  require "terminal-table"
+  true
+rescue LoadError
+  puts "⚠️  Optional benchmark dependencies not installed."
+  puts "    Run: bundle install"
+  puts "    Continuing with basic benchmark output...\n\n"
+  false
+end
+
+module BenchmarkHelpers
+  # Data generators for realistic test scenarios
+  module DataGenerator
+    # Generate JSON-like data (common in web apps)
+    def self.json_data(count: 100)
+      records = count.times.map do |i|
+        {
+          id: i,
+          name: "User #{i}",
+          email: "user#{i}@example.com",
+          created_at: Time.now.to_i - rand(100000),
+          status: %w[active pending inactive][rand(3)],
+          metadata: {
+            login_count: rand(1000),
+            last_ip: "192.168.1.#{rand(255)}"
+          }
+        }
+      end
+      records.map(&:to_json).join("\n")
+    end
+
+    # Generate log-like data (common in logging scenarios)
+    def self.log_data(count: 100)
+      levels = %w[INFO WARN ERROR DEBUG]
+      messages = [
+        "Request processed successfully",
+        "Database query took %dms",
+        "Cache miss for key: user_%d",
+        "Connection established to %s",
+        "Background job completed"
+      ]
+
+      count.times.map do |i|
+        level = levels[rand(levels.length)]
+        message = messages[rand(messages.length)]
+        message = format(message, rand(1000)) if message.include?("%")
+        timestamp = Time.now.to_i - rand(100000)
+        "[#{timestamp}] #{level}: #{message}"
+      end.join("\n")
+    end
+
+    # Generate data with repeated patterns (good for LDM testing)
+    def self.repeated_pattern_data(pattern_size: 1000, repetitions: 10, unique_middle: 500)
+      pattern = "A" * pattern_size
+      middle = "B" * unique_middle
+      (pattern * repetitions) + middle + (pattern * repetitions)
+    end
+
+    # Generate random data (worst case for compression)
+    def self.random_data(size: 10_000)
+      size.times.map { rand(256).chr }.join
+    end
+
+    # Generate highly compressible data
+    def self.compressible_data(size: 10_000)
+      "a" * size
+    end
+
+    # Mixed realistic data
+    def self.mixed_data(size: 10_000)
+      json_data(count: size / 100) + "\n" + log_data(count: size / 100)
+    end
+  end
+
+  # Formatting helpers for nice output
+  module Formatter
+    def self.header(title)
+      puts "\n#{"=" * 80}"
+      puts title.center(80)
+      puts "#{"=" * 80}\n\n"
+    end
+
+    def self.section(title)
+      puts "\n#{title}"
+      puts "-" * title.length
+    end
+
+    def self.table(data, title: nil)
+      puts "\n## #{title}\n" if title
+
+      # Create markdown table
+      if data.is_a?(Array) && data.first.is_a?(Hash)
+        headers = data.first.keys
+        rows = data.map(&:values)
+
+        if defined?(Terminal::Table)
+          table = Terminal::Table.new(headings: headers, rows: rows)
+          puts table
+          puts
+        end
+
+        # Print markdown version (always, for README)
+        puts "Markdown table (for README):" if defined?(Terminal::Table)
+        puts "| #{headers.join(" | ")} |"
+        puts "| #{headers.map { "---" }.join(" | ")} |"
+        rows.each do |row|
+          puts "| #{row.join(" | ")} |"
+        end
+      end
+      puts
+    end
+
+    def self.markdown_table(data, title: nil)
+      puts "\n## #{title}\n" if title
+
+      if data.is_a?(Array) && data.first.is_a?(Hash)
+        headers = data.first.keys
+        rows = data.map(&:values)
+
+        puts "| #{headers.join(" | ")} |"
+        puts "| #{headers.map { "---" }.join(" | ")} |"
+        rows.each do |row|
+          puts "| #{row.join(" | ")} |"
+        end
+      end
+      puts
+    end
+
+    def self.format_bytes(bytes)
+      if bytes < 1024
+        "#{bytes}B"
+      elsif bytes < 1024 * 1024
+        "#{(bytes / 1024.0).round(1)}KB"
+      else
+        "#{(bytes / (1024.0 * 1024)).round(2)}MB"
+      end
+    end
+
+    def self.format_number(num)
+      num.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+    end
+
+    def self.format_ratio(ratio)
+      "#{ratio.round(2)}x"
+    end
+  end
+
+  # Memory tracking utilities
+  module Memory
+    def self.estimate_cctx(level = 3)
+      VibeZstd::CCtx.estimate_memory(level)
+    end
+
+    def self.estimate_dctx
+      VibeZstd::DCtx.estimate_memory
+    end
+
+    def self.estimate_cdict(dict_size, level = 3)
+      VibeZstd::CDict.estimate_memory(dict_size, level)
+    end
+
+    def self.estimate_ddict(dict_size)
+      VibeZstd::DDict.estimate_memory(dict_size)
+    end
+
+    def self.current_memory_usage
+      # Try to get actual memory usage (platform-specific)
+      if RUBY_PLATFORM.match?(/darwin/)
+        # macOS
+        `ps -o rss= -p #{Process.pid}`.to_i * 1024
+      elsif RUBY_PLATFORM.match?(/linux/)
+        # Linux
+        File.read("/proc/#{Process.pid}/statm").split[1].to_i * 4096
+      else
+        # Fallback
+        0
+      end
+    end
+  end
+
+  # Benchmark result tracking
+  class BenchmarkResult
+    attr_reader :name, :iterations_per_sec, :memory_bytes, :compression_ratio, :extra_data
+
+    def initialize(name:, iterations_per_sec: nil, memory_bytes: nil, compression_ratio: nil, **extra_data)
+      @name = name
+      @iterations_per_sec = iterations_per_sec
+      @memory_bytes = memory_bytes
+      @compression_ratio = compression_ratio
+      @extra_data = extra_data
+    end
+
+    def to_h
+      h = {"Method" => name}
+      h["Speed"] = "#{Formatter.format_number(iterations_per_sec.to_i)} ops/sec" if iterations_per_sec
+      h["Memory"] = Formatter.format_bytes(memory_bytes) if memory_bytes
+      h["Ratio"] = Formatter.format_ratio(compression_ratio) if compression_ratio
+      h.merge!(extra_data.transform_values { |v| v.is_a?(Numeric) ? Formatter.format_number(v.to_i) : v.to_s })
+      h
+    end
+  end
+
+  # Helper to run a benchmark and collect results
+  def self.run_comparison(title:, &block)
+    Formatter.header(title)
+    results = []
+
+    yield results
+
+    # Display results
+    if results.any?
+      Formatter.table(results.map(&:to_h), title: "Results")
+
+      # Calculate speedups if we have iterations_per_sec
+      if results.all? { |r| r.iterations_per_sec }
+        baseline = results.first.iterations_per_sec
+        puts "\nRelative Performance:"
+        results.each do |result|
+          speedup = result.iterations_per_sec / baseline
+          puts "  #{result.name}: #{Formatter.format_ratio(speedup)}"
+        end
+      end
+    end
+
+    results
+  end
+end

data/benchmark/multithreading.rb

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "helpers"
+
+# Benchmark: Multi-threaded Compression Performance
+# Demonstrates the performance impact of using multiple worker threads
+
+BenchmarkHelpers.run_comparison(title: "Multi-threaded Compression Performance") do |results|
+  # Generate large test data (multi-threading only helps with larger data)
+  large_data = DataGenerator.mixed_data(size: 500_000)
+  puts "Test data size: #{Formatter.format_bytes(large_data.bytesize)}"
+  puts "CPU cores available: #{begin
+    `sysctl -n hw.ncpu`.strip
+  rescue
+    "unknown"
+  end}\n\n"
+
+  # Test with different worker counts
+  worker_counts = [0, 1, 2, 4, 8]
+
+  worker_counts.each do |workers|
+    Formatter.section("Testing: #{workers} worker#{(workers == 1) ? "" : "s"} #{(workers == 0) ? "(single-threaded)" : ""}")
+
+    cctx = VibeZstd::CCtx.new
+    cctx.nb_workers = workers if workers > 0
+
+    # Warm up
+    cctx.compress(large_data)
+
+    # Benchmark compression
+    compressed = nil
+    time = Benchmark.measure do
+      10.times { compressed = cctx.compress(large_data) }
+    end
+
+    ops_per_sec = 10 / time.real
+    compression_ratio = large_data.bytesize.to_f / compressed.bytesize
+    memory = Memory.estimate_cctx(3)
+
+    puts "Completed 10 iterations in #{time.real.round(3)}s"
+    puts "Throughput: #{Formatter.format_bytes((large_data.bytesize * 10 / time.real).to_i)}/sec"
+
+    results << BenchmarkResult.new(
+      :name => "#{workers} worker#{"s" unless workers == 1}",
+      :iterations_per_sec => ops_per_sec,
+      :compression_ratio => compression_ratio,
+      :memory_bytes => memory,
+      "Throughput" => "#{Formatter.format_bytes((large_data.bytesize * 10 / time.real).to_i)}/s"
+    )
+  end
+
+  puts "\n📊 Multi-threading Analysis:"
+
+  # Calculate speedup vs single-threaded
+  baseline = results[0].iterations_per_sec
+  results.each do |result|
+    speedup = result.iterations_per_sec / baseline
+    efficiency = begin
+      (speedup / result.name.split.first.to_i * 100).round(1)
+    rescue
+      100.0
+    end
+    puts "  #{result.name}: #{Formatter.format_ratio(speedup)} speedup" +
+      ((result.name != "0 workers") ? " (#{efficiency}% efficient)" : "")
+  end
+
+  # Test with job_size parameter
+  puts "\n"
+  Formatter.section("Testing: Multi-threading with different job sizes")
+
+  job_sizes = [256 * 1024, 512 * 1024, 1024 * 1024] # 256KB, 512KB, 1MB
+  job_results = []
+
+  job_sizes.each do |job_size|
+    cctx = VibeZstd::CCtx.new
+    cctx.nb_workers = 4
+    cctx.job_size = job_size
+
+    time = Benchmark.measure do
+      5.times { cctx.compress(large_data) }
+    end
+
+    ops_per_sec = 5 / time.real
+    puts "  Job size #{Formatter.format_bytes(job_size)}: #{time.real.round(3)}s for 5 iterations"
+
+    job_results << {
+      "Job Size" => Formatter.format_bytes(job_size),
+      "Time (5 ops)" => "#{time.real.round(3)}s",
+      "Ops/sec" => ops_per_sec.round(1)
+    }
+  end
+
+  Formatter.table(job_results)
+end
+
+puts "\n💡 Multi-threading Recommendations:"
+puts "  ✓ Use for data > 256KB (overhead not worth it for smaller data)"
+puts "  ✓ Optimal workers: 2-4 for most use cases (diminishing returns after)"
+puts "  ✓ More workers = higher memory usage"
+puts "  ✓ job_size affects compression ratio vs parallelism tradeoff"
+puts "\n  Typical speedups:"
+puts "    - 2 workers: 1.5-1.8x faster"
+puts "    - 4 workers: 2.0-2.5x faster"
+puts "    - 8 workers: 2.2-3.0x faster (diminishing returns)"

data/benchmark/run_all.rb

@@ -0,0 +1,150 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Run all benchmarks and generate a comprehensive report
+
+require_relative "helpers"
+
+Formatter.header("VibeZstd Comprehensive Benchmark Suite")
+
+puts "Ruby version: #{RUBY_VERSION}"
+puts "Platform: #{RUBY_PLATFORM}"
+puts "Zstd version: #{VibeZstd.version_string}"
+puts "Date: #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}"
+puts
+
+# Check for fixture
+fixture_path = File.join(__dir__, "..", "test", "fixtures", "sample.dict")
+unless File.exist?(fixture_path)
+  puts "⚠️  Dictionary fixture not found. Generating..."
+  require_relative "generate_fixture"
+  puts
+end
+
+# List of benchmarks to run
+benchmarks = [
+  {
+    name: "Context Reuse",
+    file: "context_reuse.rb",
+    description: "Compare reusing contexts vs creating new ones"
+  },
+  {
+    name: "Dictionary Usage",
+    file: "dictionary_usage.rb",
+    description: "Compare compression with and without dictionaries"
+  },
+  {
+    name: "Compression Levels",
+    file: "compression_levels.rb",
+    description: "Speed vs compression ratio across levels"
+  },
+  {
+    name: "Streaming",
+    file: "streaming.rb",
+    description: "Streaming API vs one-shot compression"
+  },
+  {
+    name: "Multi-threading",
+    file: "multithreading.rb",
+    description: "Performance with different worker counts"
+  },
+  {
+    name: "Dictionary Training",
+    file: "dictionary_training.rb",
+    description: "Compare dictionary training algorithms"
+  }
+]
+
+# Option parsing
+require "optparse"
+
+options = {
+  list: false,
+  benchmarks: []
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: ruby run_all.rb [options]"
+
+  opts.on("-l", "--list", "List available benchmarks") do
+    options[:list] = true
+  end
+
+  opts.on("-b", "--benchmark NAME", "Run specific benchmark (can specify multiple times)") do |name|
+    options[:benchmarks] << name
+  end
+
+  opts.on("-h", "--help", "Show this help") do
+    puts opts
+    exit
+  end
+end.parse!
+
+# List benchmarks if requested
+if options[:list]
+  puts "Available benchmarks:\n\n"
+  benchmarks.each_with_index do |bench, i|
+    puts "#{i + 1}. #{bench[:name]}"
+    puts "   File: #{bench[:file]}"
+    puts "   Description: #{bench[:description]}"
+    puts
+  end
+  exit
+end
+
+# Filter benchmarks if specific ones requested
+benchmarks_to_run = if options[:benchmarks].any?
+  benchmarks.select do |bench|
+    options[:benchmarks].any? { |name| bench[:name].downcase.include?(name.downcase) }
+  end
+else
+  benchmarks
+end
+
+if benchmarks_to_run.empty?
+  puts "No benchmarks match your criteria. Use --list to see available benchmarks."
+  exit 1
+end
+
+# Run benchmarks
+puts "Running #{benchmarks_to_run.size} benchmark(s)...\n\n"
+
+results = []
+start_time = Time.now
+
+benchmarks_to_run.each_with_index do |bench, i|
+  puts "\n" + "=" * 80
+  puts "Benchmark #{i + 1}/#{benchmarks_to_run.size}: #{bench[:name]}"
+  puts "=" * 80
+  puts
+
+  begin
+    load File.join(__dir__, bench[:file])
+    results << {name: bench[:name], status: "✓ Completed"}
+  rescue => e
+    puts "\n❌ Error running #{bench[:name]}: #{e.message}"
+    puts e.backtrace.first(5)
+    results << {name: bench[:name], status: "✗ Failed: #{e.message}"}
+  end
+
+  # Add separator between benchmarks
+  puts "\n" + "-" * 80 + "\n" if i < benchmarks_to_run.size - 1
+end
+
+end_time = Time.now
+duration = end_time - start_time
+
+# Summary
+puts "\n\n"
+Formatter.header("Benchmark Suite Summary")
+
+puts "Total time: #{duration.round(2)}s\n\n"
+
+results.each do |result|
+  puts "  #{result[:status].ljust(20)} #{result[:name]}"
+end
+
+puts "\n"
+puts "=" * 80
+puts "Benchmark suite completed!"
+puts "=" * 80

data/benchmark/streaming.rb

@@ -0,0 +1,154 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "helpers"
+require "stringio"
+require "tempfile"
+
+# Benchmark: Streaming vs One-Shot Compression
+# Compares streaming API vs convenience methods for different use cases
+
+BenchmarkHelpers.run_comparison(title: "Streaming vs One-Shot Compression") do |results|
+  # Generate test data
+  large_data = DataGenerator.mixed_data(size: 100_000)
+  puts "Test data size: #{Formatter.format_bytes(large_data.bytesize)}\n\n"
+
+  # Benchmark 1: One-shot compression (simple, all-in-memory)
+  Formatter.section("Testing: One-shot compression")
+  oneshot_time = Benchmark.measure do
+    10.times do
+      compressed = VibeZstd.compress(large_data)
+      VibeZstd.decompress(compressed)
+    end
+  end
+
+  oneshot_ops_per_sec = 10 / oneshot_time.real
+  oneshot_compressed = VibeZstd.compress(large_data)
+  puts "Completed 10 iterations in #{oneshot_time.real.round(3)}s"
+  puts "Compressed size: #{Formatter.format_bytes(oneshot_compressed.bytesize)}"
+
+  # Benchmark 2: Streaming compression (more control, constant memory)
+  Formatter.section("Testing: Streaming compression")
+  streaming_time = Benchmark.measure do
+    10.times do
+      # Compress
+      compressed_io = StringIO.new
+      writer = VibeZstd::Compress::Writer.new(compressed_io, level: 3)
+
+      # Write in chunks
+      large_data.scan(/.{1,1000}/m).each { |chunk| writer.write(chunk) }
+      writer.finish
+
+      # Decompress
+      compressed_io.rewind
+      reader = VibeZstd::Decompress::Reader.new(compressed_io)
+      decompressed = +""
+      while (chunk = reader.read)
+        decompressed << chunk
+      end
+    end
+  end
+
+  streaming_ops_per_sec = 10 / streaming_time.real
+  puts "Completed 10 iterations in #{streaming_time.real.round(3)}s"
+
+  # Benchmark 3: Streaming with larger chunks (optimization)
+  Formatter.section("Testing: Streaming with optimized chunk size")
+  optimized_streaming_time = Benchmark.measure do
+    10.times do
+      # Compress
+      compressed_io = StringIO.new
+      writer = VibeZstd::Compress::Writer.new(compressed_io, level: 3)
+
+      # Write in larger chunks
+      large_data.scan(/.{1,8192}/m).each { |chunk| writer.write(chunk) }
+      writer.finish
+
+      # Decompress with optimized chunk size
+      compressed_io.rewind
+      reader = VibeZstd::Decompress::Reader.new(compressed_io, initial_chunk_size: 8192)
+      decompressed = +""
+      while (chunk = reader.read)
+        decompressed << chunk
+      end
+    end
+  end
+
+  optimized_streaming_ops_per_sec = 10 / optimized_streaming_time.real
+  puts "Completed 10 iterations in #{optimized_streaming_time.real.round(3)}s"
+
+  # File-based streaming benchmark
+  Formatter.section("Testing: Streaming to file")
+  file_streaming_time = Benchmark.measure do
+    10.times do
+      Tempfile.create(["benchmark", ".zst"]) do |tmpfile|
+        # Compress to file
+        writer = VibeZstd::Compress::Writer.new(tmpfile, level: 3)
+        large_data.scan(/.{1,8192}/m).each { |chunk| writer.write(chunk) }
+        writer.finish
+
+        # Decompress from file
+        tmpfile.rewind
+        reader = VibeZstd::Decompress::Reader.new(tmpfile)
+        decompressed = +""
+        while (chunk = reader.read)
+          decompressed << chunk
+        end
+      end
+    end
+  end
+
+  file_streaming_ops_per_sec = 10 / file_streaming_time.real
+  puts "Completed 10 iterations in #{file_streaming_time.real.round(3)}s"
+
+  # Memory estimates (one-shot needs to buffer everything)
+  cctx_memory = Memory.estimate_cctx(3)
+  dctx_memory = Memory.estimate_dctx
+  oneshot_memory = cctx_memory + dctx_memory + large_data.bytesize + oneshot_compressed.bytesize
+  streaming_memory = cctx_memory + dctx_memory + 8192 # Only chunk size in memory
+
+  # Collect results
+  results << BenchmarkResult.new(
+    :name => "One-shot",
+    :iterations_per_sec => oneshot_ops_per_sec,
+    :memory_bytes => oneshot_memory,
+    "Use case" => "Small data, simplicity"
+  )
+
+  results << BenchmarkResult.new(
+    :name => "Streaming (1KB chunks)",
+    :iterations_per_sec => streaming_ops_per_sec,
+    :memory_bytes => streaming_memory,
+    "Use case" => "Large files, low memory"
+  )
+
+  results << BenchmarkResult.new(
+    :name => "Streaming (8KB chunks)",
+    :iterations_per_sec => optimized_streaming_ops_per_sec,
+    :memory_bytes => streaming_memory,
+    "Use case" => "Balanced performance"
+  )
+
+  results << BenchmarkResult.new(
+    :name => "File streaming",
+    :iterations_per_sec => file_streaming_ops_per_sec,
+    :memory_bytes => streaming_memory,
+    "Use case" => "Large files, disk I/O"
+  )
+
+  puts "\n💾 Memory Comparison:"
+  puts "  One-shot: #{Formatter.format_bytes(oneshot_memory)} (entire data in memory)"
+  puts "  Streaming: #{Formatter.format_bytes(streaming_memory)} (only chunks in memory)"
+  puts "  Memory savings: #{Formatter.format_bytes(oneshot_memory - streaming_memory)} (#{((oneshot_memory - streaming_memory).to_f / oneshot_memory * 100).round(1)}%)"
+end
+
+puts "\n💡 When to use each approach:"
+puts "  One-shot compression (VibeZstd.compress):"
+puts "    ✓ Small data (< 1MB)"
+puts "    ✓ Data already in memory"
+puts "    ✓ Simplicity is priority"
+puts "\n  Streaming compression (Writer/Reader):"
+puts "    ✓ Large files (> 1MB)"
+puts "    ✓ Memory-constrained environments"
+puts "    ✓ Incremental data (network streams, logs)"
+puts "    ✓ Need to process data on-the-fly"