omnizip 0.3.1

data/readme-docs/performance-profiler.adoc

@@ -0,0 +1,389 @@
+= Performance Profiler
+:toc:
+:toclevels: 3
+
+== Purpose
+
+The Performance Profiler provides comprehensive profiling and optimization tools to identify bottlenecks and improve compression performance.
+
+== Features
+
+* **Method profiling** - Track execution time and call counts
+* **Memory profiling** - Monitor allocation and retention
+* **Hot path analysis** - Identify performance bottlenecks
+* **Optimization suggestions** - AI-powered recommendations
+* **Report generation** - Formatted profiling reports
+
+== Basic Profiling
+
+=== Profile a Block of Code
+
+[source,ruby]
+----
+# Simple profiling
+result = Omnizip::Profiler.profile do
+  Omnizip::Formats::SevenZip::Writer.new('archive.7z') do |zip|
+    zip.add_file('large_file.dat')
+  end
+end
+
+puts "Execution time: #{result.total_time}s"
+puts "Memory allocated: #{result.memory_allocated} bytes"
+----
+
+=== Profile with Custom Name
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new(profile_name: "compression_test")
+
+profiler.profile("LZMA compression") do
+  algorithm = Omnizip::AlgorithmRegistry.get(:lzma).new(level: 9)
+  File.open('input.txt', 'rb') do |input|
+    File.open('output.lzma', 'wb') do |output|
+      algorithm.compress(input, output)
+    end
+  end
+end
+
+# Get profiling report
+report = profiler.report
+puts "Total execution time: #{report.total_execution_time}s"
+----
+
+== Hot Path Analysis
+
+=== Identify Performance Bottlenecks
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Profile multiple operations
+profiler.profile("read_file") { File.read('data.txt') }
+profiler.profile("compress") { compress_data(data) }
+profiler.profile("write_file") { File.write('output.dat', compressed) }
+
+# Analyze hot paths (operations >10% of total time)
+hot_paths = profiler.analyze_hot_paths(threshold_percentage: 10.0)
+
+hot_paths.each do |operation|
+  puts "Hot path: #{operation.operation_name}"
+  puts "  Time: #{operation.total_time}s"
+  puts "  Percentage: #{(operation.total_time / profiler.report.total_execution_time * 100).round(1)}%"
+end
+----
+
+== Bottleneck Identification
+
+=== Find CPU and Memory Bottlenecks
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Profile compression pipeline
+profiler.profile("BWT") { bwt_transform(data) }
+profiler.profile("MTF") { mtf_encode(transformed) }
+profiler.profile("Huffman") { huffman_encode(encoded) }
+
+# Identify bottlenecks
+bottlenecks = profiler.identify_bottlenecks
+
+bottlenecks.each do |bottleneck|
+  case bottleneck[:type]
+  when :cpu
+    puts "CPU bottleneck: #{bottleneck[:operation]}"
+    puts "  Time: #{bottleneck[:time]}s"
+    puts "  Severity: #{bottleneck[:severity]}"
+  when :memory
+    puts "Memory bottleneck: #{bottleneck[:operation]}"
+    puts "  Allocated: #{bottleneck[:allocated]} bytes"
+  when :gc
+    puts "GC pressure: #{bottleneck[:operation]}"
+    puts "  GC pressure: #{bottleneck[:gc_pressure]}"
+  end
+end
+----
+
+== Optimization Suggestions
+
+=== Generate Improvement Recommendations
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Run profiling
+10.times do |i|
+  profiler.profile("iteration_#{i}") do
+    # Compression operations
+  end
+end
+
+# Generate suggestions
+suggestions = profiler.generate_suggestions
+
+suggestions.each do |suggestion|
+  puts "\n#{suggestion.title}"
+  puts "  #{suggestion.description}"
+  puts "  Severity: #{suggestion.severity}"
+  puts "  Category: #{suggestion.category}"
+  puts "  Estimated impact: #{(suggestion.impact_estimate * 100).round(1)}%"
+  
+  if suggestion.recommendation
+    puts "  Recommendation: #{suggestion.recommendation}"
+  end
+end
+----
+
+== Profiling Reports
+
+=== Generate Detailed Reports
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new(profile_name: "BZip2 Compression")
+
+# Profile operations
+profiler.profile("initialization") { setup_compressor }
+profiler.profile("bwt_transform") { bwt.transform(data) }
+profiler.profile("mtf_encoding") { mtf.encode(transformed) }
+profiler.profile("huffman_coding") { huffman.encode(encoded) }
+profiler.profile("finalization") { write_output }
+
+# Get detailed report
+report = profiler.report
+
+puts "=== Profiling Report: #{report.profile_name} ==="
+puts "\nTotal execution time: #{report.total_execution_time}s"
+puts "Total memory allocated: #{report.total_memory_allocated} bytes"
+puts "Operations profiled: #{report.results.size}"
+
+puts "\n=== Slowest Operations ==="
+report.slowest_operations(limit: 3).each do |op|
+  percentage = (op.total_time / report.total_execution_time * 100).round(1)
+  puts "  #{op.operation_name}: #{op.total_time}s (#{percentage}%)"
+end
+
+puts "\n=== Memory Intensive Operations ==="
+report.memory_intensive_operations(limit: 3).each do |op|
+  mb = (op.memory_allocated.to_f / (1024 * 1024)).round(2)
+  puts "  #{op.operation_name}: #{mb}MB"
+end
+----
+
+== Method Profiling
+
+=== Profile Specific Methods
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Register method profiler
+method_profiler = Omnizip::Profiler::MethodProfiler.new
+profiler.register_profiler(:method, method_profiler)
+
+# Profile method calls
+algorithm = Omnizip::AlgorithmRegistry.get(:lzma).new
+profiler.profile_method(algorithm, :compress, input, output)
+
+# Check results
+results = profiler.report.results.find { |r| r.operation_name.include?('compress') }
+puts "Method calls: #{results.call_count}"
+puts "Average time per call: #{results.average_time}s"
+----
+
+== Memory Profiling
+
+=== Track Memory Allocation
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Register memory profiler
+memory_profiler = Omnizip::Profiler::MemoryProfiler.new
+profiler.register_profiler(:memory, memory_profiler)
+
+# Profile with memory tracking
+profiler.profile("data_processing", profiler_type: :memory) do
+  data = Array.new(1_000_000) { rand }
+  data.map { |x| x * 2 }
+end
+
+# Check memory usage
+report = profiler.report
+puts "Memory allocated: #{report.total_memory_allocated} bytes"
+puts "Memory retained: #{report.total_memory_retained} bytes"
+----
+
+== Examples
+
+=== Example 1: Find Compression Bottleneck
+
+[source,ruby]
+----
+def profile_compression(file_path)
+  profiler = Omnizip::Profiler.new(profile_name: "Compression Analysis")
+  
+  # Profile each stage
+  data = profiler.profile("read_input") do
+    File.read(file_path)
+  end
+  
+  compressed = profiler.profile("compress") do
+    algorithm = Omnizip::AlgorithmRegistry.get(:bzip2).new(level: 9)
+    output = StringIO.new
+    algorithm.compress(StringIO.new(data), output)
+    output.string
+  end
+  
+  profiler.profile("write_output") do
+    File.write("#{file_path}.bz2", compressed)
+  end
+  
+  # Analyze results
+  report = profiler.report
+  puts "\n=== Compression Profile ==="
+  puts "Total time: #{report.total_execution_time}s"
+  
+  report.results.each do |result|
+    percentage = (result.total_time / report.total_execution_time * 100).round(1)
+    puts "#{result.operation_name}: #{result.total_time}s (#{percentage}%)"
+  end
+  
+  # Generate optimization suggestions
+  suggestions = profiler.generate_suggestions
+  if suggestions.any?
+    puts "\n=== Optimization Suggestions ==="
+    suggestions.first(3).each { |s| puts "- #{s.title}" }
+  end
+end
+
+profile_compression('large_file.txt')
+----
+
+=== Example 2: Compare Algorithm Performance
+
+[source,ruby]
+----
+def compare_algorithms(data, algorithms)
+  results = {}
+  
+  algorithms.each do |algo_name|
+    profiler = Omnizip::Profiler.new(profile_name: algo_name.to_s)
+    
+    compressed_size = profiler.profile("compression") do
+      algorithm = Omnizip::AlgorithmRegistry.get(algo_name).new(level: 6)
+      output = StringIO.new
+      algorithm.compress(StringIO.new(data), output)
+      output.size
+    end
+    
+    results[algo_name] = {
+      time: profiler.report.total_execution_time,
+      size: compressed_size,
+      ratio: (1 - compressed_size.to_f / data.size) * 100
+    }
+  end
+  
+  # Print comparison
+  puts "\n=== Algorithm Comparison ==="
+  puts "Original size: #{data.size} bytes\n\n"
+  
+  results.sort_by { |_, v| v[:time] }.each do |algo, stats|
+    puts "#{algo}:"
+    puts "  Time: #{stats[:time].round(3)}s"
+    puts "  Size: #{stats[:size]} bytes"
+    puts "  Ratio: #{stats[:ratio].round(1)}%"
+  end
+end
+
+data = File.read('test_file.dat')
+compare_algorithms(data, [:deflate, :lzma, :bzip2, :zstd])
+----
+
+=== Example 3: Memory Leak Detection
+
+[source,ruby]
+----
+def detect_memory_leaks(iterations = 100)
+  profiler = Omnizip::Profiler.new(profile_name: "Memory Leak Detection")
+  memory_profiler = Omnizip::Profiler::MemoryProfiler.new
+  profiler.register_profiler(:memory, memory_profiler)
+  
+  baseline_memory = nil
+  
+  iterations.times do |i|
+    profiler.profile("iteration_#{i}", profiler_type: :memory) do
+      # Suspect operation
+      data = Array.new(10_000) { rand }
+      compress_data(data)
+    end
+    
+    current_memory = profiler.report.total_memory_allocated
+    
+    if i == 0
+      baseline_memory = current_memory
+    elsif i % 10 == 0
+      growth = current_memory - baseline_memory
+      growth_rate = (growth.to_f / baseline_memory * 100).round(2)
+      
+      puts "Iteration #{i}: #{growth} bytes growth (#{growth_rate}%)"
+      
+      if growth_rate > 50
+        puts "⚠️  Potential memory leak detected!"
+        break
+      end
+    end
+  end
+end
+
+detect_memory_leaks
+----
+
+== Profiler Configuration
+
+=== Enable/Disable Profiling
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Disable profiling (no overhead)
+profiler.disable!
+
+# Operations run without profiling
+profiler.profile("operation") { slow_operation }  # Not profiled
+
+# Re-enable profiling
+profiler.enable!
+
+# Operations are now profiled again
+profiler.profile("operation") { slow_operation }  # Profiled
+----
+
+=== Reset Profiler State
+
+[source,ruby]
+----
+profiler = Omnizip::Profiler.new
+
+# Collect some data
+profiler.profile("op1") { operation1 }
+profiler.profile("op2") { operation2 }
+
+# Reset profiler
+profiler.reset!
+
+# Start fresh profiling
+profiler.profile("op3") { operation3 }  # Previous data cleared
+----
+
+== See Also
+
+* link:../README.adoc#performance[Performance Analysis]
+* link:compression-profiles.adoc[Compression Profiles]
+* link:advanced-features.adoc[Advanced Features]

data/readme-docs/preprocessing-filters.adoc

@@ -0,0 +1,280 @@
+= Preprocessing Filters Guide
+:toc:
+:toclevels: 3
+
+== Purpose
+
+This document covers preprocessing filters that improve compression of specific data types, particularly executable files and multimedia data.
+
+== Supported Filters
+
+[cols="20,15,65",options="header"]
+|===
+|Filter |ID |Description
+
+|BCJ x86 |0x04 |Branch conversion for x86 executables
+|BCJ2 |0x0303011B |Advanced 4-stream x86 filter
+|BCJ ARM |0x05 |ARM executable filter
+|BCJ ARM64 |0x0A |ARM64/AArch64 filter
+|BCJ PPC |0x07 |PowerPC filter
+|BCJ IA-64 |0x06 |Itanium filter
+|BCJ SPARC |0x08 |SPARC filter
+|Delta |0x03 |Delta encoding (configurable distance)
+|===
+
+== BCJ (Branch-Call-Jump) Filters
+
+=== General
+
+Branch-Call-Jump filters improve compression of executable files by converting relative addresses to absolute addresses. This transformation makes the data more compressible because branch instructions share common patterns.
+
+=== Supported Architectures
+
+* **BCJ x86** - Intel/AMD x86 (32-bit and 64-bit)
+* **BCJ ARM** - ARM 32-bit executables
+* **BCJ ARM64** - ARM 64-bit (AArch64) executables
+* **BCJ PPC** - PowerPC executables
+* **BCJ SPARC** - SPARC executables
+* **BCJ IA-64** - Intel Itanium executables
+
+=== How It Works
+
+. Scans binary code for branch/call instructions
+. Converts relative offsets to absolute addresses
+. Makes patterns more regular and compressible
+. Decoder reverses the transformation after decompression
+
+=== Usage
+
+[source,ruby]
+----
+# Use with filter pipeline
+pipeline = Omnizip::FilterPipeline.new
+pipeline.add_filter(:bcj_x86)  # For x86 executables
+
+# Apply before compression
+filtered_data = pipeline.encode(executable_data)
+algorithm.compress(StringIO.new(filtered_data), output)
+
+# Different architectures
+pipeline_arm = Omnizip::FilterPipeline.new
+pipeline_arm.add_filter(:bcj_arm)
+
+pipeline_arm64 = Omnizip::FilterPipeline.new
+pipeline_arm64.add_filter(:bcj_arm64)
+----
+
+=== Typical Improvements
+
+Using BCJ filters on executable files typically improves compression by:
+
+* **10-30%** for x86/x64 executables
+* **15-35%** for ARM executables
+* **20-40%** for stripped executables (no debug symbols)
+
+== BCJ2 Filter
+
+=== General
+
+BCJ2 provides advanced 4-stream filtering for x86 code, achieving better compression than standard BCJ. It splits the filtered data into four separate streams that can be compressed independently.
+
+=== How It Works
+
+. Analyzes x86 code structure
+. Splits into 4 streams:
+  * Main stream (data and unprocessed bytes)
+  * Call stream (call instruction targets)
+  * Jump stream (jump instruction targets)
+  * Common stream (shared data)
+. Each stream is compressed separately
+. Decoder merges streams during decompression
+
+=== Usage
+
+[source,ruby]
+----
+filter = Omnizip::FilterRegistry.get(:bcj2).new
+encoded_streams = filter.encode(x86_code)
+# Returns 4 separate streams for optimal compression
+
+# Compress each stream
+encoded_streams.each_with_index do |stream, idx|
+  algorithm.compress(StringIO.new(stream), output_files[idx])
+end
+----
+
+=== When to Use BCJ2
+
+**Use BCJ2 when:**
+
+* Maximum compression is needed
+* Processing large x86 executables
+* Archive size is critical (software distribution)
+
+**Use standard BCJ when:**
+
+* Simplicity is preferred
+* Working with mixed content
+* Speed is more important than maximum compression
+
+=== Typical Improvements
+
+BCJ2 typically achieves:
+
+* **5-10% better** compression than standard BCJ
+* **15-40%** better than no filter
+* Best results on large executables (> 1MB)
+
+== Delta Filter
+
+=== General
+
+Delta encoding is effective for multimedia files and time-series data where consecutive bytes have small differences. It transforms absolute values into differences between consecutive values.
+
+=== How It Works
+
+. Computes differences between consecutive bytes
+. Stores deltas instead of absolute values
+. Makes nearly constant data highly compressible
+. Configurable distance parameter for pattern matching
+
+=== Configuration
+
+The `distance` parameter determines the stride:
+
+* **distance=1:** Byte-by-byte differences (images, audio)
+* **distance=2:** 16-bit word differences
+* **distance=4:** 32-bit word differences
+* **distance=N:** Custom stride
+
+=== Usage
+
+[source,ruby]
+----
+# Basic delta filter (distance=1)
+filter = Omnizip::FilterRegistry.get(:delta).new(distance: 1)
+filtered = filter.encode(audio_data)
+
+# For 16-bit audio samples
+filter_16bit = Omnizip::FilterRegistry.get(:delta).new(distance: 2)
+filtered_audio = filter_16bit.encode(audio_samples)
+
+# For database dumps with aligned records
+filter_db = Omnizip::FilterRegistry.get(:delta).new(distance: 4)
+filtered_db = filter_db.encode(database_dump)
+----
+
+=== Best Use Cases
+
+**Excellent for:**
+
+* Uncompressed audio (WAV, raw PCM)
+* Time-series sensor data
+* Database dumps with sequential indices
+* Bitmap images (BMP, uncompressed TIFF)
+
+**Not suitable for:**
+
+* Already compressed data (MP3, JPEG)
+* Random data
+* Text files
+* Encrypted data
+
+=== Typical Improvements
+
+Delta filter can achieve:
+
+* **50-80%** better compression on audio waveforms
+* **30-60%** better on time-series data
+* **20-40%** better on bitmap images
+* **Minimal improvement** on text or compressed data
+
+== Filter Chaining
+
+=== General
+
+Multiple filters can be chained together for optimal compression. The order matters - apply filters in sequence that makes data progressively more compressible.
+
+=== Usage
+
+[source,ruby]
+----
+# Chain multiple filters
+pipeline = Omnizip::FilterPipeline.new
+pipeline.add_filter(:bcj_x86)   # First: convert branches
+pipeline.add_filter(:delta, distance: 1)  # Then: delta encode
+
+# Apply chain
+filtered_data = pipeline.encode(executable_data)
+algorithm.compress(StringIO.new(filtered_data), output)
+----
+
+=== Recommended Filter Chains
+
+**For x86 executables:**
+```
+BCJ x86 → LZMA2
+```
+
+**For ARM executables:**
+```
+BCJ ARM → LZMA2
+```
+
+**For large x86 binaries (maximum compression):**
+```
+BCJ2 (4 streams) → LZMA2 (each stream)
+```
+
+**For uncompressed audio:**
+```
+Delta (distance=2) → LZMA2 or BZip2
+```
+
+**For bitmap images:**
+```
+Delta (distance=1) → LZMA2
+```
+
+== Performance Considerations
+
+=== Processing Overhead
+
+Filters add minimal overhead:
+
+* **BCJ filters:** < 5% processing time
+* **Delta filter:** < 3% processing time
+* **BCJ2:** 10-15% processing time (4-stream handling)
+
+The compression gains far outweigh the processing cost.
+
+=== Memory Usage
+
+* **BCJ filters:** Minimal (< 1MB)
+* **Delta filter:** Minimal (< 1MB)
+* **BCJ2:** Moderate (needs buffer for 4 streams)
+
+== Integration with .7z Archives
+
+Filters are automatically applied when using .7z archives:
+
+[source,ruby]
+----
+# Create .7z with BCJ filter
+writer = Omnizip::Formats::SevenZip::Writer.new('programs.7z')
+writer.add_filter(:bcj_x86)  # Applies to all files
+writer.add_file('program.exe')
+writer.close
+
+# Filter is automatically applied during extraction
+reader = Omnizip::Formats::SevenZip::Reader.new('programs.7z')
+reader.extract_all('output/')  # BCJ filter automatically reversed
+reader.close
+----
+
+== See Also
+
+* link:compression-algorithms.adoc[Compression Algorithms]
+* link:api-usage.adoc[Library API Usage]
+* link:archive-formats.adoc[Archive Formats]
+* link:../README.adoc[Main README]