omnizip 0.3.35 → 0.3.37

data/docs/guides/compression-algorithms/zstandard.adoc

@@ -49,29 +49,33 @@ Zstandard (zstd) is a modern, fast compression algorithm developed by Facebook t
 
 == Basic Usage
 
-=== Create Zstandard Archive
+=== Compress a File (.zst)
 
 [source,ruby]
 ----
-# Create with Zstandard compression
-Omnizip::Archive.create('fast-backup.7z', format: :seven_zip) do |archive|
-  archive.compression = :zstandard
-  archive.level = 3  # Default level
-  archive.add_directory('data/')
-end
+# Writes a real Zstandard frame (validated by the system zstd CLI)
+Omnizip.compress_file('logfile.log', 'logfile.log.zst')
+
+# Compression level (generic 0-9 scale; values above 9 pass through
+# as native zstd levels 1-22)
+Omnizip.compress_file('data.bin', 'data.bin.zst', level: 9)
+
+Omnizip.decompress_file('logfile.log.zst', 'logfile.log')
 ----
 
-=== Quick Compression
+=== Zstandard Entries in ZIP
+
+ZIP supports Zstandard as a per-entry method (method 93). Requires a
+modern reader (e.g. 7-Zip 21+):
 
 [source,ruby]
 ----
-# Fast compression for temporary archives
-Omnizip.compress_file(
-  'logfile.log',
-  'logs.7z',
-  compression: :zstandard,
-  level: 1  # Fastest compression
-)
+require 'omnizip/zip/output_stream'
+
+Omnizip::Zip::OutputStream.open('fast.zip') do |zip|
+  zip.put_next_entry('data.bin', compression: :zstandard, level: 3)
+  zip.write(data)
+end
 ----
 
 == Compression Levels
@@ -106,109 +110,63 @@ Zstandard offers levels 1-22 with fine-grained control:
 .Level Performance Example
 [source,ruby]
 ----
-# Compare different Zstandard levels
-[1, 3, 10, 19].each do |level|
+# Compare different Zstandard levels (native zstd scale)
+[1, 12, 19].each do |level|
   time = Benchmark.measure do
-    Omnizip.compress_file('data.bin', "zstd-#{level}.7z",
-      compression: :zstandard,
-      level: level
-    )
+    Omnizip.compress_file('data.bin', "zstd-#{level}.zst", level: level)
   end
 
-  size = File.size("zstd-#{level}.7z")
+  size = File.size("zstd-#{level}.zst")
   puts "Level #{level}: #{size / 1024} KB in #{time.real.round(3)}s"
 end
-
-# Output:
-# Level 1:  145 KB in 0.052s (fastest)
-# Level 3:  128 KB in 0.089s (default)
-# Level 10: 105 KB in 0.425s (better compression)
-# Level 19: 98 KB in 2.156s (approaching LZMA quality)
 ----
 
 == Performance Comparison
 
-=== Speed vs Deflate
-
-[cols="2,1,1,1"]
-|===
-|Operation |Deflate |Zstandard |Improvement
-
-|Compress 100 MB
-|10s
-|3s
-|3.3x faster
-
-|Decompress 100 MB
-|5s
-|1.2s
-|4.2x faster
-
-|Compression Ratio
-|68%
-|72%
-|4% better
-|===
-
-=== Speed vs LZMA
-
-[cols="2,1,1,1"]
-|===
-|Operation |LZMA |Zstandard |Trade-off
-
-|Compress 100 MB
-|60s
-|3s
-|20x faster, slightly worse ratio
-
-|Decompress 100 MB
-|8s
-|1.2s
-|6.7x faster
-
-|Compression Ratio
-|78%
-|72%
-|LZMA ~6% better
-|===
+No benchmark tables here: Omnizip's Zstandard is a pure Ruby
+implementation, so absolute throughput differs substantially from
+the C library. The algorithmic trade-offs hold — faster than LZMA
+per level, better ratio than Deflate — but measure on your own
+workloads.
 
 == Advanced Features
 
-=== Dictionary Training
+=== Dictionary Compression
 
-Train custom dictionaries for specific data types:
+A raw dictionary (shared prefix) dramatically improves ratios on
+small, repetitive inputs. Dictionary *training* is not implemented;
+prepare dictionary content yourself and load it with +from_raw+:
 
 [source,ruby]
 ----
-# Train dictionary on sample data
-training_data = Dir.glob('samples/*.json').map { |f| File.read(f) }
-dictionary = Omnizip::Zstandard.train_dictionary(training_data, size: 100.kilobytes)
-
-# Use trained dictionary for better compression
-Omnizip.compress_file('data.json', 'output.7z',
-  compression: :zstandard,
-  level: 5,
-  dictionary: dictionary
+dict = Omnizip::Algorithms::Zstandard::Dictionary.from_raw(
+  42, File.binread('shared-samples.bin')
 )
 
-# Can improve compression 10-30% for similar data
+zstd = Omnizip::Algorithms::Zstandard.new
+File.open('data.json', 'rb') do |input|
+  File.open('data.json.zstd', 'wb') do |output|
+    zstd.compress_with_dict(input, output, dict)
+  end
+end
+
+File.open('data.json.zstd', 'rb') do |input|
+  File.open('data.restored', 'wb') do |output|
+    zstd.decompress_with_dict(input, output, dict)
+  end
+end
 ----
 
-=== Streaming with Zstandard
+=== Streaming
 
-Excellent streaming performance:
+The codec is stream-oriented:
 
 [source,ruby]
 ----
-# Stream compression for real-time data
-Omnizip::OutputStream.open('stream.7z', format: :seven_zip) do |stream|
-  stream.compression = :zstandard
-  stream.level = 3  # Fast for real-time
-
-  # Process log entries as they arrive
-  log_stream.each do |log_entry|
-    stream.put_next_entry("log_#{log_entry.id}.txt")
-    stream.write(log_entry.content)
+zstd = Omnizip::AlgorithmRegistry.get(:zstandard).new
+File.open('input.log', 'rb') do |input|
+  File.open('input.log.zst', 'wb') do |output|
+    zstd.compress(input, output, level: 3)
   end
 end
 ----
@@ -222,15 +180,8 @@ Perfect for compressing logs quickly:
 [source,ruby]
 ----
 # Compress today's logs rapidly
-log_files = Dir.glob('/var/log/app/*.log')
-
-Omnizip.compress_files(
-  log_files,
-  "logs_#{Date.today}.7z",
-  compression: :zstandard,
-  level: 5  # Fast with good compression
-) do |progress|
-  puts "Compressing logs: #{progress.percentage}%"
+Dir.glob('/var/log/app/*.log').each do |log|
+  Omnizip.compress_file(log, "#{log}.zst", level: 3)
 end
 ----
 
@@ -240,47 +191,37 @@ Compress API responses on-the-fly:
 
 [source,ruby]
 ----
+require 'stringio'
+
 class ExportController < ApplicationController
   def download
-    # Generate export data
-    data = generate_export_data
-
-    # Compress quickly with Zstandard
-    compressed = Omnizip::Buffer.create(:seven_zip) do |archive|
-      archive.compression = :zstandard
-      archive.level = 3  # Fast compression
-
-      data.each do |filename, content|
-        archive.add(filename, content)
-      end
-    end
-
-    send_data compressed.string,
-      filename: 'export.7z',
-      type: 'application/x-7z-compressed'
+    data = generate_export_data # a String
+
+    zstd = Omnizip::AlgorithmRegistry.get(:zstandard).new
+    output = StringIO.new
+    zstd.compress(StringIO.new(data), output, level: 3)
+
+    send_data output.string,
+      filename: 'export.zst',
+      type: 'application/zstd'
   end
 end
 ----
 
 === Database Dumps
 
-Compress database backups efficiently:
+Stream a dump into a .zst frame:
 
 [source,ruby]
 ----
-# Stream database dump to compressed archive
+# Stream database dump to compressed file
 require 'open3'
 
-Omnizip::OutputStream.open('db-backup.7z', format: :seven_zip) do |stream|
-  stream.compression = :zstandard
-  stream.level = 7
-  stream.put_next_entry('database.sql')
-
+zstd = Omnizip::AlgorithmRegistry.get(:zstandard).new
+File.open('db-backup.sql.zst', 'wb') do |output|
   # Stream pg_dump output directly
-  Open3.popen3('pg_dump mydb') do |stdin, stdout, stderr, wait_thr|
-    while chunk = stdout.read(65536)
-      stream.write(chunk)
-    end
+  Open3.popen3('pg_dump mydb') do |_stdin, stdout, _stderr, _wait_thr|
+    zstd.compress(stdout, output, level: 7)
   end
 end
 ----
@@ -324,34 +265,28 @@ Zstandard is memory efficient:
 
 [source,bash]
 ----
-# Create Zstandard-compressed archive
-$ omnizip archive create backup.7z data/ \
-    --compression zstandard \
-    --level 5
-
-# Fast compression
-$ omnizip archive create logs.7z logs/ \
-    --compression zstandard \
-    --level 1 \
-    --verbose
-Algorithm: Zstandard, Level: 1
-Compression speed: 185 MB/s
-Original: 500 MB
-Compressed: 145 MB (71% reduction)
-Time: 2.7s
+$ omnizip compress data.bin data.bin.zst --algorithm zstandard
+
+# The output is a standard Zstandard frame, readable by the system
+# zstd CLI
 ----
 
 == Technical Details
 
 **Zstandard Features**:
 
-* **Fast Entropy Encoding**: Finite State Entropy (FSE) for speed
+* **Fast Entropy Encoding**: Finite State Entropy (FSE)
 * **Adaptive Window**: Dynamically sized based on level
-* **Dictionary Compression**: Optional pre-trained dictionaries
-* **Multi-Threading**: Native parallel compression support
-* **Seekable Format**: Optional frame indexes for random access
+* **Dictionary Compression**: Raw-dictionary priming (entropy-table
+  preloading from RFC dictionaries is not implemented)
+* **Frames**: Interoperable with the reference zstd CLI (verified
+  both directions)
+
+**Implementation**: Pure Ruby, part of Omnizip. No native binding.
 
-**Implementation**: Uses zstd-ruby gem (native binding)
+**Not available**: Zstandard is not offered as a 7z coder method;
+in 7z archives use LZMA2/PPMd/BZip2. Multi-threading is not
+implemented (single-threaded encoder).
 
 == See Also
 

data/docs/troubleshooting/index.adoc

@@ -121,13 +121,15 @@ omnizip compress input.txt output.lzma --level 3
 omnizip compress input.txt output.zst --algorithm zstandard
 ----
 
-3. Enable parallel processing:
+3. Run independent archives in parallel processes (see the
+link:../guides/advanced-features/parallel-processing.html[Parallel Processing guide]):
 [source,ruby]
 ----
-Omnizip.compress_file('input.txt', 'output.lzma',
-  parallel: true,
-  threads: 4
-)
+dirs = %w[data_a data_b]
+pids = dirs.map do |dir|
+  Process.fork { Omnizip.compress_directory(dir, "#{dir}.7z") }
+end
+pids.each { |pid| Process.wait(pid) }
 ----
 
 ==== Slow Decompression

data/lib/omnizip/algorithms/zstandard.rb

@@ -34,9 +34,8 @@ module Omnizip
     # - Dictionary support for small data compression
     # - Streaming and frame modes
     #
-    # This implementation uses the zstd-ruby gem which provides Ruby bindings
-    # to the official Zstandard C library. This ensures compatibility with
-    # the reference implementation and excellent performance.
+    # Pure Ruby implementation of the Zstandard format, interoperable
+    # with the reference zstd CLI in both directions.
     #
     # Zstandard is particularly effective for:
     # - Real-time compression needs

data/lib/omnizip/archive/builder.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Omnizip
+  module Archive
+    # Block argument of +Archive.create+. Translates the documented
+    # +add_file+/+add_directory+/+add_data+ vocabulary onto the
+    # handler proxy's +add(name, source_path)+ / +add_data+ calls.
+    class Builder
+      def initialize(proxy)
+        @proxy = proxy
+      end
+
+      # Add a file from disk, optionally under a different archive
+      # path: +add_file('report.pdf', 'reports/report.pdf')+.
+      def add_file(file_path, archive_path = nil)
+        @proxy.add(archive_path || ::File.basename(file_path), file_path)
+      end
+
+      # Add a directory tree under its own name:
+      # +add_directory('photos/')+ stores +photos/x.jpg+.
+      def add_directory(dir_path)
+        prefix = ::File.basename(::File.expand_path(dir_path))
+        @proxy.add("#{prefix}/")
+        walk(dir_path, prefix)
+      end
+
+      # Add in-memory content at +archive_path+.
+      def add_data(archive_path, data)
+        @proxy.add_data(archive_path, data)
+      end
+
+      private
+
+      def walk(dir, prefix)
+        ::Dir.children(dir).sort.each do |child|
+          full = ::File.join(dir, child)
+          name = "#{prefix}/#{child}"
+          if ::File.directory?(full)
+            @proxy.add("#{name}/")
+            walk(full, name)
+          else
+            @proxy.add(name, full)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnizip/archive/reader_session.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Omnizip
+  module Archive
+    # Block argument (and return value) of +Archive.open+. Delegates
+    # per operation to the resolved ArchiveHandler, which opens and
+    # closes the underlying reader on every call — the session itself
+    # holds no resources.
+    class ReaderSession
+      def initialize(handler, path, options = {})
+        @handler = handler
+        @path = path
+        @options = options
+      end
+
+      # Archive metadata records (see Archive::Entry).
+      def entries
+        @handler.list(@path, details: true, **@options).map do |h|
+          Entry.new(name: h[:name], size: h[:size], directory: h[:directory],
+                    mtime: h[:mtime] || h[:time])
+        end
+      end
+
+      def each_entry(&block)
+        entries.each(&block)
+      end
+
+      # Contents of a single entry, without extraction.
+      def read(entry_name)
+        @handler.read_entry(@path, entry_name, **@options)
+      end
+
+      # Extract one entry to an explicit destination path.
+      def extract(entry_name, dest_path)
+        entry = entries.find { |e| e.name == entry_name }
+        unless entry
+          raise Errno::ENOENT, "Entry not found: #{entry_name}"
+        end
+
+        if entry.directory?
+          ::FileUtils.mkdir_p(dest_path)
+          return dest_path
+        end
+
+        ::FileUtils.mkdir_p(::File.dirname(dest_path))
+        ::File.binwrite(dest_path, read(entry_name))
+        dest_path
+      end
+
+      def extract_all(output_dir)
+        @handler.extract_to(@path, output_dir, **@options)
+      end
+
+      # Extract every entry whose name matches +pattern+ under
+      # +output_dir+, preserving archive-relative paths.
+      def extract_matching(pattern, output_dir)
+        ::FileUtils.mkdir_p(output_dir)
+        entries.select do |e|
+          !e.directory? && ::File.fnmatch?(pattern, e.name,
+                                           ::File::FNM_PATHNAME)
+        end.map { |e| extract(e.name, ::File.join(output_dir, e.name)) }
+      end
+    end
+  end
+end

data/lib/omnizip/archive.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Omnizip
+  # High-level block-style archive facade over the registered
+  # ArchiveHandlers. This is the API the guides document:
+  #
+  #   Omnizip::Archive.create('backup.7z', format: :seven_zip,
+  #                           level: 9) do |archive|
+  #     archive.add_file('report.pdf', 'reports/report.pdf')
+  #     archive.add_directory('photos/')
+  #     archive.add_data('notes/readme.txt', "generated\n")
+  #   end
+  #
+  #   Omnizip::Archive.open('backup.7z') do |archive|
+  #     archive.entries.each { |e| puts "#{e.name} (#{e.size})" }
+  #     archive.read('notes/readme.txt')
+  #     archive.extract_all('output/')
+  #   end
+  #
+  # The format is resolved from the output path's extension when
+  # +format:+ is not given explicitly, exactly like the convenience
+  # methods. Passwords are supported for 7z only (ZIP writing and
+  # reading have no encryption support); asking for one elsewhere
+  # raises rather than silently producing an unprotected archive.
+  module Archive
+    autoload :Builder, "omnizip/archive/builder"
+    autoload :ReaderSession, "omnizip/archive/reader_session"
+
+    # Metadata record yielded by ReaderSession#entries. The +size+
+    # member intentionally shadows Struct#size: the documented
+    # contract is entry size in bytes, not the member count.
+    # rubocop:disable-next Lint/StructNewOverride -- documented byte size
+    Entry = Struct.new(:name, :size, :directory, :mtime, keyword_init: true) do
+      def directory?
+        directory
+      end
+    end
+
+    PASSWORD_FORMATS = [:seven_zip].freeze
+    private_constant :PASSWORD_FORMATS
+
+    class << self
+      # Create an archive, yielding a Builder that translates the
+      # documented +add_file+/+add_directory+/+add_data+ calls onto
+      # the resolved handler. Returns the archive path.
+      def create(path, format: Convenience::DEFAULT_FORMAT, **options, &block)
+        resolved = Omnizip.resolve_archive_format(path, format)
+        check_password_support!(resolved, options[:password])
+
+        Omnizip::ArchiveHandler.for(resolved).create(path, **options) do |proxy|
+          block&.call(Builder.new(proxy))
+        end
+        path
+      end
+
+      # Open an archive for reading, yielding a ReaderSession. Without
+      # a block, returns the session. An explicit +password:+ wins
+      # over ENV['OMNIZIP_PASSWORD'].
+      def open(path, password: nil, **options, &block)
+        resolved = Omnizip.resolve_archive_format(
+          path, Convenience::DEFAULT_FORMAT, writing: false
+        )
+        pw = password || ENV.fetch("OMNIZIP_PASSWORD", nil)
+        check_password_support!(resolved, pw)
+
+        options[:password] = pw if pw
+        session = ReaderSession.new(
+          Omnizip::ArchiveHandler.for(resolved), path, options
+        )
+        return session unless block
+
+        yield session
+      end
+
+      private
+
+      def check_password_support!(resolved, password)
+        return unless password && !PASSWORD_FORMATS.include?(resolved)
+
+        raise Omnizip::UnsupportedFormatError,
+              "password is only supported for #{PASSWORD_FORMATS.join(', ')}; " \
+              "#{resolved.inspect} has no encryption support"
+      end
+    end
+  end
+end

data/lib/omnizip/archive_handler.rb

@@ -70,6 +70,7 @@ module Omnizip
       zip: -> { Omnizip::ArchiveHandlers::ZipHandler },
       tar: -> { Omnizip::ArchiveHandlers::TarHandler },
       seven_zip: -> { Omnizip::ArchiveHandlers::SevenZipHandler },
+      rar: -> { Omnizip::ArchiveHandlers::RarHandler },
     }.freeze
     private_constant :LAZY_LOAD_TRIGGERS
   end

data/lib/omnizip/archive_handlers/rar_handler.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+module Omnizip
+  module ArchiveHandlers
+    # Read-only adapter for the RAR format. RAR compression is
+    # patented software (see link:README[RAR write support]); this
+    # handler exposes extraction/listing/reading only, so the
+    # convenience API and the Archive facade can operate on existing
+    # archives truthfully instead of raising on every operation.
+    class RarHandler
+      def create(_path, **_options)
+        raise Omnizip::UnsupportedFormatError,
+              "RAR archives cannot be created (patented format); " \
+              "Omnizip reads RAR archives only"
+      end
+
+      def extract_to(path, output_dir, password: nil, **_)
+        Omnizip::Formats::Rar::Decompressor.extract(path, output_dir,
+                                                    password: password)
+      end
+
+      def list(path, details: false, **_)
+        reader = Omnizip::Formats::Rar::Reader.new(path).open
+        entries = reader.list_files
+        if details
+          entries.map do |e|
+            { name: e.name, size: e.size, directory: e.is_dir,
+              mtime: e.mtime }
+          end
+        else
+          entries.map(&:name)
+        end
+      end
+
+      def read_entry(path, entry_name, password: nil, **_)
+        Dir.mktmpdir("omnizip-rar-entry") do |dir|
+          dest = File.join(dir, "entry")
+          Omnizip::Formats::Rar::Decompressor.extract_entry(
+            path, entry_name, dest, password: password
+          )
+          File.binread(dest)
+        end
+      end
+    end
+  end
+end
+
+Omnizip::ArchiveHandler.register(:rar, Omnizip::ArchiveHandlers::RarHandler.new)

data/lib/omnizip/archive_handlers/seven_zip_handler.rb

@@ -8,8 +8,8 @@ module Omnizip
       # Creates a .7z archive. The yielded proxy only supports
       # +add(name, source_path)+ for files — directory entries are not
       # part of the single-file convenience flow.
-      def create(path, &block)
-        Omnizip::Formats::SevenZip.create(path) do |writer|
+      def create(path, **options, &block)
+        Omnizip::Formats::SevenZip.create(path, options) do |writer|
           block&.call(FileProxy.new(writer))
         end
       end
@@ -73,6 +73,10 @@ module Omnizip
 
           @writer.add_file(source_path, name)
         end
+
+        def add_data(name, data)
+          @writer.add_data(name, data)
+        end
       end
     end
   end

data/lib/omnizip/archive_handlers/tar_handler.rb

@@ -50,6 +50,10 @@ module Omnizip
 
           @writer.add(name, source_path)
         end
+
+        def add_data(name, data)
+          @writer.add_data(name, data)
+        end
       end
 
       def read_entry(path, entry_name)