multi_compress 0.3.5 → 0.5.0

data/exe/multi_compress

@@ -0,0 +1,228 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "optparse"
+require "securerandom"
+
+if ARGV.first == "db"
+  require "multi_compress/db_deployment"
+  exit MultiCompress::DBDeployment::CLI.run(ARGV.drop(1))
+end
+
+require "multi_compress"
+
+module MultiCompress
+  class CLI
+    EXT          = { zstd: ".zst", lz4: ".mclz4", brotli: ".br" }.freeze
+    KNOWN_EXT    = { ".zst" => :zstd, ".zstd" => :zstd, ".mclz4" => :lz4, ".br" => :brotli }.freeze
+    NAMED_LEVELS = %w[fastest default best].freeze
+    READ_CHUNK   = 256 * 1024
+    MAX_OUTPUT_LIMIT = (2**(1.size * 8 - 1)) - 1
+
+    def self.run(argv)
+      new.run(argv)
+    end
+
+    def run(argv)
+      opts  = parse(argv)
+      files = opts.delete(:files)
+
+      validate_invocation!(files, opts)
+
+      if files.empty?
+        process_stream($stdin, $stdout, opts)
+      else
+        files.each { |path| process_file(path, opts) }
+      end
+      0
+    rescue OptionParser::ParseError => e
+      warn "multi_compress: #{e.message}"
+      2
+    rescue MultiCompress::Error, SystemCallError, IOError, ArgumentError, RangeError => e
+      warn "multi_compress: #{e.message}"
+      1
+    end
+
+    private
+
+    def parse(argv)
+      o = { decompress: false, keep: false, force: false, stdout: false, quiet: false }
+
+      @parser = OptionParser.new do |p|
+        p.banner = "Usage: multi_compress [OPTIONS] [FILE...]"
+        p.separator ""
+        p.on("-a", "--algo NAME", %w[zstd lz4 brotli], "zstd | lz4 | brotli (default: by extension, else zstd)") { |v| o[:algo] = v.to_sym }
+        p.on("-l", "--level LEVEL", "number, or: fastest | default | best") { |v| o[:level] = parse_level(v) }
+        p.on("-d", "--decompress", "decompress instead of compress") { o[:decompress] = true }
+        p.on("-o", "--output PATH", "write result to PATH (single input only)") { |v| o[:output] = v }
+        p.on("-c", "--stdout", "write to stdout, keep input") { o[:stdout] = true }
+        p.on("-k", "--keep", "keep input file (default: remove on success)") { o[:keep] = true }
+        p.on("-f", "--force", "overwrite existing output") { o[:force] = true }
+        p.on("-q", "--quiet", "no progress line") { o[:quiet] = true }
+        p.on("--max-output SIZE", "cap decompressed size, e.g. 64MB") { |v| o[:max_output_size] = parse_size(v) }
+        p.on("--version", "print gem + library versions") do
+          puts "multi_compress #{MultiCompress::VERSION} (zstd #{MultiCompress.version(:zstd)}, " \
+               "lz4 #{MultiCompress.version(:lz4)}, brotli #{MultiCompress.version(:brotli)})"
+          exit 0
+        end
+        p.on("-h", "--help") { puts p; exit 0 }
+      end
+
+      o[:files] = @parser.parse(argv)
+      o
+    end
+
+    def parse_level(v)
+      return v.to_sym if NAMED_LEVELS.include?(v)
+      Integer(v)
+    rescue ArgumentError
+      raise OptionParser::InvalidArgument, "invalid level: #{v}"
+    end
+
+    def parse_size(v)
+      m = v.strip.match(/\A(\d+)\s*([kmg]?)b?\z/i) or
+        raise OptionParser::InvalidArgument, "invalid size: #{v}"
+      mult = { "" => 1, "k" => 1024, "m" => 1024**2, "g" => 1024**3 }
+      bytes = m[1].to_i * mult[m[2].downcase]
+      if bytes <= 0
+        raise OptionParser::InvalidArgument, "max-output must be greater than zero: #{v}"
+      end
+      if bytes > MAX_OUTPUT_LIMIT
+        raise OptionParser::InvalidArgument, "max-output too large (limit #{MAX_OUTPUT_LIMIT} bytes): #{v}"
+      end
+      bytes
+    end
+
+    def validate_invocation!(files, opts)
+      return if opts[:output].nil?
+
+      if opts[:stdout]
+        raise OptionParser::InvalidOption, "-o and -c are mutually exclusive"
+      end
+      if files.size > 1
+        raise ArgumentError, "-o cannot be used with multiple input files (would overwrite the same output)"
+      end
+    end
+
+    def process_file(path, opts)
+      raise Errno::ENOENT, path unless File.exist?(path)
+
+      if opts[:stdout]
+        File.open(path, "rb") { |input| run_pipeline(input, $stdout, resolve_algo(path, opts), opts) }
+        return
+      end
+
+      out_path = opts[:output] || default_out_path(path, opts)
+      guard_paths!(path, out_path, opts)
+
+      in_size = File.size(path)
+      started = Time.now
+      write_atomically(out_path) do |tmp|
+        File.open(path, "rb") { |input| run_pipeline(input, tmp, resolve_algo(path, opts), opts) }
+      end
+
+      report(path, out_path, in_size, File.size(out_path), started, opts)
+      File.delete(path) unless opts[:keep]
+    end
+
+    def process_stream(input, output, opts)
+      run_pipeline(input, output, opts[:algo] || :zstd, opts)
+    end
+
+    def run_pipeline(input, output, algo, opts)
+      if opts[:decompress]
+        reader_opts = {}
+        reader_opts[:max_output_size] = opts[:max_output_size] if opts[:max_output_size]
+        reader = MultiCompress::Reader.new(input, algo: algo, **reader_opts)
+        while (chunk = reader.read(READ_CHUNK))
+          output.write(chunk)
+        end
+        reader.close
+      else
+        writer = MultiCompress::Writer.new(output, algo: algo, level: opts[:level])
+        while (chunk = input.read(READ_CHUNK))
+          writer.write(chunk)
+        end
+        writer.close
+      end
+    end
+
+    def write_atomically(dst)
+      dir  = File.dirname(dst)
+      tmp  = unique_temp(dir)
+      file = File.open(tmp, File::WRONLY | File::CREAT | File::EXCL | File::BINARY, 0o644)
+      begin
+        yield file
+        file.flush
+        file.fsync
+        file.close
+        File.rename(tmp, dst)
+        fsync_dir(dir)
+      rescue StandardError
+        file.close unless file.closed?
+        File.delete(tmp) if File.exist?(tmp)
+        raise
+      end
+    end
+
+    def unique_temp(dir)
+      loop do
+        candidate = File.join(dir, ".multi_compress.#{Process.pid}.#{SecureRandom.hex(8)}.tmp")
+        return candidate unless File.exist?(candidate)
+      end
+    end
+
+    def fsync_dir(dir)
+      d = File.open(dir)
+      begin
+        d.fsync
+      rescue NotImplementedError, Errno::EINVAL, Errno::EACCES
+        # some platforms/filesystems can't fsync a directory; best-effort
+      ensure
+        d.close
+      end
+    end
+
+    def guard_paths!(src, dst, opts)
+      if same_file?(src, dst)
+        raise ArgumentError, "input and output are the same file: #{src}"
+      end
+      if File.exist?(dst) && !opts[:force]
+        raise ArgumentError, "#{dst} already exists (use -f to overwrite)"
+      end
+    end
+
+    def same_file?(a, b)
+      return true if File.expand_path(a) == File.expand_path(b)
+      File.exist?(a) && File.exist?(b) && File.identical?(a, b)
+    end
+
+    def resolve_algo(path, opts)
+      opts[:algo] || KNOWN_EXT[File.extname(path).downcase] || :zstd
+    end
+
+    def default_out_path(path, opts)
+      if opts[:decompress]
+        stripped = strip_known_ext(path)
+        stripped == path ? "#{path}.out" : stripped
+      else
+        "#{path}#{EXT.fetch(resolve_algo(path, opts))}"
+      end
+    end
+
+    def strip_known_ext(path)
+      ext = File.extname(path).downcase
+      KNOWN_EXT.key?(ext) ? path[0...-ext.length] : path
+    end
+
+    def report(src, dst, in_size, out_size, started, opts)
+      return if opts[:quiet]
+      pct = in_size.zero? ? 0 : (100.0 * out_size / in_size)
+      dir = opts[:decompress] ? "expanded" : "reduced"
+      warn format("%s -> %s  %d -> %d bytes (%s to %.1f%%, %.2fs)",
+                  src, dst, in_size, out_size, dir, pct, Time.now - started)
+    end
+  end
+end
+
+exit MultiCompress::CLI.run(ARGV)

data/ext/multi_compress/extconf.rb

@@ -253,6 +253,10 @@ end
 $CFLAGS += " -O3"
 $CFLAGS += " -DXXH_NAMESPACE=MULTICOMPRESS_"
 
+if RUBY_PLATFORM.include?("darwin") && try_cflags("-Wno-default-const-init-field-unsafe")
+  $CFLAGS += " -Wno-default-const-init-field-unsafe"
+end
+
 case RUBY_PLATFORM
 when /x86_64|amd64|aarch64|arm64/
   $CFLAGS += " -DBROTLI_BUILD_LITTLE_ENDIAN"

data/ext/multi_compress/multi_compress.c

@@ -1560,6 +1560,35 @@ static VALUE compress_compress(int argc, VALUE *argv, VALUE self) {
     return Qnil;
 }
 
+static VALUE zstd_single_frame_info(VALUE self, VALUE data) {
+    (void)self;
+    StringValue(data);
+
+    const void *src = RSTRING_PTR(data);
+    size_t src_len = (size_t)RSTRING_LEN(data);
+    size_t frame_size = ZSTD_findFrameCompressedSize(src, src_len);
+    if (ZSTD_isError(frame_size)) {
+        rb_raise(eDataError, "zstd: invalid frame: %s", ZSTD_getErrorName(frame_size));
+    }
+    if (frame_size != src_len) {
+        rb_raise(eDataError, "zstd: expected exactly one frame with no trailing bytes");
+    }
+
+    unsigned long long content_size = ZSTD_getFrameContentSize(src, src_len);
+    if (content_size == ZSTD_CONTENTSIZE_ERROR) {
+        rb_raise(eDataError, "zstd: invalid frame content size");
+    }
+    if (content_size == ZSTD_CONTENTSIZE_UNKNOWN) {
+        rb_raise(eDataError, "zstd: frame must declare content size");
+    }
+
+    VALUE result = rb_ary_new_capa(2);
+    rb_ary_push(result, SIZET2NUM(frame_size));
+    rb_ary_push(result, ULL2NUM(content_size));
+    RB_GC_GUARD(data);
+    return result;
+}
+
 static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
     VALUE data, opts;
     scan_one_required_keywords(argc, argv, &data, &opts);
@@ -3478,6 +3507,7 @@ void Init_multi_compress(void) {
 
     rb_define_module_function(mMultiCompress, "compress", compress_compress, -1);
     rb_define_module_function(mMultiCompress, "decompress", compress_decompress, -1);
+    rb_define_module_function(mMultiCompress, "zstd_single_frame_info", zstd_single_frame_info, 1);
     rb_define_module_function(mMultiCompress, "crc32", compress_crc32, -1);
     rb_define_module_function(mMultiCompress, "adler32", compress_adler32, -1);
     rb_define_module_function(mMultiCompress, "algorithms", compress_algorithms, 0);

data/lib/multi_compress/active_record.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "multi_compress/codec"
+
+begin
+  require "active_model/type"
+rescue LoadError => e
+  raise LoadError, "multi_compress/active_record requires activemodel (#{e.message}). " \
+                   "Add activerecord/activemodel to your bundle, or use MultiCompress::Codec directly."
+end
+
+module MultiCompress
+  module ActiveRecordSupport
+    class Type < ActiveModel::Type::Value
+      def initialize(mutable: false, **codec_opts)
+        @codec   = MultiCompress::Codec.new(**codec_opts)
+        @mutable = mutable
+        super()
+      end
+
+      def type
+        :multi_compress
+      end
+
+      # user assignment -> in-memory value (no compression here)
+      def cast(value)
+        value
+      end
+
+      # DB -> in-memory (unwrap)
+      def deserialize(value)
+        @codec.load(value)
+      end
+
+      # in-memory -> DB (compress)
+      def serialize(value)
+        @codec.dump(value)
+      end
+
+      def changed_in_place?(raw_old_value, new_value)
+        return false unless @mutable
+
+        deserialize(raw_old_value) != new_value
+      end
+    end
+
+    class Coder
+      def initialize(**codec_opts)
+        @codec = MultiCompress::Codec.new(**codec_opts)
+      end
+
+      def dump(value)
+        @codec.dump(value)
+      end
+
+      def load(value)
+        @codec.load(value)
+      end
+    end
+  end
+end

data/lib/multi_compress/codec.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "multi_compress"
+
+module MultiCompress
+  class Codec
+    MAGIC       = "\x89MCDB".b       # reserved 5-byte namespace; 0x89 is an
+                                     # invalid UTF-8 lead byte, so it won't
+                                     # collide with valid text. For BINARY data
+                                     # this prefix is reserved, not collision-free.
+    VERSION     = 1
+    HEADER_SIZE = MAGIC.bytesize + 2 # magic + version byte + algo byte
+    B64_PREFIX  = "mc1:"
+    MAX_NATIVE_OUTPUT = (2**(0.size * 8 - 1)) - 1
+
+    ALGO_ID  = { zstd: 1, lz4: 2, brotli: 3 }.freeze
+    ID_ALGO  = ALGO_ID.invert.freeze
+
+    attr_reader :algo, :level, :max_output_size
+
+    def initialize(algo: :zstd, level: nil, encode: nil, serializer: nil,
+                   encoding: Encoding::UTF_8, max_output_size: nil,
+                   legacy: :reject, dictionary: nil)
+      raise ArgumentError, "unknown algo #{algo.inspect}" unless ALGO_ID.key?(algo)
+      raise ArgumentError, "encode must be nil or :base64" unless [nil, :base64].include?(encode)
+
+      @algo            = algo
+      @level           = level
+      @encode          = encode
+      @serializer      = serializer
+      @encoding        = encoding
+      limit            = Integer(max_output_size || MultiCompress.config.max_output_size)
+      raise ArgumentError, "max_output_size must be greater than 0" unless limit.positive?
+      if limit > MAX_NATIVE_OUTPUT
+        raise ArgumentError,
+              "max_output_size must be at most #{MAX_NATIVE_OUTPUT} bytes on this platform"
+      end
+
+      @max_output_size = limit
+      @legacy          = normalize_legacy(legacy)
+      @dictionary      = dictionary
+    end
+
+    def dump(value)
+      return nil if value.nil?
+
+      payload = serialize_payload(value)
+      body    = MultiCompress.compress(payload, algo: @algo, level: @level, dictionary: @dictionary)
+      framed  = header + body
+
+      @encode == :base64 ? B64_PREFIX + [framed].pack("m0") : framed
+    end
+
+    def load(stored)
+      return nil if stored.nil?
+
+      framed = decode_transport(stored)
+      return decode_payload(unwrap(framed)) if framed
+
+      read_legacy(stored)
+    end
+
+    private
+
+    def header
+      (MAGIC + [VERSION, ALGO_ID.fetch(@algo)].pack("C2")).b
+    end
+
+    def decode_transport(stored)
+      if @encode == :base64
+        s = stored.to_s
+        return nil unless s.start_with?(B64_PREFIX)
+
+        decoded = optional_base64(s.byteslice(B64_PREFIX.bytesize, s.bytesize - B64_PREFIX.bytesize))
+        decoded && decoded.start_with?(MAGIC) ? decoded : nil
+      else
+        b = stored.b
+        b.start_with?(MAGIC) ? b : nil
+      end
+    end
+
+    def unwrap(framed)
+      if framed.bytesize < HEADER_SIZE
+        raise MultiCompress::DataError, "MultiCompress::Codec: truncated envelope"
+      end
+
+      version = framed.getbyte(MAGIC.bytesize)
+      algo_id = framed.getbyte(MAGIC.bytesize + 1)
+      unless version == VERSION
+        raise MultiCompress::DataError, "MultiCompress::Codec: unsupported envelope version #{version}"
+      end
+      algo = ID_ALGO[algo_id]
+      raise MultiCompress::DataError, "MultiCompress::Codec: unknown algorithm id #{algo_id}" unless algo
+
+      body = framed.byteslice(HEADER_SIZE, framed.bytesize - HEADER_SIZE) || +"".b
+      MultiCompress.decompress(body, algo: algo, dictionary: @dictionary, max_output_size: @max_output_size)
+    end
+
+    def read_legacy(stored)
+      case @legacy[:mode]
+      when :reject
+        raise MultiCompress::DataError,
+              "MultiCompress::Codec: value has no envelope and legacy: :reject is set"
+      when :plain
+        decode_payload(stored.b.dup)
+      when :compressed
+        payload = MultiCompress.decompress(
+          stored.b, algo: @legacy[:algo], dictionary: @dictionary, max_output_size: @max_output_size
+        )
+        decode_payload(payload)
+      end
+    end
+
+    def normalize_legacy(legacy)
+      case legacy
+      when :reject, :plain
+        { mode: legacy }
+      when Hash
+        algo = legacy[:compressed]
+        raise ArgumentError, "legacy: { compressed: <algo> } required" unless ALGO_ID.key?(algo)
+
+        { mode: :compressed, algo: algo }
+      else
+        raise ArgumentError, "legacy: must be :reject, :plain, or { compressed: :algo }"
+      end
+    end
+
+    def serialize_payload(value)
+      if @serializer
+        @serializer.dump(value).b
+      else
+        unless value.is_a?(String)
+          raise TypeError, "MultiCompress::Codec expects a String (or pass serializer:); got #{value.class}"
+        end
+
+        unless @encoding == Encoding::BINARY || value.dup.force_encoding(@encoding).valid_encoding?
+          raise ArgumentError, "MultiCompress::Codec: value is not valid #{@encoding} " \
+                               "(use encoding: Encoding::BINARY for arbitrary bytes)"
+        end
+        value.b
+      end
+    end
+
+    def decode_payload(bytes)
+      return @serializer.load(bytes) if @serializer
+
+      finalize_string(bytes)
+    end
+
+    def finalize_string(bytes)
+      s = bytes.dup.force_encoding(@encoding)
+      unless @encoding == Encoding::BINARY || s.valid_encoding?
+        raise MultiCompress::DataError, "MultiCompress::Codec: decoded bytes are not valid #{@encoding}"
+      end
+      s
+    end
+
+    def optional_base64(str)
+      str.unpack1("m0")&.b
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/multi_compress/database.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "multi_compress"
+
+module MultiCompress
+  module Database
+    MAGIC        = "MCDB".b
+    VERSION      = 1
+    CODEC_ZSTD   = 1
+    FLAGS_V1     = 0
+    HEADER_SIZE  = 19                    # 4 magic +1 ver +1 codec +1 flags +8 size +4 crc32
+    MAX_OUTPUT   = 16 * 1024 * 1024      # 16 MiB decompressed, enforced on write and read
+    MAX_ENVELOPE = 16_842_771
+    ZSTD_LEVEL   = 6
+
+    module_function
+
+    def compress(text)
+      raise TypeError, "MultiCompress::Database.compress expects a String, got #{text.class}" unless text.is_a?(String)
+
+      bytes = utf8_bytes(text)
+      n = bytes.bytesize
+      if n > MAX_OUTPUT
+        raise ArgumentError, "MultiCompress::Database: input is #{n} bytes, over the #{MAX_OUTPUT}-byte v1 limit"
+      end
+
+      frame = MultiCompress.compress(bytes, algo: :zstd, level: ZSTD_LEVEL)
+      validate_canonical_zstd_frame!(frame, n)
+
+      envelope = header(n, MultiCompress.crc32(bytes)) << frame
+      if envelope.bytesize > MAX_ENVELOPE
+        raise MultiCompress::DataError,
+              "MultiCompress::Database: stored envelope is #{envelope.bytesize} bytes, over the #{MAX_ENVELOPE}-byte v1 limit"
+      end
+
+      envelope
+    end
+
+    def decompress(blob)
+      raise TypeError, "MultiCompress::Database.decompress expects a String, got #{blob.class}" unless blob.is_a?(String)
+
+      b = blob.b
+      err("truncated envelope (#{b.bytesize} < #{HEADER_SIZE} bytes)") if b.bytesize < HEADER_SIZE
+      err("stored envelope too large (#{b.bytesize} > #{MAX_ENVELOPE} bytes)") if b.bytesize > MAX_ENVELOPE
+      err("bad magic")                unless b.byteslice(0, 4) == MAGIC
+      err("unsupported version #{b.getbyte(4)}") unless b.getbyte(4) == VERSION
+      err("unsupported codec #{b.getbyte(5)}")   unless b.getbyte(5) == CODEC_ZSTD
+      err("reserved flags must be 0, got #{b.getbyte(6)}") unless b.getbyte(6) == FLAGS_V1
+
+      original_size = b.byteslice(7, 8).unpack1("Q<")
+      err("declared size #{original_size} over the #{MAX_OUTPUT}-byte v1 limit") if original_size > MAX_OUTPUT
+      expected_crc = b.byteslice(15, 4).unpack1("L<")
+
+      frame = b.byteslice(HEADER_SIZE, b.bytesize - HEADER_SIZE) || +"".b
+      validate_canonical_zstd_frame!(frame, original_size)
+      out = MultiCompress.decompress(frame, algo: :zstd, max_output_size: MAX_OUTPUT)
+      if out.bytesize != original_size
+        err("size mismatch: header says #{original_size}, got #{out.bytesize}")
+      end
+      actual_crc = MultiCompress.crc32(out)
+      err("crc32 mismatch: header #{expected_crc}, computed #{actual_crc}") if actual_crc != expected_crc
+
+      s = out.force_encoding(Encoding::UTF_8)
+      err("decompressed bytes are not valid UTF-8") unless s.valid_encoding?
+      err("decompressed text contains a NUL byte") if s.include?("\0")
+      s
+    end
+
+    def valid?(blob)
+      decompress(blob)
+      true
+    rescue StandardError
+      false
+    end
+
+    def version_string
+      "MCDB1 (zstd #{MultiCompress.version(:zstd)})"
+    end
+
+    def utf8_bytes(text)
+      s = text.encoding == Encoding::UTF_8 ? text : text.dup.force_encoding(Encoding::UTF_8)
+      raise ArgumentError, "MultiCompress::Database: input is not valid UTF-8" unless s.valid_encoding?
+      raise ArgumentError, "MultiCompress::Database: input contains a NUL byte" if s.include?("\0")
+
+      s.b
+    end
+
+    def validate_canonical_zstd_frame!(frame, expected_size)
+      _frame_size, content_size = MultiCompress.zstd_single_frame_info(frame)
+      err("zstd frame content size #{content_size} does not match header size #{expected_size}") if content_size != expected_size
+    rescue MultiCompress::DataError => e
+      err(e.message.sub(/\AMultiCompress::Database: /, ""))
+    end
+
+    def header(original_size, crc32)
+      (MAGIC + [VERSION, CODEC_ZSTD, FLAGS_V1].pack("C3") +
+        [original_size].pack("Q<") + [crc32].pack("L<")).b
+    end
+
+    def err(message)
+      raise MultiCompress::DataError, "MultiCompress::Database: #{message}"
+    end
+
+    private_class_method :utf8_bytes, :validate_canonical_zstd_frame!, :header, :err
+  end
+end