multi_compress 0.3.5 → 0.6.0

data/lib/multi_compress/codec.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "multi_compress"
+
+module MultiCompress
+  class Codec
+    MAGIC       = "\x89MCDB".b       # reserved 5-byte namespace; 0x89 is an
+                                     # invalid UTF-8 lead byte, so it won't
+                                     # collide with valid text. For BINARY data
+                                     # this prefix is reserved, not collision-free.
+    VERSION     = 1
+    HEADER_SIZE = MAGIC.bytesize + 2 # magic + version byte + algo byte
+    B64_PREFIX  = "mc1:"
+    MAX_NATIVE_OUTPUT = (2**(0.size * 8 - 1)) - 1
+
+    ALGO_ID  = { zstd: 1, lz4: 2, brotli: 3 }.freeze
+    ID_ALGO  = ALGO_ID.invert.freeze
+
+    attr_reader :algo, :level, :max_output_size
+
+    def initialize(algo: :zstd, level: nil, encode: nil, serializer: nil,
+                   encoding: Encoding::UTF_8, max_output_size: nil,
+                   legacy: :reject, dictionary: nil)
+      raise ArgumentError, "unknown algo #{algo.inspect}" unless ALGO_ID.key?(algo)
+      raise ArgumentError, "encode must be nil or :base64" unless [nil, :base64].include?(encode)
+
+      @algo            = algo
+      @level           = level
+      @encode          = encode
+      @serializer      = serializer
+      @encoding        = encoding
+      limit            = Integer(max_output_size || MultiCompress.config.max_output_size)
+      raise ArgumentError, "max_output_size must be greater than 0" unless limit.positive?
+      if limit > MAX_NATIVE_OUTPUT
+        raise ArgumentError,
+              "max_output_size must be at most #{MAX_NATIVE_OUTPUT} bytes on this platform"
+      end
+
+      @max_output_size = limit
+      @legacy          = normalize_legacy(legacy)
+      @dictionary      = dictionary
+    end
+
+    def dump(value)
+      return nil if value.nil?
+
+      payload = serialize_payload(value)
+      body    = MultiCompress.compress(payload, algo: @algo, level: @level, dictionary: @dictionary)
+      framed  = header + body
+
+      @encode == :base64 ? B64_PREFIX + [framed].pack("m0") : framed
+    end
+
+    def load(stored)
+      return nil if stored.nil?
+
+      framed = decode_transport(stored)
+      return decode_payload(unwrap(framed)) if framed
+
+      read_legacy(stored)
+    end
+
+    private
+
+    def header
+      (MAGIC + [VERSION, ALGO_ID.fetch(@algo)].pack("C2")).b
+    end
+
+    def decode_transport(stored)
+      if @encode == :base64
+        s = stored.to_s
+        return nil unless s.start_with?(B64_PREFIX)
+
+        decoded = optional_base64(s.byteslice(B64_PREFIX.bytesize, s.bytesize - B64_PREFIX.bytesize))
+        decoded && decoded.start_with?(MAGIC) ? decoded : nil
+      else
+        b = stored.b
+        b.start_with?(MAGIC) ? b : nil
+      end
+    end
+
+    def unwrap(framed)
+      if framed.bytesize < HEADER_SIZE
+        raise MultiCompress::DataError, "MultiCompress::Codec: truncated envelope"
+      end
+
+      version = framed.getbyte(MAGIC.bytesize)
+      algo_id = framed.getbyte(MAGIC.bytesize + 1)
+      unless version == VERSION
+        raise MultiCompress::DataError, "MultiCompress::Codec: unsupported envelope version #{version}"
+      end
+      algo = ID_ALGO[algo_id]
+      raise MultiCompress::DataError, "MultiCompress::Codec: unknown algorithm id #{algo_id}" unless algo
+
+      body = framed.byteslice(HEADER_SIZE, framed.bytesize - HEADER_SIZE) || +"".b
+      MultiCompress.decompress(body, algo: algo, dictionary: @dictionary, max_output_size: @max_output_size)
+    end
+
+    def read_legacy(stored)
+      case @legacy[:mode]
+      when :reject
+        raise MultiCompress::DataError,
+              "MultiCompress::Codec: value has no envelope and legacy: :reject is set"
+      when :plain
+        decode_payload(stored.b.dup)
+      when :compressed
+        payload = MultiCompress.decompress(
+          stored.b, algo: @legacy[:algo], dictionary: @dictionary, max_output_size: @max_output_size
+        )
+        decode_payload(payload)
+      end
+    end
+
+    def normalize_legacy(legacy)
+      case legacy
+      when :reject, :plain
+        { mode: legacy }
+      when Hash
+        algo = legacy[:compressed]
+        raise ArgumentError, "legacy: { compressed: <algo> } required" unless ALGO_ID.key?(algo)
+
+        { mode: :compressed, algo: algo }
+      else
+        raise ArgumentError, "legacy: must be :reject, :plain, or { compressed: :algo }"
+      end
+    end
+
+    def serialize_payload(value)
+      if @serializer
+        @serializer.dump(value).b
+      else
+        unless value.is_a?(String)
+          raise TypeError, "MultiCompress::Codec expects a String (or pass serializer:); got #{value.class}"
+        end
+
+        unless @encoding == Encoding::BINARY || value.dup.force_encoding(@encoding).valid_encoding?
+          raise ArgumentError, "MultiCompress::Codec: value is not valid #{@encoding} " \
+                               "(use encoding: Encoding::BINARY for arbitrary bytes)"
+        end
+        value.b
+      end
+    end
+
+    def decode_payload(bytes)
+      return @serializer.load(bytes) if @serializer
+
+      finalize_string(bytes)
+    end
+
+    def finalize_string(bytes)
+      s = bytes.dup.force_encoding(@encoding)
+      unless @encoding == Encoding::BINARY || s.valid_encoding?
+        raise MultiCompress::DataError, "MultiCompress::Codec: decoded bytes are not valid #{@encoding}"
+      end
+      s
+    end
+
+    def optional_base64(str)
+      str.unpack1("m0")&.b
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/multi_compress/database.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require "digest"
+require "multi_compress"
+
+module MultiCompress
+  # Long-term, SQL-readable envelopes. MCDB1 is dictionary-free. MCDB2 is an
+  # opt-in, dictionary-required format for one homogeneous database column.
+  module Database
+    MAGIC        = "MCDB".b
+    VERSION_V1   = 1
+    VERSION_V2   = 2
+    VERSION      = VERSION_V1 # MCDB1 compatibility alias
+    CODEC_ZSTD   = 1
+    FLAGS_NONE   = 0
+    FLAGS_V1     = FLAGS_NONE
+    V1_HEADER_SIZE = 19 # 4 magic + 1 version + 1 codec + 1 flags + 8 size + 4 crc32
+    V2_HEADER_SIZE = 27 # MCDB1 header + 8 byte application dictionary_ref
+    HEADER_SIZE    = V1_HEADER_SIZE
+    ORIGINAL_SIZE_OFFSET = 7
+    CRC_OFFSET           = 15
+    DICTIONARY_REF_OFFSET = 19
+    MAX_OUTPUT   = 16 * 1024 * 1024
+    MAX_DICTIONARY_BYTES = 256 * 1024
+    MAX_DICTIONARY_REF = (2**63) - 1
+    MAX_ENVELOPE_V1 = 16_842_771
+    MAX_ENVELOPE_V2 = 16_842_779
+    MAX_ENVELOPE = MAX_ENVELOPE_V2
+    ZSTD_LEVEL   = 6
+
+    # A strict, immutable MCDB2 dictionary. It is intentionally distinct from
+    # MultiCompress::Dictionary: MCDB2 accepts only conformant zstd dictionaries
+    # with a non-zero zstd DictID and an application-level registry id.
+    class Dictionary
+      attr_reader :id, :bytes, :sha256, :zstd_id, :native
+
+      def self.train(samples, id:, size: 32 * 1024)
+        new(id: id, native: MultiCompress::Zstd.train_dictionary(samples, size: size))
+      end
+
+      def self.wrap(native, id:)
+        new(id: id, native: native)
+      end
+
+      def initialize(id:, native: nil, bytes: nil)
+        @id = normalize_id(id)
+        raise ArgumentError, "provide exactly one of native: or bytes:" if (!!native) == (!!bytes)
+
+        @native = native || MultiCompress::Dictionary.new(bytes, algo: :zstd)
+        unless @native.is_a?(MultiCompress::Dictionary) && @native.algo == :zstd
+          raise TypeError, "MCDB2 dictionary must be a MultiCompress::Dictionary with algo: :zstd"
+        end
+
+        @bytes = @native.bytes.b.freeze
+        if @bytes.empty? || @bytes.bytesize > MAX_DICTIONARY_BYTES
+          raise ArgumentError,
+                "MCDB2 dictionary is #{@bytes.bytesize} bytes; allowed range is 1..#{MAX_DICTIONARY_BYTES}"
+        end
+
+        @zstd_id = @native.zstd_id
+        raise ArgumentError, "MCDB2 dictionary must be a conformant zstd dictionary with a non-zero DictID" if @zstd_id.zero?
+
+        @sha256 = Digest::SHA256.digest(@bytes).freeze
+        freeze
+      end
+
+      def sha256_hex
+        @sha256.unpack1("H*")
+      end
+
+      def registry_attributes
+        { id: @id, zstd_dict_id: @zstd_id, sha256: @sha256, bytes: @bytes }
+      end
+
+      private
+
+      def normalize_id(value)
+        raise TypeError, "MCDB2 dictionary id must be an Integer" unless value.respond_to?(:to_int)
+
+        integer = value.to_int
+        unless integer.between?(1, MAX_DICTIONARY_REF)
+          raise ArgumentError, "MCDB2 dictionary id must be in 1..#{MAX_DICTIONARY_REF}"
+        end
+        integer
+      end
+    end
+
+    module_function
+
+    # Without a dictionary this emits MCDB1. With Database::Dictionary it emits
+    # MCDB2. Choose one format per database column; mixed use is deliberately
+    # not what the generated dictionary view is optimized for.
+    def compress(text, dictionary: nil)
+      return compress_v1(text) if dictionary.nil?
+
+      compress_v2(text, dictionary: dictionary)
+    end
+
+    def compress_v1(text)
+      bytes = input_bytes!(text, format: "v1")
+      frame = MultiCompress.compress(bytes, algo: :zstd, level: ZSTD_LEVEL)
+      validate_canonical_zstd_frame!(frame, bytes.bytesize, expected_dictionary_id: 0)
+      envelope = header_v1(bytes.bytesize, MultiCompress.crc32(bytes)) << frame
+      check_envelope_size!(envelope, MAX_ENVELOPE_V1, "v1")
+      envelope
+    end
+
+    def compress_v2(text, dictionary:)
+      dictionary = database_dictionary!(dictionary)
+      bytes = input_bytes!(text, format: "v2")
+      frame = MultiCompress.compress(bytes, algo: :zstd, level: ZSTD_LEVEL, dictionary: dictionary.native)
+      validate_canonical_zstd_frame!(frame, bytes.bytesize, expected_dictionary_id: dictionary.zstd_id)
+      envelope = header_v2(bytes.bytesize, MultiCompress.crc32(bytes), dictionary.id) << frame
+      check_envelope_size!(envelope, MAX_ENVELOPE_V2, "v2")
+      envelope
+    end
+
+    def decompress(blob, dictionary: nil)
+      header = parse_header!(blob)
+      case header.fetch(:version)
+      when VERSION_V1
+        raise ArgumentError, "MCDB1 does not accept a dictionary" unless dictionary.nil?
+
+        decode_v1(blob.b, header)
+      when VERSION_V2
+        decode_v2(blob.b, header, dictionary)
+      else
+        err("unsupported version #{header.fetch(:version)}")
+      end
+    end
+
+    def valid?(blob, dictionary: nil)
+      decompress(blob, dictionary: dictionary)
+      true
+    rescue StandardError
+      false
+    end
+
+    # Parses only the envelope header. It is safe for registry/FK consistency
+    # checks and does not decompress the payload.
+    def dictionary_ref(blob)
+      header = parse_header!(blob)
+      return nil if header.fetch(:version) == VERSION_V1
+
+      header.fetch(:dictionary_ref)
+    end
+
+    def original_size(blob)
+      parse_header!(blob).fetch(:original_size)
+    end
+
+    def version_string
+      "MCDB1 + MCDB2 (zstd #{MultiCompress.version(:zstd)})"
+    end
+
+    def input_bytes!(text, format:)
+      raise TypeError, "MultiCompress::Database.compress expects a String, got #{text.class}" unless text.is_a?(String)
+
+      bytes = utf8_bytes(text)
+      n = bytes.bytesize
+      if n > MAX_OUTPUT
+        raise ArgumentError, "MultiCompress::Database: input is #{n} bytes, over the #{MAX_OUTPUT}-byte #{format} limit"
+      end
+      bytes
+    end
+    private_class_method :input_bytes!
+
+    def decode_v1(blob, header)
+      frame = blob.byteslice(V1_HEADER_SIZE, blob.bytesize - V1_HEADER_SIZE) || +"".b
+      validate_canonical_zstd_frame!(frame, header.fetch(:original_size), expected_dictionary_id: 0)
+      out = MultiCompress.decompress(frame, algo: :zstd, max_output_size: MAX_OUTPUT)
+      validate_decoded!(blob, header, out)
+    end
+    private_class_method :decode_v1
+
+    def decode_v2(blob, header, dictionary)
+      err("MCDB2 requires a dictionary") if dictionary.nil?
+      dictionary = database_dictionary!(dictionary)
+      if dictionary.id != header.fetch(:dictionary_ref)
+        err("dictionary reference #{header.fetch(:dictionary_ref)} does not match supplied dictionary #{dictionary.id}")
+      end
+
+      frame = blob.byteslice(V2_HEADER_SIZE, blob.bytesize - V2_HEADER_SIZE) || +"".b
+      validate_canonical_zstd_frame!(frame, header.fetch(:original_size), expected_dictionary_id: dictionary.zstd_id)
+      out = MultiCompress.decompress(
+        frame,
+        algo: :zstd,
+        dictionary: dictionary.native,
+        max_output_size: MAX_OUTPUT
+      )
+      validate_decoded!(blob, header, out)
+    end
+    private_class_method :decode_v2
+
+    def parse_header!(blob)
+      raise TypeError, "MultiCompress::Database.decompress expects a String, got #{blob.class}" unless blob.is_a?(String)
+
+      b = blob.b
+      err("truncated envelope (#{b.bytesize} < #{V1_HEADER_SIZE} bytes)") if b.bytesize < V1_HEADER_SIZE
+      err("bad magic") unless b.byteslice(0, 4) == MAGIC
+
+      version = b.getbyte(4)
+      case version
+      when VERSION_V1
+        header_size = V1_HEADER_SIZE
+        max_envelope = MAX_ENVELOPE_V1
+      when VERSION_V2
+        header_size = V2_HEADER_SIZE
+        max_envelope = MAX_ENVELOPE_V2
+      else
+        err("unsupported version #{version}")
+      end
+
+      err("truncated #{format_name(version)} envelope (#{b.bytesize} < #{header_size} bytes)") if b.bytesize < header_size
+      err("stored #{format_name(version)} envelope is #{b.bytesize} bytes, over the #{max_envelope}-byte limit") if b.bytesize > max_envelope
+      err("unsupported codec #{b.getbyte(5)}") unless b.getbyte(5) == CODEC_ZSTD
+      err("reserved flags must be 0, got #{b.getbyte(6)}") unless b.getbyte(6) == FLAGS_NONE
+
+      original_size = b.byteslice(ORIGINAL_SIZE_OFFSET, 8).unpack1("Q<")
+      err("declared size #{original_size} over the #{MAX_OUTPUT}-byte limit") if original_size > MAX_OUTPUT
+
+      header = {
+        version: version,
+        header_size: header_size,
+        original_size: original_size,
+        expected_crc: b.byteslice(CRC_OFFSET, 4).unpack1("L<")
+      }
+      if version == VERSION_V2
+        dictionary_ref = b.byteslice(DICTIONARY_REF_OFFSET, 8).unpack1("Q<")
+        unless dictionary_ref.between?(1, MAX_DICTIONARY_REF)
+          err("invalid dictionary reference #{dictionary_ref}")
+        end
+        header[:dictionary_ref] = dictionary_ref
+      end
+      header
+    end
+    private_class_method :parse_header!
+
+    def validate_decoded!(blob, header, out)
+      if out.bytesize != header.fetch(:original_size)
+        err("size mismatch: header says #{header.fetch(:original_size)}, got #{out.bytesize}")
+      end
+      actual_crc = MultiCompress.crc32(out)
+      if actual_crc != header.fetch(:expected_crc)
+        err("crc32 mismatch: header #{header.fetch(:expected_crc)}, computed #{actual_crc}")
+      end
+      s = out.force_encoding(Encoding::UTF_8)
+      err("decompressed bytes are not valid UTF-8") unless s.valid_encoding?
+      err("decompressed text contains a NUL byte") if s.include?("\0")
+      s
+    end
+    private_class_method :validate_decoded!
+
+    def utf8_bytes(text)
+      s = text.encoding == Encoding::UTF_8 ? text : text.dup.force_encoding(Encoding::UTF_8)
+      raise ArgumentError, "MultiCompress::Database: input is not valid UTF-8" unless s.valid_encoding?
+      raise ArgumentError, "MultiCompress::Database: input contains a NUL byte" if s.include?("\0")
+
+      s.b
+    end
+    private_class_method :utf8_bytes
+
+    def validate_canonical_zstd_frame!(frame, expected_size, expected_dictionary_id:)
+      _frame_size, content_size = MultiCompress.zstd_single_frame_info(frame)
+      err("zstd frame content size #{content_size} does not match header size #{expected_size}") if content_size != expected_size
+      actual_dictionary_id = MultiCompress.zstd_frame_dictionary_id(frame)
+      if actual_dictionary_id != expected_dictionary_id
+        err("zstd frame dictionary id #{actual_dictionary_id} does not match expected #{expected_dictionary_id}")
+      end
+    rescue MultiCompress::DataError => e
+      err(e.message.sub(/\AMultiCompress::Database: /, ""))
+    end
+    private_class_method :validate_canonical_zstd_frame!
+
+    def database_dictionary!(value)
+      return value if value.is_a?(Dictionary)
+
+      raise TypeError, "MCDB2 requires MultiCompress::Database::Dictionary, got #{value.class}"
+    end
+    private_class_method :database_dictionary!
+
+    def check_envelope_size!(envelope, max, format)
+      return if envelope.bytesize <= max
+
+      raise MultiCompress::DataError,
+            "MultiCompress::Database: stored #{format} envelope is #{envelope.bytesize} bytes, over the #{max}-byte limit"
+    end
+    private_class_method :check_envelope_size!
+
+    def header_v1(original_size, crc32)
+      (MAGIC + [VERSION_V1, CODEC_ZSTD, FLAGS_NONE].pack("C3") +
+        [original_size].pack("Q<") + [crc32].pack("L<")).b
+    end
+    private_class_method :header_v1
+
+    def header_v2(original_size, crc32, dictionary_ref)
+      (MAGIC + [VERSION_V2, CODEC_ZSTD, FLAGS_NONE].pack("C3") +
+        [original_size].pack("Q<") + [crc32].pack("L<") + [dictionary_ref].pack("Q<")).b
+    end
+    private_class_method :header_v2
+
+    def format_name(version)
+      version == VERSION_V2 ? "MCDB2" : "MCDB1"
+    end
+    private_class_method :format_name
+
+    def err(message)
+      raise MultiCompress::DataError, "MultiCompress::Database: #{message}"
+    end
+    private_class_method :err
+  end
+end