multi_compress 0.5.0 → 0.6.1

data/lib/multi_compress/writer.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  class Writer
+    CHUNK_BUFFER_SIZE = 8192
+
+    def self.open(path_or_io, algo: nil, level: nil, dictionary: nil, &block)
+      io, algo, owned = StreamIO.resolve_io(path_or_io, algo, mode: "wb")
+      writer = new(io, algo: algo || :zstd, level: level, dictionary: dictionary)
+      writer.instance_variable_set(:@owned_io, owned)
+
+      return writer unless block
+
+      begin
+        yield writer
+      ensure
+        writer.close
+      end
+    end
+
+    def initialize(io, algo: :zstd, level: nil, dictionary: nil)
+      @io       = io
+      @deflater = Deflater.new(algo: algo, level: level, dictionary: dictionary)
+      @closed   = false
+      @owned_io = false
+    end
+
+    def write(data)
+      ensure_open!
+      bytes = data.to_s
+      flush_compressed(@deflater.write(bytes))
+      bytes.bytesize
+    end
+
+    def flush
+      ensure_open!
+      flush_compressed(@deflater.flush)
+      @io.flush if @io.respond_to?(:flush)
+      self
+    end
+
+    def close
+      return if @closed
+
+      flush_compressed(@deflater.finish)
+      @deflater.close
+      @io.close if @owned_io && @io.respond_to?(:close)
+      @closed = true
+      nil
+    end
+
+    def closed?
+      @closed
+    end
+
+    def <<(data)
+      write(data)
+      self
+    end
+
+    def puts(*args)
+      args.flatten.each do |arg|
+        str = arg.to_s
+        write(str)
+        write("\n") unless str.end_with?("\n")
+      end
+      nil
+    end
+
+    def print(*args)
+      args.each { |arg| write(arg.to_s) }
+      nil
+    end
+
+    private
+
+    def ensure_open!
+      raise StreamError, "writer is closed" if @closed
+    end
+
+    def flush_compressed(compressed)
+      @io.write(compressed) if compressed && !compressed.empty?
+    end
+  end
+end

data/lib/multi_compress.rb

@@ -5,11 +5,11 @@ require "multi_compress/version"
 begin
   require "multi_compress/multi_compress"
 rescue LoadError
-  ext_dir = File.expand_path("../multi_compress", __FILE__)
+  ext_dir = File.expand_path("multi_compress", __dir__)
 
   so_path = %w[.bundle .so]
-    .map { |ext| File.join(ext_dir, "multi_compress#{ext}") }
-    .find { |path| File.exist?(path) }
+            .map { |ext| File.join(ext_dir, "multi_compress#{ext}") }
+            .find { |path| File.exist?(path) }
 
   raise LoadError, "Could not find compiled MultiCompress extension. Run: bundle exec rake compile" unless so_path
 
@@ -25,10 +25,10 @@ module MultiCompress
   DEFAULT_STREAMING_MAX_OUTPUT_SIZE = 2 * 1024 * 1024 * 1024
 
   EXTENSION_MAP = {
-    ".zst"  => :zstd,
+    ".zst" => :zstd,
     ".zstd" => :zstd,
-    ".lz4"  => :lz4,
-    ".br"   => :brotli,
+    ".lz4" => :lz4,
+    ".br" => :brotli
   }.freeze
 
   private_constant :EXTENSION_MAP
@@ -91,361 +91,70 @@ module MultiCompress
     end
 
     unless private_method_defined?(:_c_decompress)
-      alias_method :_c_decompress, :decompress
+      alias _c_decompress decompress
       private :_c_decompress
     end
-  end
-
-  def self.zstd(data, level: nil)
-    compress(data, algo: :zstd, level: level)
-  end
-
-  def self.lz4(data, level: nil, format: nil)
-    if format
-      compress(data, algo: :lz4, level: level, format: format)
-    else
-      compress(data, algo: :lz4, level: level)
-    end
-  end
-
-  def self.brotli(data, level: nil)
-    compress(data, algo: :brotli, level: level)
-  end
-
-  def self.decompress(data, **opts)
-    _c_decompress(data, **resolved_one_shot_options(opts))
-  end
-
-  def self.zstd_decompress(data)
-    decompress(data, algo: :zstd)
-  end
-
-  def self.lz4_decompress(data, format: nil)
-    if format
-      decompress(data, algo: :lz4, format: format)
-    else
-      decompress(data, algo: :lz4)
-    end
-  end
-
-  def self.brotli_decompress(data)
-    decompress(data, algo: :brotli)
-  end
-
-  def self.algo_from_ext(path)
-    EXTENSION_MAP[File.extname(path).downcase]
-  end
-
-  def self.resolved_one_shot_options(opts)
-    return opts.merge(max_output_size: config.max_output_size) unless opts.key?(:max_output_size)
-
-    opts
-  end
-
-  private_class_method :resolved_one_shot_options
-
-  module InflaterDefaults
-    def initialize(*args, **opts)
-      resolved = opts.dup
-      resolved[:max_output_size] = MultiCompress.config.streaming_max_output_size unless resolved.key?(:max_output_size)
-      super(*args, **resolved)
-    end
-  end
-
-  Inflater.prepend(InflaterDefaults)
 
-  class Writer
-    CHUNK_BUFFER_SIZE = 8192
-
-    def self.open(path_or_io, algo: nil, level: nil, dictionary: nil, &block)
-      io, algo, owned = resolve_io(path_or_io, algo, mode: "wb")
-      writer = new(io, algo: algo || :zstd, level: level, dictionary: dictionary)
-      writer.instance_variable_set(:@owned_io, owned)
-
-      return writer unless block
-
-      begin
-        yield writer
-      ensure
-        writer.close
-      end
+    def zstd(data, level: nil)
+      compress(data, algo: :zstd, level: level)
     end
 
-    def initialize(io, algo: :zstd, level: nil, dictionary: nil)
-      @io       = io
-      @deflater = Deflater.new(algo: algo, level: level, dictionary: dictionary)
-      @closed   = false
-      @owned_io = false
-    end
-
-    def write(data)
-      ensure_open!
-      bytes = data.to_s
-      flush_compressed(@deflater.write(bytes))
-      bytes.bytesize
-    end
-
-    def flush
-      ensure_open!
-      flush_compressed(@deflater.flush)
-      @io.flush if @io.respond_to?(:flush)
-      self
-    end
-
-    def close
-      return if @closed
-
-      flush_compressed(@deflater.finish)
-      @deflater.close
-      @io.close if @owned_io && @io.respond_to?(:close)
-      @closed = true
-      nil
-    end
-
-    def closed?
-      @closed
-    end
-
-    def <<(data)
-      write(data)
-      self
-    end
-
-    def puts(*args)
-      args.flatten.each do |arg|
-        str = arg.to_s
-        write(str)
-        write("\n") unless str.end_with?("\n")
-      end
-      nil
-    end
-
-    def print(*args)
-      args.each { |arg| write(arg.to_s) }
-      nil
-    end
-
-    private
-
-    def ensure_open!
-      raise StreamError, "writer is closed" if @closed
-    end
-
-    def flush_compressed(compressed)
-      @io.write(compressed) if compressed && !compressed.empty?
-    end
-
-    def self.resolve_io(path_or_io, algo, mode:)
-      if path_or_io.is_a?(String)
-        [File.open(path_or_io, mode), algo || MultiCompress.algo_from_ext(path_or_io), true]
+    def lz4(data, level: nil, format: nil)
+      if format
+        compress(data, algo: :lz4, level: level, format: format)
       else
-        [path_or_io, algo, false]
+        compress(data, algo: :lz4, level: level)
       end
     end
 
-    private_class_method :resolve_io
-  end
-
-  class Reader
-    CHUNK_SIZE = 8192
-    BUFFER_COMPACT_THRESHOLD = 64 * 1024
-
-    def self.open(path_or_io, algo: nil, dictionary: nil, **opts, &block)
-      io, algo, owned = resolve_io(path_or_io, algo, mode: "rb")
-      reader = new(io, algo: algo, dictionary: dictionary, **opts)
-      reader.instance_variable_set(:@owned_io, owned)
-
-      return reader unless block
-
-      begin
-        yield reader
-      ensure
-        reader.close
-      end
+    def brotli(data, level: nil)
+      compress(data, algo: :brotli, level: level)
     end
 
-    def initialize(io, algo: nil, dictionary: nil, **opts)
-      @io          = io
-      @inflater    = Inflater.new(algo: algo, dictionary: dictionary, **opts)
-      @closed      = false
-      @owned_io    = false
-      @buffer      = +"".b
-      @buffer_pos  = 0
-      @eof         = false
+    def decompress(data, **opts)
+      _c_decompress(data, **resolved_one_shot_options(opts))
     end
 
-    def read(length = nil)
-      ensure_open!
-      return read_all if length.nil?
-
-      read_exactly(length)
-    end
-
-    def gets(separator = "\n")
-      ensure_open!
-      return nil if @eof && buffer_empty?
-
-      fill_buffer_until { buffer_includes?(separator) }
-
-      return extract_line(separator) if buffer_includes?(separator)
-      return consume_buffer unless buffer_empty?
-
-      nil
+    def zstd_decompress(data)
+      decompress(data, algo: :zstd)
     end
 
-    def close
-      return if @closed
-
-      @inflater.close
-      @io.close if @owned_io && @io.respond_to?(:close)
-      @closed = true
-      nil
-    end
-
-    def closed?
-      @closed
-    end
-
-    def eof?
-      @eof && buffer_empty?
-    end
-
-    def each_line
-      return enum_for(:each_line) unless block_given?
-
-      while (line = gets)
-        yield line
+    def lz4_decompress(data, format: nil)
+      if format
+        decompress(data, algo: :lz4, format: format)
+      else
+        decompress(data, algo: :lz4)
       end
     end
 
-    def each_chunk(size)
-      return enum_for(:each_chunk, size) unless block_given?
-
-      while (chunk = read(size))
-        yield chunk
-      end
+    def brotli_decompress(data)
+      decompress(data, algo: :brotli)
     end
 
-    def readline
-      gets || raise(EOFError, "end of file reached")
+    def algo_from_ext(path)
+      EXTENSION_MAP[File.extname(path).downcase]
     end
 
     private
 
-    def ensure_open!
-      raise StreamError, "reader is closed" if @closed
-    end
-
-    def buffer_size
-      @buffer.bytesize - @buffer_pos
-    end
-
-    def buffer_empty?
-      @buffer_pos >= @buffer.bytesize
-    end
-
-    def buffer_append(data)
-      compact_buffer_if_needed
-      @buffer << data
-    end
-
-    def compact_buffer_if_needed
-      return if @buffer_pos == 0
-
-      total = @buffer.bytesize
-      return unless @buffer_pos >= BUFFER_COMPACT_THRESHOLD && @buffer_pos * 2 >= total
+    def resolved_one_shot_options(opts)
+      return opts.merge(max_output_size: config.max_output_size) unless opts.key?(:max_output_size)
 
-      @buffer = @buffer.byteslice(@buffer_pos, total - @buffer_pos)
-      @buffer_pos = 0
-    end
-
-    def buffer_includes?(separator)
-      idx = @buffer.index(separator, @buffer_pos)
-      !idx.nil?
-    end
-
-    def read_all
-      return nil if @eof && buffer_empty?
-
-      result = if buffer_empty?
-        +"".b
-      else
-        @buffer.byteslice(@buffer_pos, @buffer.bytesize - @buffer_pos) || +"".b
-      end
-      @buffer.clear
-      @buffer_pos = 0
-
-      until @eof
-        chunk = read_compressed_chunk
-        break unless chunk && !chunk.empty?
-
-        decompressed = @inflater.write(chunk)
-        result << decompressed if decompressed
-      end
-
-      final = @inflater.finish
-      result << final if final && !final.empty?
-
-      result.empty? ? nil : result
-    end
-
-    def read_exactly(length)
-      return nil if @eof && buffer_empty?
-
-      fill_buffer_until { buffer_size >= length }
-
-      if buffer_size >= length
-        result = @buffer.byteslice(@buffer_pos, length)
-        @buffer_pos += length
-        compact_buffer_if_needed
-        result
-      elsif !buffer_empty?
-        consume_buffer
-      end
-    end
-
-    def fill_buffer_until
-      until yield || @eof
-        chunk = read_compressed_chunk
-        if chunk.nil? || chunk.empty?
-          @eof = true
-          break
-        end
-        decompressed = @inflater.write(chunk)
-        buffer_append(decompressed) if decompressed
-      end
-    end
-
-    def extract_line(separator)
-      idx = @buffer.index(separator, @buffer_pos)
-      end_pos = idx + separator.bytesize
-      result = @buffer.byteslice(@buffer_pos, end_pos - @buffer_pos)
-      @buffer_pos = end_pos
-      compact_buffer_if_needed
-      result
-    end
-
-    def consume_buffer
-      result = @buffer.byteslice(@buffer_pos, @buffer.bytesize - @buffer_pos) || +"".b
-      @buffer.clear
-      @buffer_pos = 0
-      result
+      opts
     end
+  end
 
-    def read_compressed_chunk
-      @io.read(CHUNK_SIZE)
-    rescue EOFError
-      nil
+  module InflaterDefaults
+    def initialize(*args, **opts)
+      resolved = opts.dup
+      resolved[:max_output_size] = MultiCompress.config.streaming_max_output_size unless resolved.key?(:max_output_size)
+      super(*args, **resolved)
     end
+  end
 
-    def self.resolve_io(path_or_io, algo, mode:)
-      if path_or_io.is_a?(String)
-        [File.open(path_or_io, mode), algo || MultiCompress.algo_from_ext(path_or_io), true]
-      else
-        [path_or_io, algo, false]
-      end
-    end
+  Inflater.prepend(InflaterDefaults)
 
-    private_class_method :resolve_io
-  end
+  require "multi_compress/stream_io"
+  require "multi_compress/writer"
+  require "multi_compress/reader"
 end

data/mysql_udf/README.md

@@ -93,3 +93,48 @@ invokes it.
 `make verify` catches accidental corruption after extraction; it does not
 authenticate an archive. Use an out-of-band SHA-256, minisign/GPG signature, or
 signed release provenance for a real trust chain.
+
+## MCDB2 dictionary-backed columns
+
+MCDB2 is opt-in for one homogeneous column with many similar small JSON/text
+values. Upgrade the UDF library first; the controlled `make upgrade` flow keeps
+all MCDB1 and MCDB2 functions registered.
+
+Dictionary bytes are application data. Generate the append-only registry in the
+application database, register immutable versions, and retain all referenced
+versions in backups and replicas:
+
+```bash
+bundle exec multi_compress db registry mysql --database app \
+  --output db/mcdb_dictionary_registry.sql
+```
+
+For a new MCDB2 column add `payload_dictionary_id BIGINT UNSIGNED NOT NULL` with
+a foreign key to `mcdb_dictionary_versions(id)`, then generate the view:
+
+```bash
+bundle exec multi_compress db view mysql \
+  --table app.events \
+  --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+The generated MCDB2 view requests `ALGORITHM=MERGE SQL SECURITY DEFINER` and uses
+a source-first `STRAIGHT_JOIN` to the registry. That keeps an indexed outer source
+predicate from starting at the registry. It passes payload, registry id, SHA-256
+and dictionary bytes to the UDF. Keep the definition free of `DISTINCT`, aggregates,
+`GROUP BY`, `UNION`, subqueries and inner `LIMIT`; validate real queries with
+`EXPLAIN`: source must be a selective `range`/`ref` access and no `<derived>` row
+may appear. MySQL 5.7 can still report `Using temporary; Using filesort` for an
+outer `ORDER BY` over the joined projection; that is sorting, not view
+materialization. A stable definer account must remain part of backup/restore
+procedures.
+
+Details are in [`docs/database-envelope-v2.md`](../docs/database-envelope-v2.md).
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/mysql_udf/sql/examples.sql

@@ -9,3 +9,8 @@ FROM events
 WHERE multi_compress_db_is_valid(payload_compressed) = 0;
 
 SELECT multi_compress_db_version();
+
+-- MCDB2: use the generated ALGORITHM=MERGE readable view. Avoid filtering on
+-- decoded payload text; filter/order by indexed source columns before LIMIT.
+--
+-- SELECT id, payload FROM admin.events_readable WHERE id >= 100 ORDER BY id LIMIT 100;

data/mysql_udf/sql/install.sql

@@ -1,4 +1,4 @@
--- Install the MultiCompress MCDB1 reader UDFs (MySQL 5.7).
+-- Install the MultiCompress MCDB1/MCDB2 reader UDFs (MySQL 5.7).
 
 CREATE FUNCTION multi_compress_db_version
   RETURNS STRING
@@ -12,4 +12,28 @@ CREATE FUNCTION multi_compress_db_decompress
   RETURNS STRING
   SONAME 'multi_compress_mysql.so';
 
+CREATE FUNCTION multi_compress_db_original_size
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_ref
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_zstd_id
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_sha256
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_is_valid_dict
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_decompress_dict
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
 SELECT multi_compress_db_version();

data/mysql_udf/sql/uninstall.sql

@@ -1,4 +1,10 @@
--- Remove the MultiCompress MCDB1 reader UDFs.
+-- Run only after verifying no generated readable view depends on these UDFs.
+DROP FUNCTION IF EXISTS multi_compress_db_decompress_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_is_valid_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_sha256;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_zstd_id;
+DROP FUNCTION IF EXISTS multi_compress_db_original_size;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_ref;
 DROP FUNCTION IF EXISTS multi_compress_db_decompress;
 DROP FUNCTION IF EXISTS multi_compress_db_is_valid;
 DROP FUNCTION IF EXISTS multi_compress_db_version;

data/mysql_udf/sql/views.sql

@@ -1,8 +1,20 @@
--- Example read-side view for DBeaver / DataGrip / mysql console.
-
+-- MCDB1 view example. Generate production SQL with:
+--   multi_compress db view mysql ...
 CREATE OR REPLACE VIEW events_readable AS
 SELECT
   id,
-  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload,
-  created_at
+  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload
 FROM events;
+
+-- MCDB2 dictionary-backed shape. The generated view uses ALGORITHM=MERGE and
+-- a source-first STRAIGHT_JOIN to the append-only registry, so an indexed outer
+-- WHERE predicate is pushed into the source scan in MySQL 5.7. An outer ORDER BY
+-- may still use filesort; that is not view materialization.
+--
+-- CREATE ALGORITHM=MERGE SQL SECURITY DEFINER VIEW events_readable AS
+-- SELECT e.id,
+--   CONVERT(multi_compress_db_decompress_dict(
+--     e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes
+--   ) USING utf8mb4) AS payload
+-- FROM events e
+-- STRAIGHT_JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id;