multi_compress 0.3.5 → 0.6.0

data/lib/multi_compress/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MultiCompress
-  VERSION = "0.3.5"
+  VERSION = "0.6.0"
 end

data/multi_compress.gemspec

@@ -0,0 +1,55 @@
+require_relative "lib/multi_compress/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "multi_compress"
+  spec.version       = MultiCompress::VERSION
+  spec.authors       = ["Roman Haydarov"]
+  spec.email         = ["romnhajdarov@gmail.com"]
+
+  spec.summary       = "Modern fiber-friendly compression for Ruby: zstd, lz4, brotli in one gem"
+  spec.description   = "Unified C-extension gem for zstd, lz4, and brotli compression. " \
+                        "One-shot, streaming, IO wrappers, dictionary support. " \
+                        "Fiber-friendly: cooperates with Fiber::Scheduler (async, falcon) " \
+                        "so CPU-heavy compression never blocks the event loop. " \
+                        "Ships vendored sources — no system libraries required."
+  spec.homepage      = "https://github.com/roman-haidarov/multi_compress"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.7.1"
+
+  spec.metadata["homepage_uri"]    = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"]   = "#{spec.homepage}/blob/main/CHANGELOG.md"
+
+  spec.files = Dir[
+    "lib/**/*.rb",
+    "exe/*",
+    "docs/**/*.md",
+    "ext/**/*.{c,h,rb}",
+    "ext/multi_compress/vendor/**/*",
+    "ext/multi_compress/vendor/.vendored",
+    "db_core/**/*.{c,h}",
+    "postgres_extension/**/*",
+    "mysql_udf/**/*",
+    "db_deployment/**/*",
+    "multi_compress.gemspec",
+    "README.md",
+    "GET_STARTED.md",
+    "LICENSE.txt",
+    "THIRD_PARTY_NOTICES.md",
+    "CHANGELOG.md"
+  ].reject do |path|
+    File.basename(path) == ".DS_Store" ||
+      path.split(File::SEPARATOR).include?("build") ||
+      %w[.o .so .bundle .dylib .bc .d].include?(File.extname(path))
+  end
+
+  spec.bindir        = "exe"
+  spec.executables   = ["multi_compress"]
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/multi_compress/extconf.rb"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rake-compiler", "~> 1.2"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/mysql_udf/Makefile

@@ -0,0 +1,96 @@
+ZSTD_DIR      ?= ../ext/multi_compress/vendor/zstd/lib
+CORE_DIR      ?= ../db_core
+BUILD         ?= build
+CC            ?= gcc
+AR            ?= ar
+MYSQL_UDF_ABI ?= mysql57
+MYSQL_INCLUDE ?= /usr/include/mysql
+FIXTURES      ?= ../test/fixtures/database_v1
+
+ifeq ($(MYSQL_UDF_ABI),mysql57)
+MYSQL_UDF_CPPFLAGS := -DMCDB_MYSQL_UDF_ABI_57
+MYSQL_UDF_HEADER_CHECK :=
+else ifeq ($(MYSQL_UDF_ABI),system)
+MYSQL_UDF_CPPFLAGS := -I$(MYSQL_INCLUDE)
+MYSQL_UDF_HEADER_CHECK := test -f "$(MYSQL_INCLUDE)/mysql.h" || { echo "missing mysql.h: set MYSQL_INCLUDE=/path/to/mysql/include" >&2; exit 2; }
+else
+$(error MYSQL_UDF_ABI must be mysql57 or system)
+endif
+
+WARNFLAGS     ?= -Wall -Wextra -Werror
+CFLAGS        ?= -O2 -fPIC -std=c99 $(WARNFLAGS) -I$(ZSTD_DIR) -I$(CORE_DIR)/include
+ZSTD_CFLAGS   ?= -O2 -fPIC -std=c99 -I$(ZSTD_DIR) -I$(ZSTD_DIR)/common
+
+ZSTD_OBJ_DIR  := $(BUILD)/zstd
+ZSTD_C_SRCS   := $(wildcard $(ZSTD_DIR)/common/*.c) $(wildcard $(ZSTD_DIR)/decompress/*.c)
+ZSTD_C_OBJS   := $(patsubst $(ZSTD_DIR)/%.c,$(ZSTD_OBJ_DIR)/%.o,$(ZSTD_C_SRCS))
+
+ARCH := $(shell uname -m)
+ifneq (,$(filter x86_64 amd64,$(ARCH)))
+ZSTD_ASM_SRC  := $(ZSTD_DIR)/decompress/huf_decompress_amd64.S
+ifneq (,$(wildcard $(ZSTD_ASM_SRC)))
+ZSTD_ASM_OBJ  := $(ZSTD_OBJ_DIR)/decompress/huf_decompress_amd64.o
+endif
+endif
+
+ZSTD_OBJS     := $(ZSTD_C_OBJS) $(ZSTD_ASM_OBJ)
+ZSTD_LIB      := $(BUILD)/libzstd_dec.a
+UDF_SO        := $(BUILD)/multi_compress_mysql.so
+CLI           := $(BUILD)/mcdb_cli
+VALID_FIXTURES   := utf8_text empty_text large_text
+CORRUPT_FIXTURES := corrupt_magic wrong_version corrupt_payload corrupt_crc invalid_utf8 nul_text trailing_skippable trailing_frame
+
+.PHONY: all udf test fixtures clean
+all: udf
+
+$(BUILD):
+	mkdir -p $@
+
+$(ZSTD_OBJ_DIR)/%.o: $(ZSTD_DIR)/%.c
+	@mkdir -p $(dir $@)
+	$(CC) $(ZSTD_CFLAGS) -MMD -MP -c $< -o $@
+
+$(ZSTD_OBJ_DIR)/%.o: $(ZSTD_DIR)/%.S
+	@mkdir -p $(dir $@)
+	$(CC) -fPIC -I$(ZSTD_DIR) -c $< -o $@
+
+$(ZSTD_LIB): $(ZSTD_OBJS) | $(BUILD)
+	$(AR) rcs $@ $^
+
+$(UDF_SO): src/multi_compress_mysql.c $(CORE_DIR)/src/mcdb_format.c $(CORE_DIR)/include/mcdb_format.h $(ZSTD_LIB) | $(BUILD)
+	@$(MYSQL_UDF_HEADER_CHECK)
+	$(CC) $(CFLAGS) $(MYSQL_UDF_CPPFLAGS) -shared \
+	  src/multi_compress_mysql.c $(CORE_DIR)/src/mcdb_format.c $(ZSTD_LIB) \
+	  -o $@
+	@echo "built $@"
+
+udf: $(UDF_SO)
+
+$(CLI): $(CORE_DIR)/src/mcdb_format.c $(CORE_DIR)/include/mcdb_format.h test/mcdb_cli.c $(ZSTD_LIB) | $(BUILD)
+	$(CC) $(CFLAGS) $(CORE_DIR)/src/mcdb_format.c test/mcdb_cli.c $(ZSTD_LIB) -o $@
+
+fixtures:
+	RUBYLIB=../lib ruby $(FIXTURES)/generate.rb
+
+test: $(CLI)
+	@echo "== MCDB1 Ruby<->C parity =="
+	@fail=0; \
+	for name in $(VALID_FIXTURES); do \
+	  f=$(FIXTURES)/$$name.mcdb; exp=$(FIXTURES)/$$name.expected; \
+	  if [ ! -f $$f ] || [ ! -f $$exp ]; then echo "  MISSING $$name (run: make fixtures)"; fail=1; continue; fi; \
+	  if $(CLI) decode $$f | cmp -s - $$exp; then echo "  valid   $$name  OK"; \
+	  else echo "  valid   $$name  FAIL"; fail=1; fi; \
+	done; \
+	for name in $(CORRUPT_FIXTURES); do \
+	  f=$(FIXTURES)/$$name.mcdb; \
+	  if [ ! -f $$f ]; then echo "  MISSING $$name (run: make fixtures)"; fail=1; continue; fi; \
+	  if $(CLI) decode $$f >/dev/null 2>&1; then echo "  corrupt $$name  UNEXPECTEDLY VALID"; fail=1; \
+	  else echo "  corrupt $$name  correctly rejected"; fi; \
+	done; \
+	if [ $$fail -ne 0 ]; then echo "PARITY FAILED"; exit 1; fi; \
+	echo "PARITY OK"
+
+clean:
+	rm -rf $(BUILD)
+
+-include $(ZSTD_C_OBJS:.o=.d)

data/mysql_udf/README.md

@@ -0,0 +1,140 @@
+# MultiCompress MySQL 5.7 UDF (`MCDB1` reader)
+
+Decode-only MySQL loadable functions for the MultiCompress Database envelope v1.
+The shared format contract is [`docs/database-envelope-v1.md`](../docs/database-envelope-v1.md).
+
+## SQL API
+
+| Function | Returns | Purpose |
+|---|---:|---|
+| `multi_compress_db_version()` | STRING | reader build, MCDB1 and zstd version smoke test |
+| `multi_compress_db_is_valid(blob)` | INTEGER | `1` for valid MCDB1, `0` for corruption, `NULL` for SQL NULL |
+| `multi_compress_db_decompress(blob)` | STRING | decompressed UTF-8 or SQL `NULL` for corruption/SQL NULL |
+
+A MySQL readable view cannot distinguish an original SQL `NULL` from a corrupt
+MCDB1 envelope by testing `payload IS NULL`. Diagnose stored values with
+`multi_compress_db_is_valid(payload_compressed)`.
+
+## Production deployment: generated DBA bundle
+
+Generate a source bundle from the installed gem:
+
+```bash
+bundle exec multi_compress db package mysql --output tmp/multi-compress-mysql.tar.gz
+```
+
+On the MySQL 5.7 host, use its local Unix socket:
+
+```bash
+tar -xzf multi-compress-mysql.tar.gz
+cd multi_compress-mysql-0.5.0
+make verify
+make doctor MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+sudo make install MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+make enable MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+make status MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+```
+
+`install` is fresh-install only and refuses to replace a loaded library. A
+healthy second `enable` is a no-op. For a library update, use the controlled
+maintenance flow:
+
+```bash
+sudo make upgrade CONFIRM=UPGRADE_MULTI_COMPRESS \
+  MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+```
+
+`upgrade` captures registrations, drops MultiCompress-owned functions, backs up
+the old library, atomically replaces the library, registers functions, and
+smoke-checks the reader. On failure it restores the previous library and
+registrations. `disable` and `uninstall` are destructive and require:
+
+```bash
+make disable CONFIRM=DISABLE_MULTI_COMPRESS \
+  MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+sudo make uninstall CONFIRM=REMOVE_MULTI_COMPRESS \
+  MYSQL_DEFAULTS_FILE=/etc/mysql/admin.cnf MYSQL_SOCKET=/run/mysqld/mysqld.sock
+```
+
+The DBA account needs `INSERT` on `mysql` to register functions, `DELETE` to
+drop them, and `SELECT` on `mysql.func` for `make status`. The bundle's mysql
+client uses `--max_allowed_packet=32M`; mysqld, the Ruby driver, and
+DBeaver/JDBC must each support payloads up to the 16 MiB MCDB1 limit.
+
+## DBeaver readable view
+
+The application stores `MultiCompress::Database.compress(text)` in a `LONGBLOB`
+column and commits generated view SQL in a migration:
+
+```bash
+bundle exec multi_compress db view mysql \
+  --table app.events \
+  --column payload_compressed \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+MySQL views default to `SQL SECURITY DEFINER`; create the migration with a
+stable technical account and do not remove that definer without recreating the
+view. Grant the DBeaver role `SELECT` on the resulting view. The generated view
+uses `CONVERT(... USING utf8mb4)` so Cyrillic, Kazakh text and emoji display
+correctly. Filter by indexed uncompressed columns before selecting through it.
+
+## Multi-host rollout
+
+Build/install the native library on every primary, replica, reporting, failover,
+and restore host that will read through the view before registering UDFs or
+applying the view migration. A read-only view does not itself replicate the UDF;
+replication requires the UDF on a replica only when a replicated statement
+invokes it.
+
+`make verify` catches accidental corruption after extraction; it does not
+authenticate an archive. Use an out-of-band SHA-256, minisign/GPG signature, or
+signed release provenance for a real trust chain.
+
+## MCDB2 dictionary-backed columns
+
+MCDB2 is opt-in for one homogeneous column with many similar small JSON/text
+values. Upgrade the UDF library first; the controlled `make upgrade` flow keeps
+all MCDB1 and MCDB2 functions registered.
+
+Dictionary bytes are application data. Generate the append-only registry in the
+application database, register immutable versions, and retain all referenced
+versions in backups and replicas:
+
+```bash
+bundle exec multi_compress db registry mysql --database app \
+  --output db/mcdb_dictionary_registry.sql
+```
+
+For a new MCDB2 column add `payload_dictionary_id BIGINT UNSIGNED NOT NULL` with
+a foreign key to `mcdb_dictionary_versions(id)`, then generate the view:
+
+```bash
+bundle exec multi_compress db view mysql \
+  --table app.events \
+  --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+The generated MCDB2 view requests `ALGORITHM=MERGE SQL SECURITY DEFINER` and uses
+a source-first `STRAIGHT_JOIN` to the registry. That keeps an indexed outer source
+predicate from starting at the registry. It passes payload, registry id, SHA-256
+and dictionary bytes to the UDF. Keep the definition free of `DISTINCT`, aggregates,
+`GROUP BY`, `UNION`, subqueries and inner `LIMIT`; validate real queries with
+`EXPLAIN`: source must be a selective `range`/`ref` access and no `<derived>` row
+may appear. MySQL 5.7 can still report `Using temporary; Using filesort` for an
+outer `ORDER BY` over the joined projection; that is sorting, not view
+materialization. A stable definer account must remain part of backup/restore
+procedures.
+
+Details are in [`docs/database-envelope-v2.md`](../docs/database-envelope-v2.md).
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/mysql_udf/sql/examples.sql

@@ -0,0 +1,16 @@
+-- Point lookup an employee runs from DBeaver:
+SELECT
+  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload
+FROM events
+WHERE id = 123;
+
+SELECT id
+FROM events
+WHERE multi_compress_db_is_valid(payload_compressed) = 0;
+
+SELECT multi_compress_db_version();
+
+-- MCDB2: use the generated ALGORITHM=MERGE readable view. Avoid filtering on
+-- decoded payload text; filter/order by indexed source columns before LIMIT.
+--
+-- SELECT id, payload FROM admin.events_readable WHERE id >= 100 ORDER BY id LIMIT 100;

data/mysql_udf/sql/install.sql

@@ -0,0 +1,39 @@
+-- Install the MultiCompress MCDB1/MCDB2 reader UDFs (MySQL 5.7).
+
+CREATE FUNCTION multi_compress_db_version
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_is_valid
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_decompress
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_original_size
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_ref
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_zstd_id
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_sha256
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_is_valid_dict
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_decompress_dict
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+SELECT multi_compress_db_version();

data/mysql_udf/sql/uninstall.sql

@@ -0,0 +1,10 @@
+-- Run only after verifying no generated readable view depends on these UDFs.
+DROP FUNCTION IF EXISTS multi_compress_db_decompress_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_is_valid_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_sha256;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_zstd_id;
+DROP FUNCTION IF EXISTS multi_compress_db_original_size;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_ref;
+DROP FUNCTION IF EXISTS multi_compress_db_decompress;
+DROP FUNCTION IF EXISTS multi_compress_db_is_valid;
+DROP FUNCTION IF EXISTS multi_compress_db_version;

data/mysql_udf/sql/views.sql

@@ -0,0 +1,20 @@
+-- MCDB1 view example. Generate production SQL with:
+--   multi_compress db view mysql ...
+CREATE OR REPLACE VIEW events_readable AS
+SELECT
+  id,
+  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload
+FROM events;
+
+-- MCDB2 dictionary-backed shape. The generated view uses ALGORITHM=MERGE and
+-- a source-first STRAIGHT_JOIN to the append-only registry, so an indexed outer
+-- WHERE predicate is pushed into the source scan in MySQL 5.7. An outer ORDER BY
+-- may still use filesort; that is not view materialization.
+--
+-- CREATE ALGORITHM=MERGE SQL SECURITY DEFINER VIEW events_readable AS
+-- SELECT e.id,
+--   CONVERT(multi_compress_db_decompress_dict(
+--     e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes
+--   ) USING utf8mb4) AS payload
+-- FROM events e
+-- STRAIGHT_JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id;