multi_compress 0.3.5 → 0.6.0

data/mysql_udf/test/run_e2e.sh

@@ -0,0 +1,414 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+
+DEFAULT_MYSQL_IMAGE="mysql:5.7.44@sha256:dab0a802b44617303694fb17d166501de279c3031ddeb28c56ecf7fcab5ef0da"
+IMAGE="${MCDB_MYSQL_IMAGE:-$DEFAULT_MYSQL_IMAGE}"
+PLATFORM="${MCDB_MYSQL_PLATFORM:-linux/amd64}"
+CONTAINER="${MCDB_MYSQL_CONTAINER:-mcdb-e2e}"
+WAIT_SECONDS="${MCDB_MYSQL_WAIT_SECONDS:-90}"
+WORK_ROOT="${MCDB_MYSQL_WORK_ROOT:-$(mktemp -d -t multi-compress-mysql.XXXXXX)}"
+REMOVE_WORK_ROOT=0
+ARCHIVE="$WORK_ROOT/multi_compress-mysql.tar.gz"
+GEM_UNPACK_DIR="$WORK_ROOT/unpacked"
+GEM_FILE="$WORK_ROOT/multi_compress.gem"
+PACKAGED_GEM_ROOT=""
+BUNDLE_PARENT="/opt"
+
+command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 2; }
+
+cleanup() {
+  docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+  [ "$REMOVE_WORK_ROOT" -eq 1 ] && rm -rf "$WORK_ROOT"
+}
+trap cleanup EXIT
+
+if [ -z "${MCDB_MYSQL_WORK_ROOT:-}" ]; then
+  REMOVE_WORK_ROOT=1
+fi
+
+ensure_ruby_extension() {
+  if ruby -Ilib -e 'require "multi_compress"; require "multi_compress/database"' >/dev/null 2>&1; then
+    return
+  fi
+
+  echo "building Ruby extension..."
+  if command -v bundle >/dev/null 2>&1; then
+    bundle exec rake compile
+  else
+    rake compile
+  fi
+  ruby -Ilib -e 'require "multi_compress"; require "multi_compress/database"' >/dev/null
+}
+
+require_fixture() {
+  local path="$1"
+  [ -f "$path" ] || { echo "missing committed fixture: $path" >&2; exit 2; }
+}
+
+fixture_hex() {
+  ruby -e 'print File.binread(ARGV.fetch(0)).unpack1("H*")' "$1"
+}
+
+prepare_packaged_gem() {
+  gem build multi_compress.gemspec --output "$GEM_FILE" >/dev/null
+  mkdir -p "$GEM_UNPACK_DIR"
+  gem unpack "$GEM_FILE" --target "$GEM_UNPACK_DIR" >/dev/null
+  shopt -s nullglob
+  roots=("$GEM_UNPACK_DIR"/*)
+  shopt -u nullglob
+  [ "${#roots[@]}" -eq 1 ] && [ -d "${roots[0]}" ] || { echo "could not unpack exactly one multi_compress gem root" >&2; exit 2; }
+  PACKAGED_GEM_ROOT="${roots[0]}"
+  [ -f "$PACKAGED_GEM_ROOT/multi_compress.gemspec" ] || { echo "packaged gem lacks gemspec" >&2; exit 2; }
+  [ -d "$PACKAGED_GEM_ROOT/lib" ] || { echo "packaged gem lacks lib" >&2; exit 2; }
+  [ -d "$PACKAGED_GEM_ROOT/exe" ] || { echo "packaged gem lacks exe" >&2; exit 2; }
+  [ -f "$PACKAGED_GEM_ROOT/lib/multi_compress/db_deployment.rb" ] || { echo "packaged gem lacks db deployment CLI" >&2; exit 2; }
+}
+
+db_cli() {
+  ruby -I "$PACKAGED_GEM_ROOT/lib" "$PACKAGED_GEM_ROOT/exe/multi_compress" db "$@"
+}
+
+compress_hex() {
+  MCDB_TEXT="$1" ruby -Ilib -r multi_compress -r multi_compress/database \
+    -e 'print MultiCompress::Database.compress(ENV.fetch("MCDB_TEXT")).unpack1("H*")'
+}
+
+mcdb2_fixture() {
+  MCDB_TEXT="$1" ruby -Ilib -r multi_compress -r multi_compress/database <<'RUBY'
+    samples = 256.times.map do |i|
+      %({"kind":"event","tenant":#{i % 8},"metadata":{"source":"worker","version":1,"name":"same-shape-#{i % 16}"},"payload":"#{"x" * (48 + i % 64)}"})
+    end
+
+    dictionary = MultiCompress::Database::Dictionary.train(samples, id: 42, size: 4096)
+    blob = MultiCompress::Database.compress(ENV.fetch("MCDB_TEXT"), dictionary: dictionary)
+
+    puts dictionary.id
+    puts dictionary.zstd_id
+    puts dictionary.sha256.unpack1("H*")
+    puts dictionary.bytes.unpack1("H*")
+    puts blob.unpack1("H*")
+RUBY
+}
+
+bundle_name() {
+  ruby -I "$PACKAGED_GEM_ROOT/lib" -r multi_compress/version \
+    -e 'print "multi_compress-mysql-#{MultiCompress::VERSION}"'
+}
+
+build_deployment_bundle() {
+  db_cli package mysql --output "$ARCHIVE" --force
+  test -s "$ARCHIVE"
+}
+
+install_mysql_build_tools() {
+  docker exec -u 0 "$CONTAINER" sh -ceu '
+    if command -v yum >/dev/null 2>&1; then
+      yum install -y gcc make binutils tar gzip
+    elif command -v microdnf >/dev/null 2>&1; then
+      microdnf install -y gcc make binutils tar gzip
+    elif command -v apt-get >/dev/null 2>&1; then
+      apt-get update
+      DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential tar gzip
+    else
+      echo "no supported package manager in MySQL container" >&2
+      exit 2
+    fi
+
+    command -v cc
+    command -v make
+    command -v ar
+  '
+}
+
+install_bundle_on_db_host() {
+  local bundle_name="$1"
+  local bundle_root="${BUNDLE_PARENT}/${bundle_name}"
+
+  tar -tzf "$ARCHIVE" | grep -Fx "${bundle_name}/THIRD_PARTY_NOTICES.md" >/dev/null
+  if tar -tzf "$ARCHIVE" | grep -E '/\.DS_Store$' >/dev/null; then
+    echo 'deployment bundle must not contain .DS_Store' >&2
+    exit 2
+  fi
+  docker cp "$ARCHIVE" "$CONTAINER:/tmp/${bundle_name}.tar.gz"
+  docker exec -u 0 -e "MCDB_BUNDLE_ROOT=$bundle_root" "$CONTAINER" sh -ceu '
+    rm -rf "$MCDB_BUNDLE_ROOT"
+    mkdir -p "$(dirname "$MCDB_BUNDLE_ROOT")"
+    tar -xzf "/tmp/$(basename "$MCDB_BUNDLE_ROOT").tar.gz" -C "$(dirname "$MCDB_BUNDLE_ROOT")"
+    cd "$MCDB_BUNDLE_ROOT"
+    make verify
+    make doctor
+    make install
+    make enable
+    make status
+  '
+}
+
+apply_readable_view() {
+  local path="$1"
+  db_cli view mysql \
+    --table app.events \
+    --column payload_compressed \
+    --view admin.events_readable \
+    --columns id \
+    --output "$path"
+  docker exec -i "$CONTAINER" mysql --default-character-set=utf8mb4 < "$path"
+}
+
+apply_dictionary_readable_view() {
+  local path="$1"
+  local warnings
+  db_cli view mysql \
+    --table app.dictionary_events \
+    --column payload_compressed \
+    --dictionary-table app.mcdb_dictionary_versions \
+    --dictionary-id-column payload_dictionary_id \
+    --view admin.dictionary_events_readable \
+    --columns id \
+    --output "$path"
+  warnings=$( { cat "$path"; printf '\nSHOW WARNINGS;\n'; } | docker exec -i "$CONTAINER" mysql \
+    --default-character-set=utf8mb4 --batch --skip-column-names --raw)
+  if printf '%s\n' "$warnings" | grep -Eqi '(undefined|merge)'; then
+    echo "MySQL refused the required MERGE view algorithm:" >&2
+    printf '%s\n' "$warnings" >&2
+    exit 1
+  fi
+}
+
+ensure_ruby_extension
+prepare_packaged_gem
+build_deployment_bundle
+
+for name in corrupt_magic corrupt_payload corrupt_crc invalid_utf8 nul_text trailing_skippable trailing_frame; do
+  require_fixture "test/fixtures/database_v1/${name}.mcdb"
+done
+
+TEXT='Привет, DBeaver! Сәлем! 🌍'
+BIG=$(ruby -e 'print "a" * 300')
+VALID_HEX=$(compress_hex "$TEXT")
+BIG_HEX=$(compress_hex "$BIG")
+CORRUPT_HEX=$(fixture_hex test/fixtures/database_v1/corrupt_crc.mcdb)
+MCDB2_FIXTURE_OUTPUT="$(mcdb2_fixture "$TEXT")"
+readarray -t MCDB2_FIXTURE <<< "$MCDB2_FIXTURE_OUTPUT"
+if [ "${#MCDB2_FIXTURE[@]}" -ne 5 ] || [ -z "${MCDB2_FIXTURE[0]}" ] || [ -z "${MCDB2_FIXTURE[4]}" ]; then
+  echo "MCDB2 fixture generator returned an invalid payload" >&2
+  exit 2
+fi
+DICT_REF="${MCDB2_FIXTURE[0]}"
+DICT_ZSTD_ID="${MCDB2_FIXTURE[1]}"
+DICT_SHA_HEX="${MCDB2_FIXTURE[2]}"
+DICT_HEX="${MCDB2_FIXTURE[3]}"
+DICT_BLOB_HEX="${MCDB2_FIXTURE[4]}"
+TEXT_BYTES=$(MCDB_TEXT="$TEXT" ruby -e 'print ENV.fetch("MCDB_TEXT").bytesize')
+BUNDLE_NAME="$(bundle_name)"
+VIEW_SQL="$(mktemp -t multi-compress-mysql-view.XXXXXX.sql)"
+DICT_VIEW_SQL="$(mktemp -t multi-compress-mysql-dictionary-view.XXXXXX.sql)"
+REGISTRY_SQL="$(mktemp -t multi-compress-mysql-registry.XXXXXX.sql)"
+trap 'rm -f "$VIEW_SQL" "$DICT_VIEW_SQL" "$REGISTRY_SQL"; cleanup' EXIT
+
+docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
+docker run --platform "$PLATFORM" -d --name "$CONTAINER" \
+  -e MYSQL_ALLOW_EMPTY_PASSWORD=1 "$IMAGE" \
+  --character-set-server=utf8mb4 \
+  --collation-server=utf8mb4_unicode_ci >/dev/null
+
+echo "waiting for mysqld in $IMAGE..."
+deadline=$(( $(date +%s) + WAIT_SECONDS ))
+until docker exec "$CONTAINER" mysqladmin ping --silent 2>/dev/null; do
+  if [ "$(date +%s)" -ge "$deadline" ]; then
+    echo "mysqld did not become ready within ${WAIT_SECONDS}s" >&2
+    docker logs "$CONTAINER" >&2 || true
+    exit 1
+  fi
+  sleep 2
+done
+
+q() {
+  docker exec "$CONTAINER" mysql \
+    --default-character-set=utf8mb4 \
+    --batch \
+    --skip-column-names \
+    --raw \
+    -e "$1"
+}
+qfail() {
+  if q "$1" >/dev/null 2>&1; then return 1; else return 0; fi
+}
+
+install_mysql_build_tools
+install_bundle_on_db_host "$BUNDLE_NAME"
+
+q "DROP DATABASE IF EXISTS app;
+   DROP DATABASE IF EXISTS admin;
+   CREATE DATABASE app CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+   CREATE DATABASE admin CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+   USE app;
+   CREATE TABLE events (id INT PRIMARY KEY, payload_compressed LONGBLOB NULL);
+   INSERT INTO events VALUES
+     (1, UNHEX('$VALID_HEX')),
+     (2, NULL),
+     (3, UNHEX('$BIG_HEX')),
+     (4, UNHEX('$VALID_HEX')),
+     (5, UNHEX('$CORRUPT_HEX')),
+     (6, UNHEX('$VALID_HEX'));"
+apply_readable_view "$VIEW_SQL"
+
+# MCDB2: registry DDL lives in app and the generated view supplies dictionary bytes through an INNER JOIN.
+q "USE app;
+   CREATE TABLE dictionary_events (
+     id INT PRIMARY KEY,
+     payload_compressed LONGBLOB NOT NULL,
+     payload_dictionary_id BIGINT UNSIGNED NOT NULL
+   ) ENGINE=InnoDB;"
+db_cli registry mysql --database app \
+  --payload-table dictionary_events --payload-column payload_compressed \
+  --payload-dictionary-id-column payload_dictionary_id \
+  --output "$REGISTRY_SQL"
+docker exec -i "$CONTAINER" mysql --default-character-set=utf8mb4 < "$REGISTRY_SQL"
+q "USE app;
+   INSERT INTO mcdb_dictionary_versions (id, family, zstd_dict_id, sha256, bytes)
+     VALUES ($DICT_REF, 'events_payload_v1', $DICT_ZSTD_ID, UNHEX('$DICT_SHA_HEX'), UNHEX('$DICT_HEX'));
+   INSERT INTO dictionary_events (id, payload_compressed, payload_dictionary_id)
+   SELECT seq.id, UNHEX('$DICT_BLOB_HEX'), $DICT_REF
+   FROM (
+     SELECT ones.n + tens.n * 10 + hundreds.n * 100 + 1 AS id
+     FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+           UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS ones
+     CROSS JOIN (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+                 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS tens
+     CROSS JOIN (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+                 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS hundreds
+   ) AS seq
+   WHERE seq.id <= 1000;"
+apply_dictionary_readable_view "$DICT_VIEW_SQL"
+
+fails=0
+check() {
+  if [ "$2" = "$3" ]; then echo "  PASS  $1"; else echo "  FAIL  $1 (got [$2] want [$3])"; fails=$((fails + 1)); fi
+}
+check_err() {
+  if qfail "$2"; then echo "  PASS  $1 (rejected)"; else echo "  FAIL  $1 (should have errored)"; fails=$((fails + 1)); fi
+}
+
+check "deployment bundle version() smoke" "$(q 'SELECT LOCATE(CHAR(77,67,68,66,49), multi_compress_db_version()) > 0;')" "1"
+check "MySQL connection uses utf8mb4" \
+  "$(q "SELECT CONCAT(@@character_set_client, '/', @@character_set_connection, '/', @@character_set_results);")" \
+  "utf8mb4/utf8mb4/utf8mb4"
+check "decode roundtrip" \
+  "$(q 'USE app; SELECT CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) FROM events WHERE id=1;')" \
+  "$TEXT"
+check "NULL -> NULL" "$(q 'USE app; SELECT multi_compress_db_decompress(payload_compressed) IS NULL FROM events WHERE id=2;')" "1"
+check "big (>255, >input)" "$(q 'USE app; SELECT OCTET_LENGTH(multi_compress_db_decompress(payload_compressed)) FROM events WHERE id=3;')" "300"
+check "generated readable view" "$(q 'SELECT COUNT(*) FROM admin.events_readable WHERE id IN (1, 4);')" "2"
+check "generated readable view returns Unicode" \
+  "$(q 'SELECT payload FROM admin.events_readable WHERE id=1;')" \
+  "$TEXT"
+check "valid-corrupt-valid scan preserves later valid row" \
+  "$(q "USE app; SELECT GROUP_CONCAT(CASE WHEN multi_compress_db_decompress(payload_compressed) IS NULL THEN '<NULL>' ELSE CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) END ORDER BY id SEPARATOR '|') FROM events WHERE id IN (1, 5, 6);")" \
+  "$TEXT|<NULL>|$TEXT"
+check "is_valid true" "$(q 'USE app; SELECT multi_compress_db_is_valid(payload_compressed) FROM events WHERE id=1;')" "1"
+check "is_valid on NULL" "$(q 'USE app; SELECT multi_compress_db_is_valid(payload_compressed) IS NULL FROM events WHERE id=2;')" "1"
+
+for name in corrupt_magic corrupt_payload corrupt_crc invalid_utf8 nul_text trailing_skippable trailing_frame; do
+  hex=$(fixture_hex "test/fixtures/database_v1/${name}.mcdb")
+  check "$name -> NULL" \
+    "$(q "USE app; SELECT multi_compress_db_decompress(UNHEX('$hex')) IS NULL;")" \
+    "1"
+done
+check "is_valid false (crc)" \
+  "$(q "SELECT multi_compress_db_is_valid(UNHEX('$(fixture_hex test/fixtures/database_v1/corrupt_crc.mcdb)'));")" \
+  "0"
+check "MCDB2 dictionary reference" "$(q 'USE app; SELECT multi_compress_db_dictionary_ref(payload_compressed) FROM dictionary_events WHERE id=1;')" "$DICT_REF"
+check "MCDB2 original size" "$(q 'USE app; SELECT multi_compress_db_original_size(payload_compressed) FROM dictionary_events WHERE id=1;')" "$TEXT_BYTES"
+check "MCDB2 generated readable view" "$(q 'SELECT payload FROM admin.dictionary_events_readable WHERE id=1;')" "$TEXT"
+check "MCDB2 dictionary is valid" \
+  "$(q 'USE app; SELECT multi_compress_db_is_valid_dict(e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes) FROM dictionary_events e JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id WHERE e.id=1;')" \
+  "1"
+check_err "MCDB2 payload/FK mismatch rejected" "USE app; INSERT INTO dictionary_events VALUES (1001, UNHEX('$DICT_BLOB_HEX'), $((DICT_REF + 1)));"
+check "MCDB2 rejects wrong dictionary digest" \
+  "$(q 'USE app; SELECT multi_compress_db_is_valid_dict(e.payload_compressed, e.payload_dictionary_id, UNHEX(REPEAT("00", 32)), d.bytes) FROM dictionary_events e JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id WHERE e.id=1;')" \
+  "0"
+MYSQL_DICT_PLAN="$(q 'EXPLAIN SELECT id, payload FROM admin.dictionary_events_readable WHERE id BETWEEN 100 AND 900 ORDER BY id LIMIT 100;')"
+if printf '%s\n' "$MYSQL_DICT_PLAN" | grep -Fq '<derived'; then
+  echo "  FAIL  MCDB2 view materialized as a derived table"
+  printf '%s\n' "$MYSQL_DICT_PLAN" >&2
+  fails=$((fails + 1))
+elif ! printf '%s\n' "$MYSQL_DICT_PLAN" | awk -F '\t' '$3 == "source" && $5 == "range" && $7 == "PRIMARY" { found = 1 } END { exit(found ? 0 : 1) }'; then
+  echo "  FAIL  MCDB2 view did not use the source PRIMARY range scan"
+  printf '%s\n' "$MYSQL_DICT_PLAN" >&2
+  fails=$((fails + 1))
+else
+  # `Using temporary` / `Using filesort` here belongs to the outer ORDER BY
+  # over a joined projection. It is not evidence that the view was materialized:
+  # MySQL reports materialized views as <derived...>. The important contract is
+  # that the source predicate is merged and reaches the PRIMARY range scan.
+  echo "  PASS  MCDB2 view is MERGE-able and keeps the source PRIMARY range scan"
+fi
+check "max_allowed_packet is reported" "$(q 'SELECT @@GLOBAL.max_allowed_packet > 0;')" "1"
+
+MYSQL_SOCKET_PATH="$(q 'SELECT @@socket;' | tr -d '\r\n')"
+# Simulate an installed 0.5.x server: only the three MCDB1 UDF registrations
+# remain. 0.6 must recognise this as upgradeable, not partial, then use the
+# documented DROP -> replace library -> CREATE lifecycle to register all nine.
+docker exec -u 0 \
+  -e "MCDB_BUNDLE_ROOT=${BUNDLE_PARENT}/${BUNDLE_NAME}" \
+  -e "MCDB_MYSQL_SOCKET=$MYSQL_SOCKET_PATH" \
+  "$CONTAINER" sh -ceu '
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "
+      DROP FUNCTION multi_compress_db_decompress_dict;
+      DROP FUNCTION multi_compress_db_is_valid_dict;
+      DROP FUNCTION multi_compress_db_dictionary_sha256;
+      DROP FUNCTION multi_compress_db_dictionary_zstd_id;
+      DROP FUNCTION multi_compress_db_dictionary_ref;
+      DROP FUNCTION multi_compress_db_original_size;"
+    cd "$MCDB_BUNDLE_ROOT"
+    make status MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "state: MCDB1-only"
+    make doctor MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "server socket: $MCDB_MYSQL_SOCKET"
+    make upgrade CONFIRM=UPGRADE_MULTI_COMPRESS MYSQL_SOCKET="$MCDB_MYSQL_SOCKET"
+    make status MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "state: enabled"
+  '
+check "upgrade from MCDB1-only restores MCDB2 functions" "$(q 'SELECT multi_compress_db_original_size(payload_compressed) FROM app.dictionary_events WHERE id=1;')" "$TEXT_BYTES"
+check "upgrade keeps version() available" "$(q 'SELECT LOCATE(CHAR(77,67,68,66,49), multi_compress_db_version()) > 0;')" "1"
+
+# Register one expected name to a different SONAME and prove enable refuses without
+# dropping it. The copied library makes this a real MySQL registration, not a mock.
+docker exec -u 0 \
+  -e "MCDB_BUNDLE_ROOT=${BUNDLE_PARENT}/${BUNDLE_NAME}" \
+  -e "MCDB_MYSQL_SOCKET=$MYSQL_SOCKET_PATH" \
+  "$CONTAINER" sh -ceu '
+    plugin_dir="$(mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" --batch --skip-column-names -e "SELECT @@plugin_dir;")"
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "DROP FUNCTION multi_compress_db_version;"
+    cp "$plugin_dir/multi_compress_mysql.so" "$plugin_dir/foreign_multi_compress.so"
+    foreign_soname="$(printf "\\047foreign_multi_compress.so\\047")"
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "CREATE FUNCTION multi_compress_db_version RETURNS STRING SONAME $foreign_soname;"
+    cd "$MCDB_BUNDLE_ROOT"
+    if make enable MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" >/tmp/mcdb-foreign-enable.out 2>&1; then
+      echo "enable unexpectedly accepted a foreign registration" >&2
+      exit 1
+    fi
+    grep -F "foreign SONAME" /tmp/mcdb-foreign-enable.out
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" --batch --skip-column-names \
+      -e "SELECT dl FROM mysql.func WHERE name = $(printf "\\047multi_compress_db_version\\047");" | grep -Fx "foreign_multi_compress.so"
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "DROP FUNCTION multi_compress_db_version;"
+    owned_soname="$(printf "\\047multi_compress_mysql.so\\047")"
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "CREATE FUNCTION multi_compress_db_version RETURNS STRING SONAME $owned_soname;"
+    rm -f "$plugin_dir/foreign_multi_compress.so"
+  '
+
+docker exec -u 0 -e "MCDB_BUNDLE_ROOT=${BUNDLE_PARENT}/${BUNDLE_NAME}" "$CONTAINER" sh -ceu '
+  cd "$MCDB_BUNDLE_ROOT"
+  if make disable >/dev/null 2>&1; then
+    echo "disable succeeded without CONFIRM" >&2
+    exit 1
+  fi
+  if make uninstall >/dev/null 2>&1; then
+    echo "uninstall succeeded without CONFIRM" >&2
+    exit 1
+  fi
+  make enable | grep -F "already enabled; no changes made."
+  make uninstall CONFIRM=REMOVE_MULTI_COMPRESS
+'
+check_err "uninstall removes UDF" "SELECT multi_compress_db_version();"
+
+if [ "$fails" -ne 0 ]; then echo "E2E FAILED ($fails)"; exit 1; fi
+echo "E2E OK"

data/postgres_extension/Makefile

@@ -0,0 +1,74 @@
+EXTENSION   = multi_compress
+MODULE_big  = multi_compress_pg
+OBJS        = src/multi_compress_pg.o src/mcdb_format.o
+DATA        = sql/multi_compress--0.5.0.sql sql/multi_compress--0.6.0.sql sql/multi_compress--0.5.0--0.6.0.sql
+
+PG_CONFIG   ?= pg_config
+ZSTD_DIR    ?= ../ext/multi_compress/vendor/zstd/lib
+CORE_DIR    ?= ../db_core
+BUILD       ?= build
+CC          ?= cc
+AR          ?= ar
+
+WARNFLAGS   ?= -Wall -Wextra -Werror
+PG_CPPFLAGS += -I$(CORE_DIR)/include -I$(ZSTD_DIR)
+PG_CFLAGS   += -O2 -fPIC -std=c99 $(WARNFLAGS)
+EXPORT_MAP  := $(CURDIR)/multi_compress_pg.exports
+SHLIB_LINK  += -Wl,--version-script=$(EXPORT_MAP) $(ZSTD_LIB)
+EXTRA_CLEAN += $(BUILD)
+
+ZSTD_CFLAGS  ?= -O2 -fPIC -std=c99 -I$(ZSTD_DIR) -I$(ZSTD_DIR)/common
+ZSTD_OBJ_DIR := $(BUILD)/zstd
+ZSTD_C_SRCS  := $(wildcard $(ZSTD_DIR)/common/*.c) $(wildcard $(ZSTD_DIR)/decompress/*.c)
+ZSTD_C_OBJS  := $(patsubst $(ZSTD_DIR)/%.c,$(ZSTD_OBJ_DIR)/%.o,$(ZSTD_C_SRCS))
+
+ARCH := $(shell uname -m)
+ifneq (,$(filter x86_64 amd64,$(ARCH)))
+ZSTD_ASM_SRC := $(ZSTD_DIR)/decompress/huf_decompress_amd64.S
+ifneq (,$(wildcard $(ZSTD_ASM_SRC)))
+ZSTD_ASM_OBJ := $(ZSTD_OBJ_DIR)/decompress/huf_decompress_amd64.o
+endif
+endif
+
+ZSTD_OBJS := $(ZSTD_C_OBJS) $(ZSTD_ASM_OBJ)
+ZSTD_LIB  := $(BUILD)/libzstd_dec.a
+
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+
+.PHONY: all test parity e2e fixtures
+
+all: $(ZSTD_LIB)
+
+$(MODULE_big)$(DLSUFFIX): $(ZSTD_LIB)
+
+$(BUILD):
+	mkdir -p $@
+
+$(ZSTD_OBJ_DIR)/%.o: $(ZSTD_DIR)/%.c
+	@mkdir -p $(dir $@)
+	$(CC) $(ZSTD_CFLAGS) -MMD -MP -c $< -o $@
+
+$(ZSTD_OBJ_DIR)/%.o: $(ZSTD_DIR)/%.S
+	@mkdir -p $(dir $@)
+	$(CC) -fPIC -I$(ZSTD_DIR) -c $< -o $@
+
+$(ZSTD_LIB): $(ZSTD_OBJS) | $(BUILD)
+	$(AR) rcs $@ $^
+
+# The source wrapper includes the shared MCDB1/MCDB2 decoder; it is not a fork.
+src/mcdb_format.o: src/mcdb_format.c $(CORE_DIR)/src/mcdb_format.c $(CORE_DIR)/include/mcdb_format.h
+
+# Golden fixtures are committed. Regeneration is a development action only.
+fixtures:
+	RUBYLIB=../lib ruby ../test/fixtures/database_v1/generate.rb
+
+parity:
+	$(MAKE) -C ../mysql_udf test
+
+test: all parity
+
+e2e:
+	./test/run_e2e.sh
+
+-include $(ZSTD_C_OBJS:.o=.d)

data/postgres_extension/README.md

@@ -0,0 +1,160 @@
+# MultiCompress PostgreSQL extension (`MCDB1` reader)
+
+This PostgreSQL extension reads the same frozen `MCDB1` envelopes as the Ruby
+API and the MySQL UDF. Rails stores `MultiCompress::Database.compress(text)` in
+`bytea`; PostgreSQL exposes decoded UTF-8 through SQL and a view for DBeaver.
+
+It is a separate server-native artifact, not part of `gem install`. The shared
+format contract is [`docs/database-envelope-v1.md`](../docs/database-envelope-v1.md).
+
+## SQL API
+
+| Function | Returns | Purpose |
+|---|---:|---|
+| `multi_compress_db_version()` | `text` | reader build, MCDB1 and zstd version smoke test |
+| `multi_compress_db_is_valid(bytea)` | `boolean` | validates an MCDB1 blob without raising |
+| `multi_compress_db_decompress(bytea)` | `text` | returns UTF-8 text or raises on corruption |
+
+`multi_compress_db_decompress` requires a UTF-8 PostgreSQL database. MCDB1 does
+not permit NUL bytes, so decoded values are valid PostgreSQL `text` values.
+
+## Production deployment: generated DBA bundle
+
+Do **not** ask a DBA to clone this repository, find a release tag, manually copy
+`multi_compress_pg.so`, or guess PostgreSQL install paths. The application
+developer generates a source bundle from the exact installed gem version:
+
+```bash
+bundle exec multi_compress db package postgres --output tmp/multi-compress-postgres.tar.gz
+```
+
+The DBA receives that one archive and runs this on the PostgreSQL host:
+
+```bash
+tar -xzf multi-compress-postgres.tar.gz
+cd multi_compress-postgres-0.5.0
+make verify
+make doctor
+sudo make install
+sudo -u postgres make enable DB=app_production \
+  MIGRATION_ROLE=app_migrations READ_ROLE=dbeaver_readonly
+```
+
+`install` compiles the extension against the selected host `pg_config` and puts
+its library/control/SQL files in the directories reported by that exact server.
+Install the matching `postgresql-server-dev-<major>` package and a C toolchain
+beforehand; `make doctor` verifies both.
+For a non-default PostgreSQL install use:
+
+```bash
+sudo make install PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config
+```
+
+`enable` rejects non-UTF8 databases and an extension already installed in a
+different schema. It runs `CREATE EXTENSION` once per application database,
+revokes public access, grants the migration role `USAGE` on the extension schema,
+and grants both named roles `EXECUTE` on the decoder functions. The application
+migration must grant the read role `SELECT` on each generated view. Repeat the
+native install on every reporting, failover and restore target before enabling
+or querying MCDB1 there.
+
+## DBeaver: generated readable view
+
+Generate the view in the application repository and put the output in a Rails
+migration or versioned SQL. It is application schema, not a hand-typed DBA task:
+
+```bash
+bundle exec multi_compress db view postgres \
+  --table app.events \
+  --column payload_compressed \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+DBeaver users only query `admin.events_readable`. Their role needs `USAGE` on
+`admin`, `SELECT` on this view, and `EXECUTE` on the decoder functions granted by
+`make enable`; it does not need `USAGE` on `multi_compress`. If `make enable`
+used a different extension schema, pass it to the generator with
+`--extension-schema that_schema`:
+
+```sql
+SELECT id, created_at, status, payload
+FROM admin.events_readable
+WHERE id = 123;
+```
+
+The view decompresses selected rows. Filter with indexed ordinary columns before
+reading it; do not scan decoded payloads with an unbounded `LIKE '%text%'`.
+
+## Migration and constraints
+
+Do not reinterpret an existing plaintext column as MCDB1. Add a new nullable
+`bytea` column, dual-write it from the application, backfill in batches, verify
+`multi_compress.multi_compress_db_is_valid(blob)`, switch reads, then keep the
+old column for a rollback window. MCDB1 v1 is zstd-only, accepts valid UTF-8
+text without NUL bytes, and has a 16 MiB decompressed limit. The target database
+must use UTF-8 encoding for the text-returning function.
+
+## Development source build
+
+The source target remains useful for contributors and CI:
+
+```bash
+cd postgres_extension
+make PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config
+make test PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config
+make e2e
+```
+
+`test/run_e2e.sh` builds against a pinned PostgreSQL Docker target and is the
+server-side acceptance gate. It is not part of the end-user installation flow.
+
+## MCDB2 dictionary-backed columns
+
+MCDB2 is an opt-in companion for one homogeneous column containing many similar
+small JSON/text values. It does not replace MCDB1. First install the 0.6 reader
+on every host, then run `make enable` once again to execute `ALTER EXTENSION
+multi_compress UPDATE` and refresh function grants.
+
+The dictionary is application data, not an extension file. Create the append-only
+registry under a dedicated NOLOGIN owner and retain every version referenced by a
+payload:
+
+```bash
+bundle exec multi_compress db registry postgres \
+  --schema app \
+  --owner mcdb_dictionary_owner \
+  --migration-role app_migrations \
+  --output db/mcdb_dictionary_registry.sql
+```
+
+For a new MCDB2 column, add a `payload_dictionary_id bigint NOT NULL` FK to
+`app.mcdb_dictionary_versions`, register one strict
+`MultiCompress::Database::Dictionary`, then generate the readable view:
+
+```bash
+bundle exec multi_compress db view postgres \
+  --table app.events \
+  --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+The generated view uses an `INNER JOIN` and calls the four-argument native
+function with payload, registry id, dictionary SHA-256 and bytes. It keeps raw
+payloads/dictionary bytes out of the DBeaver role. The view owner needs source
+and registry access; DBeaver needs `SELECT` on the view and the decoder
+`EXECUTE` grant made by `make enable`.
+
+Check production query shape with `EXPLAIN (ANALYZE, BUFFERS)` on an indexed
+filter plus `LIMIT`. Do not search/order/group by decoded payload text. See
+[`docs/database-envelope-v2.md`](../docs/database-envelope-v2.md) for the full
+format and rollback contract.
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/postgres_extension/multi_compress.control

@@ -0,0 +1,4 @@
+comment = 'MultiCompress MCDB1/MCDB2 zstd reader'
+default_version = '0.6.0'
+module_pathname = '$libdir/multi_compress_pg'
+relocatable = true

data/postgres_extension/multi_compress_pg.exports

@@ -0,0 +1,24 @@
+{
+  global:
+    Pg_magic_func;
+    pg_finfo_multi_compress_db_version;
+    pg_finfo_multi_compress_db_is_valid;
+    pg_finfo_multi_compress_db_decompress;
+    pg_finfo_multi_compress_db_original_size;
+    pg_finfo_multi_compress_db_dictionary_ref;
+    pg_finfo_multi_compress_db_dictionary_zstd_id;
+    pg_finfo_multi_compress_db_dictionary_sha256;
+    pg_finfo_multi_compress_db_is_valid_dict;
+    pg_finfo_multi_compress_db_decompress_dict;
+    multi_compress_db_version;
+    multi_compress_db_is_valid;
+    multi_compress_db_decompress;
+    multi_compress_db_original_size;
+    multi_compress_db_dictionary_ref;
+    multi_compress_db_dictionary_zstd_id;
+    multi_compress_db_dictionary_sha256;
+    multi_compress_db_is_valid_dict;
+    multi_compress_db_decompress_dict;
+  local:
+    *;
+};