multi_compress 0.3.5 → 0.5.0

data/db_deployment/postgres/bin/enable

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DATABASE="${PGDATABASE:-}"
+SCHEMA="multi_compress"
+MIGRATION_ROLE=""
+READ_ROLE=""
+PSQL="${PSQL:-psql}"
+HOST=""
+PORT=""
+USERNAME=""
+
+usage() {
+  cat <<'TEXT'
+Usage: ./bin/enable --database DATABASE --migration-role ROLE --read-role ROLE [OPTIONS]
+
+Options:
+  --schema NAME          extension schema (default: multi_compress)
+  --migration-role ROLE  role that creates the generated view migration
+  --read-role ROLE       role that reads the generated view
+  --host HOST            PostgreSQL host passed to psql
+  --port PORT            PostgreSQL port passed to psql
+  --username USER        PostgreSQL user passed to psql
+  --psql PATH            psql binary (default: psql)
+TEXT
+}
+
+die() {
+  echo "multi_compress postgres enable: $*" >&2
+  exit 2
+}
+
+validate_identifier() {
+  local value="$1"
+  local option="$2"
+  printf '%s' "$value" | grep -Eq '^[A-Za-z_][A-Za-z0-9_]*$' || die "$option must be a simple SQL identifier"
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --database)
+      [ "$#" -ge 2 ] || die "--database requires a value"
+      DATABASE="$2"
+      shift 2
+      ;;
+    --schema)
+      [ "$#" -ge 2 ] || die "--schema requires a value"
+      SCHEMA="$2"
+      shift 2
+      ;;
+    --migration-role)
+      [ "$#" -ge 2 ] || die "--migration-role requires a value"
+      MIGRATION_ROLE="$2"
+      shift 2
+      ;;
+    --read-role)
+      [ "$#" -ge 2 ] || die "--read-role requires a value"
+      READ_ROLE="$2"
+      shift 2
+      ;;
+    --host)
+      [ "$#" -ge 2 ] || die "--host requires a value"
+      HOST="$2"
+      shift 2
+      ;;
+    --port)
+      [ "$#" -ge 2 ] || die "--port requires a value"
+      PORT="$2"
+      shift 2
+      ;;
+    --username)
+      [ "$#" -ge 2 ] || die "--username requires a value"
+      USERNAME="$2"
+      shift 2
+      ;;
+    --psql)
+      [ "$#" -ge 2 ] || die "--psql requires a path"
+      PSQL="$2"
+      shift 2
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+[ -n "$DATABASE" ] || die "--database is required"
+[ -n "$MIGRATION_ROLE" ] || die "--migration-role is required"
+[ -n "$READ_ROLE" ] || die "--read-role is required"
+validate_identifier "$SCHEMA" '--schema'
+validate_identifier "$MIGRATION_ROLE" '--migration-role'
+validate_identifier "$READ_ROLE" '--read-role'
+
+if [[ "$PSQL" == */* ]]; then
+  [ -x "$PSQL" ] || die "psql is not executable: $PSQL"
+else
+  PSQL="$(command -v "$PSQL" || true)"
+  [ -n "$PSQL" ] || die "psql was not found"
+fi
+
+ARGS=(-X -v ON_ERROR_STOP=1 -d "$DATABASE")
+[ -n "$HOST" ] && ARGS+=(-h "$HOST")
+[ -n "$PORT" ] && ARGS+=(-p "$PORT")
+[ -n "$USERNAME" ] && ARGS+=(-U "$USERNAME")
+
+SERVER_ENCODING="$("$PSQL" "${ARGS[@]}" -Atq -c 'SHOW server_encoding;' | tr -d '\r\n')"
+[ "$SERVER_ENCODING" = 'UTF8' ] || die "database $DATABASE uses $SERVER_ENCODING; multi_compress requires UTF8"
+
+EXISTING_SCHEMA="$("$PSQL" "${ARGS[@]}" -Atq -c "SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'multi_compress';" | tr -d '\r\n')"
+if [ -n "$EXISTING_SCHEMA" ] && [ "$EXISTING_SCHEMA" != "$SCHEMA" ]; then
+  die "multi_compress is already installed in schema $EXISTING_SCHEMA; requested schema is $SCHEMA"
+fi
+
+if [ -z "$EXISTING_SCHEMA" ]; then
+  "$PSQL" "${ARGS[@]}" <<SQL
+CREATE SCHEMA IF NOT EXISTS "$SCHEMA";
+CREATE EXTENSION multi_compress WITH SCHEMA "$SCHEMA";
+SQL
+fi
+
+"$PSQL" "${ARGS[@]}" <<SQL
+REVOKE ALL ON SCHEMA "$SCHEMA" FROM PUBLIC;
+REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA "$SCHEMA" FROM PUBLIC;
+GRANT USAGE ON SCHEMA "$SCHEMA" TO "$MIGRATION_ROLE";
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA "$SCHEMA" TO "$MIGRATION_ROLE", "$READ_ROLE";
+SELECT "$SCHEMA".multi_compress_db_version() AS multi_compress_version;
+SQL
+
+printf '%s\n' \
+  "MultiCompress is enabled in $DATABASE.$SCHEMA." \
+  "Grant SELECT on each generated readable view to $READ_ROLE in the application migration." \
+  'Apply the generated readable-view SQL in the application migration, then query that view from DBeaver.'

data/db_deployment/postgres/bin/install

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DEFAULT_PG_CONFIG="${PG_CONFIG:-pg_config}"
+BUILD_PG_CONFIG="$DEFAULT_PG_CONFIG"
+RUNTIME_PG_CONFIG="$DEFAULT_PG_CONFIG"
+
+usage() {
+  cat <<'TEXT'
+Usage: sudo ./bin/install [--pg-config PATH]
+
+Builds the extension on this PostgreSQL host and installs its library, control
+file and extension SQL into the directories reported by the selected pg_config.
+
+Advanced: --build-pg-config and --runtime-pg-config let a container build with
+separate development headers while installing into the running server's paths.
+Ordinary installations should use only --pg-config, or no option at all.
+TEXT
+}
+
+die() {
+  echo "multi_compress postgres install: $*" >&2
+  exit 2
+}
+
+resolve_pg_config() {
+  local candidate="$1"
+
+  if [[ "$candidate" == */* ]]; then
+    [ -x "$candidate" ] || die "pg_config is not executable: $candidate"
+    printf '%s\n' "$candidate"
+  else
+    candidate="$(command -v "$candidate" || true)"
+    [ -n "$candidate" ] || die "pg_config was not found"
+    printf '%s\n' "$candidate"
+  fi
+}
+
+pg_major() {
+  "$1" --version | awk '{print $2}' | cut -d. -f1
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --pg-config)
+      [ "$#" -ge 2 ] || die "--pg-config requires a path"
+      BUILD_PG_CONFIG="$2"
+      RUNTIME_PG_CONFIG="$2"
+      shift 2
+      ;;
+    --build-pg-config)
+      [ "$#" -ge 2 ] || die "--build-pg-config requires a path"
+      BUILD_PG_CONFIG="$2"
+      shift 2
+      ;;
+    --runtime-pg-config)
+      [ "$#" -ge 2 ] || die "--runtime-pg-config requires a path"
+      RUNTIME_PG_CONFIG="$2"
+      shift 2
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+[ "${EUID:-$(id -u)}" -eq 0 ] || die "run as root: sudo ./bin/install"
+BUILD_PG_CONFIG="$(resolve_pg_config "$BUILD_PG_CONFIG")"
+RUNTIME_PG_CONFIG="$(resolve_pg_config "$RUNTIME_PG_CONFIG")"
+
+BUILD_MAJOR="$(pg_major "$BUILD_PG_CONFIG")"
+RUNTIME_MAJOR="$(pg_major "$RUNTIME_PG_CONFIG")"
+case "$BUILD_MAJOR" in
+  ""|*[!0-9]*) die "could not determine PostgreSQL build major version" ;;
+esac
+case "$RUNTIME_MAJOR" in
+  ""|*[!0-9]*) die "could not determine PostgreSQL runtime major version" ;;
+esac
+[ "$BUILD_MAJOR" = "$RUNTIME_MAJOR" ] || die "build PostgreSQL major $BUILD_MAJOR does not match runtime major $RUNTIME_MAJOR"
+
+"$ROOT/bin/doctor" --pg-config "$BUILD_PG_CONFIG"
+
+make -C "$ROOT/postgres_extension" clean all PG_CONFIG="$BUILD_PG_CONFIG"
+
+PKGLIBDIR="$($RUNTIME_PG_CONFIG --pkglibdir)"
+EXTENSION_DIR="$($RUNTIME_PG_CONFIG --sharedir)/extension"
+install -d -m 0755 "$PKGLIBDIR" "$EXTENSION_DIR"
+install -m 0755 "$ROOT/postgres_extension/multi_compress_pg.so" "$PKGLIBDIR/multi_compress_pg.so"
+install -m 0644 "$ROOT/postgres_extension/multi_compress.control" "$EXTENSION_DIR/multi_compress.control"
+install -m 0644 "$ROOT/postgres_extension/sql/multi_compress--0.5.0.sql" "$EXTENSION_DIR/multi_compress--0.5.0.sql"
+
+printf '%s\n' \
+  'MultiCompress PostgreSQL files installed.' \
+  "  library: $PKGLIBDIR/multi_compress_pg.so" \
+  "  control: $EXTENSION_DIR/multi_compress.control" \
+  "  SQL:     $EXTENSION_DIR/multi_compress--0.5.0.sql" \
+  '' \
+  'Activate once per application database:' \
+  '  sudo -u postgres make enable DB=app_production'

data/db_deployment/postgres/bin/uninstall

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PG_CONFIG="${PG_CONFIG:-pg_config}"
+
+usage() {
+  cat <<'TEXT'
+Usage: sudo ./bin/uninstall [--pg-config PATH]
+
+Removes installed PostgreSQL extension files from this host. Drop the extension
+from every database first; this command cannot discover all databases/views. The
+Makefile requires CONFIRM=REMOVE_MULTI_COMPRESS before this script can run.
+TEXT
+}
+
+die() {
+  echo "multi_compress postgres uninstall: $*" >&2
+  exit 2
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --pg-config)
+      [ "$#" -ge 2 ] || die "--pg-config requires a path"
+      PG_CONFIG="$2"
+      shift 2
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+[ "${EUID:-$(id -u)}" -eq 0 ] || die "run as root: sudo ./bin/uninstall"
+
+if [[ "$PG_CONFIG" == */* ]]; then
+  [ -x "$PG_CONFIG" ] || die "pg_config is not executable: $PG_CONFIG"
+else
+  PG_CONFIG="$(command -v "$PG_CONFIG" || true)"
+  [ -n "$PG_CONFIG" ] || die "pg_config was not found"
+fi
+PKGLIBDIR="$($PG_CONFIG --pkglibdir)"
+EXTENSION_DIR="$($PG_CONFIG --sharedir)/extension"
+
+rm -f "$PKGLIBDIR/multi_compress_pg.so" \
+      "$EXTENSION_DIR/multi_compress.control" \
+      "$EXTENSION_DIR/multi_compress--0.5.0.sql"
+
+echo "MultiCompress PostgreSQL files removed from this host."

data/db_deployment/postgres/bin/verify

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+if command -v sha256sum >/dev/null 2>&1; then
+  sha256sum -c SHA256SUMS
+elif command -v shasum >/dev/null 2>&1; then
+  shasum -a 256 -c SHA256SUMS
+else
+  echo "multi_compress postgres verify: sha256sum or shasum is required" >&2
+  exit 2
+fi

data/docs/database-envelope-v1.md

@@ -0,0 +1,159 @@
+# MultiCompress Database Envelope — Format v1 (`MCDB1`)
+
+Status: **frozen**. This is a long-term storage contract. Once data is written
+to a database column, these bytes must remain readable forever, so v1 will not
+change. New capabilities go into a future `v2` with a new version byte.
+
+Two independent implementations MUST agree on this document byte-for-byte:
+
+* the Ruby writer/reader — `MultiCompress::Database` (`lib/multi_compress/database.rb`)
+* the MySQL 5.7 reader — `multi_compress_db_decompress()` UDF (`mysql_udf/`)
+* the PostgreSQL reader — `multi_compress_db_decompress(bytea)` extension function
+  (`postgres_extension/`)
+
+## Scope of v1 (deliberately narrow)
+
+| Aspect              | v1 rule                                              |
+|---------------------|------------------------------------------------------|
+| Algorithm           | **zstd only** (no lz4, brotli)                       |
+| Dictionaries        | **not allowed**                                      |
+| Transport encoding  | **raw bytes** (no base64)                            |
+| Payload text        | **UTF-8 without U+0000/NUL** (validated on write)    |
+| Column type (MySQL) | **`LONGBLOB`** recommended; `MEDIUMBLOB` only if the stored envelope stays < 16 MiB (see note) |
+| Column type (PostgreSQL) | **`bytea`**                                      |
+| Max decompressed    | **16 MiB** (16777216 bytes), enforced on read AND write |
+
+Anything outside this table is a v2 concern and is out of scope for v1.
+
+**Column sizing.** The 16 MiB cap is on the *decompressed* text. What the column
+must hold is the *stored envelope* (`19 bytes + zstd frame`), which is normally
+far smaller than the text. But `MEDIUMBLOB` tops out at `2^24 - 1 = 16,777,215`
+bytes, one byte below the plaintext cap, and incompressible input can make the
+envelope exceed the plaintext size. So v1 recommends **`LONGBLOB`**. Use
+`MEDIUMBLOB` only if the application also checks the final `blob.bytesize` before
+insert.
+
+## Byte layout
+
+All multi-byte integers are **little-endian**. The envelope is a fixed 19-byte
+header followed by a standard zstd frame:
+
+```
+ offset  size  field           value / meaning
+ ------  ----  --------------  -----------------------------------------------
+   0      4    magic           ASCII "MCDB" = 0x4D 0x43 0x44 0x42
+   4      1    format_version  0x01
+   5      1    codec           0x01 = zstd   (only value allowed in v1)
+   6      1    flags           0x00          (reserved; MUST be 0 in v1)
+   7      8    original_size   uint64 LE: byte length of the original UTF-8 text
+  15      4    crc32           uint32 LE: CRC-32 (zlib/IEEE) of the original text
+  19      N    zstd_frame      a standard zstd frame produced from the text
+```
+
+Total stored size = `19 + N` bytes.
+
+`original_size` is the length of the **decompressed** UTF-8 bytes, stored
+explicitly so a reader can pre-allocate exactly. `crc32` is the standard
+zlib/IEEE CRC-32 of those same original bytes.
+
+**Why an explicit CRC-32.** A raw zstd frame carries no content checksum by
+default, so a single flipped byte in the payload can decode to a value of the
+correct length with corrupted contents that zstd cannot detect. v1 therefore
+stores a CRC-32 of the plaintext and every reader verifies it after
+decompression. This matches Ruby's `MultiCompress.crc32`.
+
+CRC-32 detects accidental corruption (bit flips, truncation, wrong bytes) with a
+small residual collision probability; it is **not** a cryptographic integrity or
+authenticity mechanism. For trusted database storage that is the intended use.
+If envelopes could be altered by an untrusted party, a keyed cryptographic
+digest would be required — that is a different, future format, not v1.
+
+### Empty text
+
+An empty string is valid: `original_size = 0`, and `zstd_frame` is the zstd
+frame of a zero-length input. The reader returns an empty string.
+
+## Writer algorithm (compress)
+
+1. Require input is a `String` whose bytes are valid UTF-8 and contain no NUL byte; else error.
+2. Let `n = bytesize`. If `n > 16777216`, error (too large for v1).
+3. `crc = crc32(bytes)` (zlib/IEEE).
+4. `frame = zstd_compress(bytes)`.
+5. Emit `"MCDB" + 0x01 + 0x01 + 0x00 + le_uint64(n) + le_uint32(crc) + frame`.
+
+## Reader algorithm (decompress / validate)
+
+A reader MUST reject any input that fails any check below. Ruby raises
+`MultiCompress::DataError`, PostgreSQL raises a SQL error, and the MySQL UDF
+returns SQL `NULL` while `multi_compress_db_is_valid(blob)` returns `0`.
+
+1. `19 <= length <= 16842771` (a complete, bounded envelope).
+2. `magic == "MCDB"`.
+3. `format_version == 0x01`.
+4. `codec == 0x01`.
+5. `flags == 0x00`.
+6. `original_size <= 16777216`.
+7. The payload is **exactly one** standard zstd frame: no skippable frame,
+   concatenated frame, or trailing byte is allowed.
+8. The zstd frame declares a content size and that size equals `original_size`.
+9. `zstd_decompress(frame)` succeeds with a bounded output of at most
+   `original_size` bytes.
+10. The produced byte length equals `original_size` exactly.
+11. `crc32(produced) == crc32` from the header.
+12. The produced bytes are valid UTF-8 and contain no NUL byte.
+
+On success the reader returns a UTF-8 string. MySQL callers can expose it with
+`CONVERT(... USING utf8mb4)`. PostgreSQL callers use the `text`-returning
+extension function and therefore require a UTF-8 database encoding.
+
+`multi_compress_db_is_valid(blob)` performs checks 1–12 and returns 1/0 without
+raising, so it can be used in `WHERE`/`CASE`.
+
+## Error taxonomy
+
+| Condition | Ruby | PostgreSQL extension | MySQL UDF |
+|-----------|------|----------------------|-----------|
+| truncated / oversized envelope / bad magic / version | `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+| codec/flags not v1 | `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+| `original_size` over the cap | `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+| invalid, trailing, or concatenated zstd frame / size mismatch | `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+| crc32 mismatch (payload corruption) | `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+| non-UTF-8 or NUL text on write or read | `ArgumentError` / `MultiCompress::DataError` | SQL error | `NULL`, `is_valid = 0` |
+
+
+**MySQL NULL diagnostics.** In a MySQL readable view, `payload IS NULL` cannot
+distinguish an original SQL `NULL` from a corrupt MCDB1 envelope. Diagnose a
+specific stored value with `multi_compress_db_is_valid(payload_compressed)`.
+
+## Versioning policy
+
+* The 4-byte magic `MCDB` is a reserved namespace.
+* `format_version` gates the whole layout. A reader that does not recognise the
+  version MUST error, not guess.
+* v1 is immutable. Format changes ship as v2 (`format_version = 0x02`) with its
+  own spec document; readers dispatch on the version byte.
+
+## Migration note
+
+v1 has **no legacy/plaintext auto-detection**. To migrate an existing text
+column, add a new `LONGBLOB` column and write the envelope there; never
+reinterpret old plaintext bytes as an envelope. See the rollout steps in
+`mysql_udf/README.md` or `postgres_extension/README.md`.
+
+## Deployment and integrity notes
+
+`make verify` detects accidental corruption after extraction. It does not
+authenticate an archive because the archive's own Makefile is executed during
+installation. Use an out-of-band SHA-256, a minisign/GPG signature, or signed
+release provenance for a real trust chain.
+
+For MySQL, `@@GLOBAL.max_allowed_packet`, the Ruby driver, and the DBeaver/JDBC
+connection must all allow MCDB1 payloads up to the 16 MiB limit; the deployment
+bundle's mysql client uses `--max_allowed_packet=32M` but cannot configure Rails
+or DBeaver.
+
+Install the native library on every host that will read through a generated
+view—primary, replicas, reporting, failover, and restore targets—before
+registering UDFs, creating the PostgreSQL extension, or applying a view
+migration. A read-only view does not itself enter MySQL replication; replication
+requires the UDF on a replica only when a replicated statement invokes it.