multi_compress 0.5.0 → 0.6.0

data/db_deployment/mysql/README.md

@@ -40,3 +40,22 @@ correctly. Filter by indexed, uncompressed columns before selecting through it.
 `make verify` catches accidental corruption after extraction; it does not
 authenticate an archive. The mysql client uses `--max_allowed_packet=32M`, but
 mysqld, the Ruby driver and DBeaver/JDBC must each allow the 16 MiB MCDB1 limit.
+
+## MCDB2 after reader installation
+
+After the 0.6 UDF is installed/enabled, the application owns dictionaries:
+
+```bash
+multi_compress db registry mysql --database app
+multi_compress db view mysql \
+  --table app.events --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable --columns id,created_at,status
+```
+
+Do not put dictionary files in this bundle. MCDB2 dictionary bytes are immutable
+application records that must travel with backups and replication. Keep generated
+MySQL views MERGE-able and verify `EXPLAIN` before granting DBeaver access.
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/db_deployment/mysql/bin/common

@@ -1,11 +1,22 @@
 #!/usr/bin/env bash
 
 MCDB_UDF_LIBRARY='multi_compress_mysql.so'
+MCDB_V1_UDF_NAMES=(
+  multi_compress_db_version
+  multi_compress_db_is_valid
+  multi_compress_db_decompress
+)
 MCDB_EMPTY_ENVELOPE_HEX='4d43444201010000000000000000000000000028b52ffd2000010000'
 MCDB_UDF_NAMES=(
   multi_compress_db_version
   multi_compress_db_is_valid
   multi_compress_db_decompress
+  multi_compress_db_original_size
+  multi_compress_db_dictionary_ref
+  multi_compress_db_dictionary_zstd_id
+  multi_compress_db_dictionary_sha256
+  multi_compress_db_is_valid_dict
+  multi_compress_db_decompress_dict
 )
 
 mcdb_die() {
@@ -83,24 +94,31 @@ mcdb_read_udf_state() {
   MCDB_UNEXPECTED_LIBRARY=0
   MCDB_REGISTERED_ROWS=()
 
-  rows="$(mcdb_query "SELECT name, dl FROM mysql.func WHERE dl = '${MCDB_UDF_LIBRARY}' OR name IN ('multi_compress_db_version', 'multi_compress_db_is_valid', 'multi_compress_db_decompress') ORDER BY name;")"
+  local names_sql=""
+  local expected=()
+  for name in "${MCDB_UDF_NAMES[@]}"; do
+    expected+=("'${name}'")
+  done
+  names_sql="$(IFS=,; printf '%s' "${expected[*]}")"
+
+  rows="$(mcdb_query "SELECT name, dl FROM mysql.func WHERE dl = '${MCDB_UDF_LIBRARY}' OR name IN (${names_sql}) ORDER BY name;")"
   while IFS=$'\t' read -r name dl; do
     [ -n "$name" ] || continue
     MCDB_REGISTERED_ROWS+=("$name"$'\t'"$dl")
-    case "$name" in
-      multi_compress_db_version|multi_compress_db_is_valid|multi_compress_db_decompress)
-        if [ "$dl" = "$MCDB_UDF_LIBRARY" ]; then
-          MCDB_EXPECTED_OWNED=$((MCDB_EXPECTED_OWNED + 1))
-        else
-          MCDB_FOREIGN_EXPECTED=$((MCDB_FOREIGN_EXPECTED + 1))
-        fi
-        ;;
-      *)
-        if [ "$dl" = "$MCDB_UDF_LIBRARY" ]; then
-          MCDB_UNEXPECTED_LIBRARY=$((MCDB_UNEXPECTED_LIBRARY + 1))
-        fi
-        ;;
-    esac
+    local expected_name=0
+    local candidate
+    for candidate in "${MCDB_UDF_NAMES[@]}"; do
+      if [ "$candidate" = "$name" ]; then expected_name=1; break; fi
+    done
+    if [ "$expected_name" -eq 1 ]; then
+      if [ "$dl" = "$MCDB_UDF_LIBRARY" ]; then
+        MCDB_EXPECTED_OWNED=$((MCDB_EXPECTED_OWNED + 1))
+      else
+        MCDB_FOREIGN_EXPECTED=$((MCDB_FOREIGN_EXPECTED + 1))
+      fi
+    elif [ "$dl" = "$MCDB_UDF_LIBRARY" ]; then
+      MCDB_UNEXPECTED_LIBRARY=$((MCDB_UNEXPECTED_LIBRARY + 1))
+    fi
   done <<< "$rows"
 }
 
@@ -109,7 +127,31 @@ mcdb_state_is_clean_absent() {
 }
 
 mcdb_state_is_clean_enabled() {
-  [ "$MCDB_EXPECTED_OWNED" -eq 3 ] && [ "$MCDB_FOREIGN_EXPECTED" -eq 0 ] && [ "$MCDB_UNEXPECTED_LIBRARY" -eq 0 ]
+  [ "$MCDB_EXPECTED_OWNED" -eq "${#MCDB_UDF_NAMES[@]}" ] && [ "$MCDB_FOREIGN_EXPECTED" -eq 0 ] && [ "$MCDB_UNEXPECTED_LIBRARY" -eq 0 ]
+}
+
+mcdb_has_owned_registration() {
+  local target="$1" registration name dl
+  for registration in "${MCDB_REGISTERED_ROWS[@]}"; do
+    IFS=$'\t' read -r name dl <<< "$registration"
+    [ "$name" = "$target" ] && [ "$dl" = "$MCDB_UDF_LIBRARY" ] && return 0
+  done
+  return 1
+}
+
+# A 0.5.x host has only the three MCDB1 UDF registrations. It is a known,
+# upgradeable state, not a partial installation. The 0.6 upgrade replaces the
+# library using MySQL's required DROP -> replace -> CREATE order and registers
+# the full MCDB1/MCDB2 surface.
+mcdb_state_is_clean_v1_enabled() {
+  local name
+  [ "$MCDB_EXPECTED_OWNED" -eq "${#MCDB_V1_UDF_NAMES[@]}" ] || return 1
+  [ "$MCDB_FOREIGN_EXPECTED" -eq 0 ] || return 1
+  [ "$MCDB_UNEXPECTED_LIBRARY" -eq 0 ] || return 1
+  for name in "${MCDB_V1_UDF_NAMES[@]}"; do
+    mcdb_has_owned_registration "$name" || return 1
+  done
+  return 0
 }
 
 mcdb_require_clean_owned_state() {
@@ -121,17 +163,27 @@ mcdb_require_clean_owned_state() {
   fi
 }
 
-mcdb_smoke_check() {
+mcdb_v1_smoke_check() {
   local version decoded
   version="$(mcdb_query 'SELECT multi_compress_db_version();' | tr -d '\r\n')"
   case "$version" in
     *'MCDB1'*) ;;
-    *) mcdb_die "version() smoke check failed: $version" ;;
+    *) mcdb_die "MCDB1 version() smoke check failed: $version" ;;
   esac
   decoded="$(mcdb_query "SELECT multi_compress_db_decompress(UNHEX('${MCDB_EMPTY_ENVELOPE_HEX}')) = '';" | tr -d '\r\n')"
   [ "$decoded" = '1' ] || mcdb_die 'decode smoke check failed'
 }
 
+mcdb_smoke_check() {
+  local version
+  mcdb_v1_smoke_check
+  version="$(mcdb_query 'SELECT multi_compress_db_version();' | tr -d '\r\n')"
+  case "$version" in
+    *'MCDB1'*'MCDB2'*) ;;
+    *) mcdb_die "MCDB1/MCDB2 version() smoke check failed: $version" ;;
+  esac
+}
+
 mcdb_drop_owned_udfs() {
   mcdb_read_udf_state
   mcdb_require_clean_owned_state

data/db_deployment/mysql/bin/enable

@@ -10,7 +10,7 @@ usage() {
   cat <<'TEXT'
 Usage: MYSQL_SOCKET=/run/mysqld/mysqld.sock ./bin/enable [--mysql PATH] [--defaults-extra-file PATH]
 
-Registers the three MultiCompress UDFs once. Existing healthy registrations are
+Registers the MultiCompress MCDB1/MCDB2 UDFs once. Existing healthy registrations are
 left untouched; this command never drops functions automatically.
 TEXT
 }
@@ -53,5 +53,9 @@ if mcdb_state_is_clean_enabled; then
   exit 0
 fi
 
+if mcdb_state_is_clean_v1_enabled; then
+  mcdb_die 'MCDB1-only UDF registrations were found; run make upgrade to install the MCDB2 reader surface'
+fi
+
 mcdb_require_clean_owned_state
 mcdb_die 'UDF registrations are partial; run make status and resolve the state before enabling'

data/db_deployment/mysql/bin/status

@@ -57,6 +57,8 @@ fi
 if mcdb_state_is_clean_enabled; then
   mcdb_smoke_check
   echo 'state: enabled'
+elif mcdb_state_is_clean_v1_enabled; then
+  echo 'state: MCDB1-only; run make upgrade before writing MCDB2 values'
 elif mcdb_state_is_clean_absent; then
   echo 'state: absent'
 else

data/db_deployment/mysql/bin/upgrade

@@ -44,9 +44,20 @@ mcdb_resolve_mysql
 mcdb_mysql_args
 mcdb_read_udf_state
 mcdb_require_clean_owned_state
-mcdb_state_is_clean_enabled || mcdb_die 'upgrade requires all three MultiCompress UDFs to be enabled'
+LEGACY_MCDB1_ONLY=0
+if mcdb_state_is_clean_v1_enabled; then
+  LEGACY_MCDB1_ONLY=1
+elif ! mcdb_state_is_clean_enabled; then
+  mcdb_die 'upgrade requires either a complete MCDB1-only or MCDB1/MCDB2 owned UDF state'
+fi
 CAPTURED_REGISTRATIONS=("${MCDB_REGISTERED_ROWS[@]}")
-mcdb_smoke_check
+if [ "$LEGACY_MCDB1_ONLY" -eq 1 ]; then
+  # A genuine 0.5.x library does not advertise MCDB2 yet. Verify only its
+  # MCDB1 surface before the required DROP -> replace -> CREATE transition.
+  mcdb_v1_smoke_check
+else
+  mcdb_smoke_check
+fi
 
 restore_captured_registrations() {
   local registration name dl returns
@@ -54,8 +65,8 @@ restore_captured_registrations() {
   for registration in "${CAPTURED_REGISTRATIONS[@]}"; do
     IFS=$'\t' read -r name dl <<< "$registration"
     case "$name" in
-      multi_compress_db_version|multi_compress_db_decompress) returns='STRING' ;;
-      multi_compress_db_is_valid) returns='INTEGER' ;;
+      multi_compress_db_version|multi_compress_db_decompress|multi_compress_db_dictionary_sha256|multi_compress_db_decompress_dict) returns='STRING' ;;
+      multi_compress_db_is_valid|multi_compress_db_original_size|multi_compress_db_dictionary_ref|multi_compress_db_dictionary_zstd_id|multi_compress_db_is_valid_dict) returns='INTEGER' ;;
       *) mcdb_die "captured unexpected UDF registration: $name" ;;
     esac
     mcdb_query "CREATE FUNCTION ${name} RETURNS ${returns} SONAME '${dl}';"
@@ -89,7 +100,8 @@ fi
 
 mv -f "$TEMP_LIBRARY" "$LIBRARY"
 if mcdb_register_expected_udfs && mcdb_smoke_check; then
-  [ "${#CAPTURED_REGISTRATIONS[@]}" -eq 3 ] || mcdb_die 'captured registration state is incomplete'
+  mcdb_read_udf_state
+  mcdb_state_is_clean_enabled || mcdb_die 'upgrade did not register the complete MCDB1/MCDB2 UDF surface'
   rm -f "$BACKUP"
   RESTORED=1
   echo 'MultiCompress MySQL UDFs upgraded successfully.'

data/db_deployment/postgres/README.md

@@ -58,3 +58,27 @@ columns first; do not use it for unbounded `LIKE '%text%'` searches.
 `make verify` catches accidental corruption after extraction; it does not
 authenticate an archive. Use an out-of-band SHA-256, a minisign/GPG signature,
 or signed release provenance for a real trust chain.
+
+## MCDB2 after reader installation
+
+Reader installation is deliberately separate from application data. Once a 0.6
+reader is installed and `make enable` has run, generate application-owned
+append-only dictionary registry DDL and a dictionary-readable view from the gem:
+
+```bash
+multi_compress db registry postgres \
+  --schema app --owner mcdb_dictionary_owner --migration-role app_migrations
+
+multi_compress db view postgres \
+  --table app.events --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable --columns id,created_at,status
+```
+
+Do not place trained dictionaries in this DBA bundle. They belong in the registry
+so backups, replicas and restored databases retain the exact bytes needed by each
+MCDB2 row. The registry generator grants schema `USAGE` to its dedicated NOLOGIN
+owner because PostgreSQL internal FK checks execute with relation-owner privileges.
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/db_deployment/postgres/bin/enable

@@ -120,6 +120,8 @@ if [ -z "$EXISTING_SCHEMA" ]; then
 CREATE SCHEMA IF NOT EXISTS "$SCHEMA";
 CREATE EXTENSION multi_compress WITH SCHEMA "$SCHEMA";
 SQL
+else
+  "$PSQL" "${ARGS[@]}" -c 'ALTER EXTENSION multi_compress UPDATE;'
 fi
 
 "$PSQL" "${ARGS[@]}" <<SQL

data/db_deployment/postgres/bin/install

@@ -92,13 +92,15 @@ EXTENSION_DIR="$($RUNTIME_PG_CONFIG --sharedir)/extension"
 install -d -m 0755 "$PKGLIBDIR" "$EXTENSION_DIR"
 install -m 0755 "$ROOT/postgres_extension/multi_compress_pg.so" "$PKGLIBDIR/multi_compress_pg.so"
 install -m 0644 "$ROOT/postgres_extension/multi_compress.control" "$EXTENSION_DIR/multi_compress.control"
-install -m 0644 "$ROOT/postgres_extension/sql/multi_compress--0.5.0.sql" "$EXTENSION_DIR/multi_compress--0.5.0.sql"
+for sql in "$ROOT"/postgres_extension/sql/multi_compress--*.sql; do
+  install -m 0644 "$sql" "$EXTENSION_DIR/$(basename "$sql")"
+done
 
 printf '%s\n' \
   'MultiCompress PostgreSQL files installed.' \
   "  library: $PKGLIBDIR/multi_compress_pg.so" \
   "  control: $EXTENSION_DIR/multi_compress.control" \
-  "  SQL:     $EXTENSION_DIR/multi_compress--0.5.0.sql" \
+  "  SQL:     $EXTENSION_DIR/multi_compress--0.6.0.sql (plus upgrade scripts)" \
   '' \
   'Activate once per application database:' \
   '  sudo -u postgres make enable DB=app_production'

data/docs/database-envelope-v2.md

@@ -0,0 +1,147 @@
+# MCDB2 database envelope (dictionary-backed, version 2)
+
+MCDB2 is an opt-in, dictionary-backed companion to [MCDB1](database-envelope-v1.md).
+It exists for a **single homogeneous database column** whose values share a stable
+text/JSON shape. Every MCDB2 value requires the exact immutable dictionary version
+recorded with it. MCDB1 remains the default for dictionary-free, heterogeneous, or
+large values.
+
+MCDB2 is compression, not encryption. Its CRC-32 detects accidental corruption; it
+does not authenticate data.
+
+## Wire format
+
+All integer fields are little-endian.
+
+| Offset | Size | Field | Required value |
+|---:|---:|---|---|
+| 0 | 4 | magic | ASCII `MCDB` |
+| 4 | 1 | format version | `0x02` |
+| 5 | 1 | codec | `0x01` (zstd) |
+| 6 | 1 | reserved flags | `0x00` |
+| 7 | 8 | original UTF-8 byte size | `0..16 MiB` |
+| 15 | 4 | CRC-32 | IEEE/zlib CRC of plaintext |
+| 19 | 8 | `dictionary_ref` | application registry id, `1..INT64_MAX` |
+| 27 | ... | zstd frame | exactly one frame with a non-zero zstd DictID |
+
+The maximum stored envelope is 16,842,779 bytes (`27 + ZSTD_compressBound(16 MiB)`).
+
+`dictionary_ref` is **not** the zstd DictID. It is a signed-bigint-compatible,
+application-level immutable primary key. The zstd frame contains the zstd DictID;
+readers require that it equals `ZSTD_getDictID_fromDict(dictionary_bytes)`.
+
+## Writer requirements
+
+A writer MUST:
+
+1. accept only valid UTF-8 text without NUL bytes and at most 16 MiB;
+2. use a conformant zstd dictionary no larger than 256 KiB with non-zero DictID;
+3. use a positive registry id at most `INT64_MAX`;
+4. emit a zstd frame through a prepared CDict so its DictID and content size are set;
+5. emit the 27-byte MCDB2 header followed by exactly that frame.
+
+Ruby API:
+
+```ruby
+samples = historical_payloads
+
+dictionary = MultiCompress::Database::Dictionary.train(
+  samples,
+  id: 42,
+  size: 32 * 1024
+)
+
+blob = MultiCompress::Database.compress(payload_json, dictionary: dictionary)
+```
+
+## Reader requirements
+
+The supported SQL ABI has four arguments:
+
+```sql
+multi_compress_db_decompress_dict(
+  blob,
+  dictionary_id,
+  dictionary_sha256,
+  dictionary_bytes
+)
+```
+
+The reader rejects any value unless all checks hold:
+
+1. the MCDB2 header is valid and `header.dictionary_ref == dictionary_id`;
+2. `dictionary_sha256` is 32 bytes and equals SHA-256 of `dictionary_bytes`;
+3. dictionary bytes are a conformant zstd dictionary with a non-zero DictID;
+4. the frame has exactly one zstd frame and its DictID equals the supplied dictionary's zstd DictID;
+5. the frame content size equals the header size; decompression, CRC-32 and UTF-8/NUL checks pass.
+
+`multi_compress_db_is_valid_dict(...)` returns false/0 rather than raising for an
+invalid non-NULL input. PostgreSQL `decompress_dict` raises a SQL error; MySQL 5.7
+returns SQL `NULL`, matching the MCDB1 UDF policy.
+
+## Dictionary registry
+
+Dictionary bytes are **application data**, not part of the DBA reader bundle.
+Store them in an append-only registry table in the application database. A rotation
+creates a new version row and changes only a movable `dictionary_heads` pointer;
+old rows and old payloads continue to reference their original version.
+
+Generate starter DDL:
+
+```bash
+multi_compress db registry postgres \
+  --schema app --owner mcdb_dictionary_owner \
+  --migration-role app_migrations \
+  --payload-table events --payload-column payload_compressed \
+  --payload-dictionary-id-column payload_dictionary_id \
+  > db/mcdb_dictionary_registry.sql
+
+multi_compress db registry mysql --database app \
+  --payload-table events --payload-column payload_compressed \
+  --payload-dictionary-id-column payload_dictionary_id \
+  > db/mcdb_dictionary_registry.sql
+```
+
+The registry DDL validates id/hash/zstd-ID/size at insert time and rejects updates
+and deletes. When the three `--payload-*` options are supplied, it also creates the
+payload FK and a trigger requiring `header.dictionary_ref == payload_dictionary_id`.
+For PostgreSQL it grants `USAGE` on the application schema to the dedicated NOLOGIN
+registry owner: internal FK checks run with relation-owner privileges and require
+that schema privilege. Backups, replication and restores therefore carry dictionaries
+with the compressed rows that require them.
+
+## Readable views
+
+Generate an MCDB2 view with an INNER JOIN. MySQL 5.7 output uses a source-first
+`STRAIGHT_JOIN` internally so an indexed source predicate does not begin from the
+registry. A MySQL 5.7 `EXPLAIN` may still report `Using temporary; Using filesort`
+for an outer `ORDER BY` over the joined projection; this is sorting, not view
+materialization. The view must expose no raw compressed column or dictionary bytes
+to DBeaver users:
+
+```bash
+multi_compress db view postgres \
+  --table app.events --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable --columns id,created_at,status --as payload
+```
+
+For MySQL 5.7 the generator emits `ALGORITHM=MERGE SQL SECURITY DEFINER`. Do not
+add `DISTINCT`, aggregates, `GROUP BY`, `UNION`, subqueries, or an inner `LIMIT` to
+the generated view; they can force materialization and make decoding happen before
+an outer filter/limit.
+
+Filter and order only by ordinary indexed columns. Keep decoding in the SELECT list.
+Use `EXPLAIN` for production query shapes such as `WHERE id >= ... ORDER BY id LIMIT 100`.
+
+## Rollout and rollback
+
+1. Install 0.6 readers on every DB reader host before writing an MCDB2 row.
+2. PostgreSQL: run `ALTER EXTENSION multi_compress UPDATE`; MySQL: use the bundle's
+   controlled UDF upgrade flow, which preserves both MCDB1 and MCDB2 functions.
+3. Create registry, registry validation/freeze triggers, source FK, and readable view.
+4. Train/register a dictionary and enable the MCDB2 writer for its new column.
+
+After the first MCDB2 write, a reader at 0.5.x cannot read that row. Roll back the
+application writer first; retain 0.6 readers until all MCDB2 values are retired.

data/docs/rfcs/0002-mcdb2-dictionaries.md

@@ -0,0 +1,75 @@
+# RFC 0002: MCDB2 dictionary-backed SQL-readable database envelopes
+
+**Status:** accepted for MultiCompress 0.6.0
+
+## Decision
+
+Introduce MCDB2 as an opt-in storage format for columns containing many small,
+homogeneous UTF-8 documents. MCDB2 uses a zstd dictionary and remains readable from
+Ruby, PostgreSQL and MySQL 5.7. MCDB1 is unchanged and remains dictionary-free.
+
+## Non-goals
+
+- No adaptive mixing of MCDB1 and MCDB2 inside one newly designed column.
+- No dictionary files in `db package` archives.
+- No session registration, global process registry, filesystem lookup, SPI lookup,
+  or implicit dictionary lookup inside the native reader.
+- No content search on decoded payloads.
+
+## Design
+
+MCDB2 uses the 27-byte format specified in `database-envelope-v2.md`. Its registry
+is application-owned, append-only data. Payload rows hold a NOT NULL FK to one
+registry version, and a generated trigger rejects a row unless its MCDB2 header
+`dictionary_ref` equals that FK. Generated views JOIN that version and pass all four
+dictionary arguments to a pure native decoder.
+
+```sql
+SELECT e.id,
+       multi_compress_db_decompress_dict(
+         e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes
+       ) AS payload
+FROM app.events e
+JOIN app.mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id;
+```
+
+The function cache is a bounded LRU implementation detail, not a source of data.
+Its key is `dictionary_id + dictionary_sha256`; cache hits also compare canonical
+stored bytes before reuse. This preserves result correctness for direct calls with
+arbitrary SQL arguments.
+
+## Access model
+
+The view owner has access to the source table and dictionary registry. DBeaver users
+receive only `SELECT` on the view, never source-table or registry-table access.
+
+PostgreSQL needs `EXECUTE` on decoder functions for a role selecting the view; grant
+that deliberately to the read role, revoke from PUBLIC, and do not treat the SQL ABI
+as a public arbitrary-dictionary API. MySQL views use a stable `SQL SECURITY DEFINER`
+account; include that account in backup/restore checks.
+
+## Registry invariants
+
+- version rows are INSERT-only under a dedicated NOLOGIN owner in PostgreSQL;
+- `id > 0`, `id <= INT64_MAX` for cross-DB compatibility;
+- SHA-256 matches bytes on INSERT;
+- zstd DictID is non-zero and matches bytes on INSERT;
+- dictionary bytes are at most 256 KiB;
+- payload FK deletion is restricted; dictionary bytes are never updated in place.
+
+## Performance contract
+
+The decoder is intentionally in the projection, after a selective indexed predicate.
+CI validates PostgreSQL and MySQL query plans for `WHERE id >= ... ORDER BY id LIMIT
+10/100/500`. MySQL's generated MCDB2 view uses a source-first `STRAIGHT_JOIN`; the
+plan must remain MERGE-able and read the source through a selective range/ref access
+before dictionary lookup, with no warning or `<derived>` row. MySQL 5.7 may use a
+temporary/filesort implementation for the outer `ORDER BY` over a joined projection;
+that does not mean the view itself was materialized. Benchmark acceptance is separate
+from timing-sensitive CI: it uses a time-split corpus, warm/cold runs, and concurrent
+sessions.
+
+An optional `multi_compress_db_original_size(blob)` helper can support size-aware
+administrative queries. PostgreSQL may use it in a stored generated column; MySQL
+5.7 must use an ordinary maintained column because loadable UDFs are not supported
+in generated columns.

data/ext/multi_compress/multi_compress.c

@@ -1560,6 +1560,20 @@ static VALUE compress_compress(int argc, VALUE *argv, VALUE self) {
     return Qnil;
 }
 
+static VALUE zstd_frame_dictionary_id(VALUE self, VALUE data) {
+    (void)self;
+    StringValue(data);
+
+    const void *src = RSTRING_PTR(data);
+    size_t src_len = (size_t)RSTRING_LEN(data);
+    size_t frame_size = ZSTD_findFrameCompressedSize(src, src_len);
+    if (ZSTD_isError(frame_size) || frame_size != src_len)
+        rb_raise(eDataError, "zstd: expected exactly one valid frame with no trailing bytes");
+
+    RB_GC_GUARD(data);
+    return UINT2NUM(ZSTD_getDictID_fromFrame(src, src_len));
+}
+
 static VALUE zstd_single_frame_info(VALUE self, VALUE data) {
     (void)self;
     StringValue(data);
@@ -3471,6 +3485,20 @@ static VALUE dict_algo(VALUE self) {
     return algo_to_sym(d->algo);
 }
 
+static VALUE dict_bytes(VALUE self) {
+    dictionary_t *d;
+    TypedData_Get_Struct(self, dictionary_t, &dictionary_type, d);
+    return rb_binary_str_new((const char *)d->data, (long)d->size);
+}
+
+static VALUE dict_zstd_id(VALUE self) {
+    dictionary_t *d;
+    TypedData_Get_Struct(self, dictionary_t, &dictionary_type, d);
+    if (d->algo != ALGO_ZSTD)
+        rb_raise(eUnsupportedError, "zstd_id is available only for zstd dictionaries");
+    return UINT2NUM(ZSTD_getDictID_fromDict(d->data, d->size));
+}
+
 static VALUE dict_size(VALUE self) {
     dictionary_t *d;
     TypedData_Get_Struct(self, dictionary_t, &dictionary_type, d);
@@ -3508,6 +3536,8 @@ void Init_multi_compress(void) {
     rb_define_module_function(mMultiCompress, "compress", compress_compress, -1);
     rb_define_module_function(mMultiCompress, "decompress", compress_decompress, -1);
     rb_define_module_function(mMultiCompress, "zstd_single_frame_info", zstd_single_frame_info, 1);
+    rb_define_module_function(mMultiCompress, "zstd_frame_dictionary_id", zstd_frame_dictionary_id,
+                              1);
     rb_define_module_function(mMultiCompress, "crc32", compress_crc32, -1);
     rb_define_module_function(mMultiCompress, "adler32", compress_adler32, -1);
     rb_define_module_function(mMultiCompress, "algorithms", compress_algorithms, 0);
@@ -3542,6 +3572,8 @@ void Init_multi_compress(void) {
     rb_define_singleton_method(cDictionary, "load", dict_load, -1);
     rb_define_method(cDictionary, "save", dict_save, 1);
     rb_define_method(cDictionary, "algo", dict_algo, 0);
+    rb_define_method(cDictionary, "bytes", dict_bytes, 0);
+    rb_define_method(cDictionary, "zstd_id", dict_zstd_id, 0);
     rb_define_method(cDictionary, "size", dict_size, 0);
     rb_define_singleton_method(mZstd, "train_dictionary", zstd_train_dictionary, -1);
     rb_define_singleton_method(mBrotli, "train_dictionary", brotli_train_dictionary, -1);