multi_compress 0.5.0 → 0.6.0

data/mysql_udf/src/multi_compress_mysql.c

@@ -1,3 +1,4 @@
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -24,6 +25,145 @@ typedef my_bool mcdb_udf_init_result_t;
 
 #include "mcdb_format.h"
 
+#define MCDB_MYSQL_DDICT_CACHE_ENTRIES 4
+
+typedef struct {
+    uint64_t dictionary_ref;
+    unsigned char sha256[MCDB_SHA256_BYTES];
+    unsigned char *bytes;
+    size_t bytes_len;
+    ZSTD_DDict *ddict;
+    uint64_t used_at;
+} mcdb_mysql_cache_entry;
+
+typedef struct {
+    char *output;
+    ZSTD_DCtx *dctx;
+    uint64_t clock;
+    mcdb_mysql_cache_entry entries[MCDB_MYSQL_DDICT_CACHE_ENTRIES];
+} mcdb_mysql_dictionary_state;
+
+static void mcdb_mysql_state_free(mcdb_mysql_dictionary_state *state) {
+    int i;
+    if (!state)
+        return;
+    free(state->output);
+    if (state->dctx)
+        ZSTD_freeDCtx(state->dctx);
+    for (i = 0; i < MCDB_MYSQL_DDICT_CACHE_ENTRIES; i++) {
+        free(state->entries[i].bytes);
+        if (state->entries[i].ddict)
+            ZSTD_freeDDict(state->entries[i].ddict);
+    }
+    free(state);
+}
+
+static mcdb_mysql_dictionary_state *mcdb_mysql_state_new(void) {
+    mcdb_mysql_dictionary_state *state = (mcdb_mysql_dictionary_state *)calloc(1, sizeof(*state));
+    if (!state)
+        return NULL;
+    state->dctx = ZSTD_createDCtx();
+    if (!state->dctx) {
+        free(state);
+        return NULL;
+    }
+    return state;
+}
+
+static int mcdb_mysql_parse_ref(UDF_ARGS *args, uint64_t *out_ref) {
+    long long value;
+    if (!args->args[1])
+        return 0;
+    memcpy(&value, args->args[1], sizeof(value));
+    if (value <= 0)
+        return 0;
+    *out_ref = (uint64_t)value;
+    return 1;
+}
+
+static int mcdb_mysql_cache_victim(mcdb_mysql_dictionary_state *state, int same_key) {
+    int i;
+    int victim = 0;
+    uint64_t oldest = UINT64_MAX;
+    if (same_key >= 0)
+        return same_key;
+    for (i = 0; i < MCDB_MYSQL_DDICT_CACHE_ENTRIES; i++) {
+        if (!state->entries[i].ddict)
+            return i;
+        if (state->entries[i].used_at < oldest) {
+            oldest = state->entries[i].used_at;
+            victim = i;
+        }
+    }
+    return victim;
+}
+
+static ZSTD_DDict *mcdb_mysql_cached_ddict(mcdb_mysql_dictionary_state *state,
+                                           uint64_t dictionary_ref,
+                                           const unsigned char *dictionary_sha256,
+                                           const unsigned char *dictionary, size_t dictionary_len) {
+    unsigned char actual_sha256[MCDB_SHA256_BYTES];
+    int i;
+    int same_key = -1;
+    int victim;
+    ZSTD_DDict *ddict;
+
+    for (i = 0; i < MCDB_MYSQL_DDICT_CACHE_ENTRIES; i++) {
+        mcdb_mysql_cache_entry *entry = &state->entries[i];
+        if (!entry->ddict || entry->dictionary_ref != dictionary_ref ||
+            memcmp(entry->sha256, dictionary_sha256, MCDB_SHA256_BYTES) != 0)
+            continue;
+        same_key = i;
+        if (entry->bytes_len == dictionary_len &&
+            memcmp(entry->bytes, dictionary, dictionary_len) == 0) {
+            entry->used_at = ++state->clock;
+            return entry->ddict;
+        }
+        break;
+    }
+
+    mcdb_sha256(dictionary, dictionary_len, actual_sha256);
+    if (memcmp(actual_sha256, dictionary_sha256, MCDB_SHA256_BYTES) != 0)
+        return NULL;
+    if (mcdb_validate_dictionary(dictionary, dictionary_len, NULL) != MCDB_OK)
+        return NULL;
+    ddict = ZSTD_createDDict(dictionary, dictionary_len);
+    if (!ddict)
+        return NULL;
+
+    victim = mcdb_mysql_cache_victim(state, same_key);
+    free(state->entries[victim].bytes);
+    if (state->entries[victim].ddict)
+        ZSTD_freeDDict(state->entries[victim].ddict);
+    state->entries[victim].bytes = (unsigned char *)malloc(dictionary_len);
+    if (!state->entries[victim].bytes) {
+        ZSTD_freeDDict(ddict);
+        state->entries[victim].ddict = NULL;
+        state->entries[victim].bytes_len = 0;
+        return NULL;
+    }
+    memcpy(state->entries[victim].bytes, dictionary, dictionary_len);
+    state->entries[victim].dictionary_ref = dictionary_ref;
+    memcpy(state->entries[victim].sha256, dictionary_sha256, MCDB_SHA256_BYTES);
+    state->entries[victim].bytes_len = dictionary_len;
+    state->entries[victim].ddict = ddict;
+    state->entries[victim].used_at = ++state->clock;
+    return ddict;
+}
+
+static mcdb_udf_init_result_t mcdb_one_blob_init(UDF_INIT *initid, UDF_ARGS *args, char *message,
+                                                 const char *signature, unsigned long max_length) {
+    if (args->arg_count != 1) {
+        snprintf(message, 255, "%s takes exactly one argument", signature);
+        return 1;
+    }
+    args->arg_type[0] = STRING_RESULT;
+    initid->maybe_null = 1;
+    initid->const_item = 0;
+    initid->max_length = max_length;
+    return 0;
+}
+
 mcdb_udf_init_result_t multi_compress_db_version_init(UDF_INIT *initid, UDF_ARGS *args,
                                                       char *message) {
     (void)args;
@@ -33,101 +173,300 @@ mcdb_udf_init_result_t multi_compress_db_version_init(UDF_INIT *initid, UDF_ARGS
     }
     initid->maybe_null = 0;
     initid->const_item = 1;
-    initid->max_length = 64;
+    initid->max_length = 96;
     return 0;
 }
 
 char *multi_compress_db_version(UDF_INIT *initid, UDF_ARGS *args, char *result,
                                 unsigned long *length, char *is_null, char *error) {
+    int n;
     (void)initid;
     (void)args;
     *is_null = 0;
     *error = 0;
-    /* result buffer is 255 bytes per the UDF ABI */
-    int n = snprintf(result, 255, "multi_compress reader %s; MCDB1; zstd %s", MCDB_READER_VERSION,
-                     ZSTD_versionString());
+    n = snprintf(result, 255, "multi_compress reader %s; MCDB1 + MCDB2; zstd %s",
+                 MCDB_READER_VERSION, ZSTD_versionString());
     *length = (unsigned long)(n > 0 ? n : 0);
     return result;
 }
 
 mcdb_udf_init_result_t multi_compress_db_is_valid_init(UDF_INIT *initid, UDF_ARGS *args,
                                                        char *message) {
-    if (args->arg_count != 1) {
-        strcpy(message, "multi_compress_db_is_valid(blob) takes exactly one argument");
-        return 1;
-    }
-    args->arg_type[0] = STRING_RESULT;
-    initid->maybe_null = 1;
-    initid->const_item = 0;
-    initid->max_length = 1;
-    return 0;
+    return mcdb_one_blob_init(initid, args, message, "multi_compress_db_is_valid(blob)", 1);
 }
 
 long long multi_compress_db_is_valid(UDF_INIT *initid, UDF_ARGS *args, char *is_null, char *error) {
+    unsigned char *out = NULL;
+    size_t out_len = 0;
+    char err[MCDB_ERRLEN];
+    mcdb_status st;
     (void)initid;
     *error = 0;
-    if (args->args[0] == NULL) {
+    if (!args->args[0]) {
         *is_null = 1;
         return 0;
     }
     *is_null = 0;
-
-    unsigned char *out = NULL;
-    size_t out_len = 0;
-    char err[MCDB_ERRLEN];
-    mcdb_status st = mcdb_decode((const unsigned char *)args->args[0], (size_t)args->lengths[0],
-                                 &out, &out_len, err);
+    st = mcdb_decode((const unsigned char *)args->args[0], (size_t)args->lengths[0], &out, &out_len,
+                     err);
     free(out);
     return st == MCDB_OK ? 1 : 0;
 }
 
 mcdb_udf_init_result_t multi_compress_db_decompress_init(UDF_INIT *initid, UDF_ARGS *args,
                                                          char *message) {
-    if (args->arg_count != 1) {
-        strcpy(message, "multi_compress_db_decompress(blob) takes exactly one argument");
-        return 1;
-    }
-    args->arg_type[0] = STRING_RESULT;
-    initid->maybe_null = 1;
-    initid->const_item = 0;
-    initid->max_length = MCDB_MAX_OUTPUT; /* result can be up to 16 MiB, not the input size */
-    initid->ptr = NULL;                   /* holds the malloc'd result between _decompress calls */
+    mcdb_udf_init_result_t result = mcdb_one_blob_init(
+        initid, args, message, "multi_compress_db_decompress(blob)", MCDB_MAX_OUTPUT);
+    if (result)
+        return result;
+    initid->ptr = NULL;
     return 0;
 }
 
 void multi_compress_db_decompress_deinit(UDF_INIT *initid) {
-    if (initid->ptr) {
-        free(initid->ptr);
-        initid->ptr = NULL;
-    }
+    free(initid->ptr);
+    initid->ptr = NULL;
 }
 
 char *multi_compress_db_decompress(UDF_INIT *initid, UDF_ARGS *args, char *result,
                                    unsigned long *length, char *is_null, char *error) {
+    unsigned char *out = NULL;
+    size_t out_len = 0;
+    char err[MCDB_ERRLEN];
+    mcdb_status st;
     (void)result;
     *error = 0;
-    if (args->args[0] == NULL) {
+    if (!args->args[0]) {
         *is_null = 1;
         return NULL;
     }
-    if (initid->ptr) {
-        free(initid->ptr);
-        initid->ptr = NULL;
+    free(initid->ptr);
+    initid->ptr = NULL;
+    st = mcdb_decode((const unsigned char *)args->args[0], (size_t)args->lengths[0], &out, &out_len,
+                     err);
+    if (st != MCDB_OK) {
+        *is_null = 1;
+        free(out);
+        return NULL;
     }
+    initid->ptr = (char *)out;
+    *length = (unsigned long)out_len;
+    *is_null = 0;
+    return (char *)out;
+}
 
-    unsigned char *out = NULL;
-    size_t out_len = 0;
+mcdb_udf_init_result_t multi_compress_db_original_size_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                            char *message) {
+    return mcdb_one_blob_init(initid, args, message, "multi_compress_db_original_size(blob)", 20);
+}
+
+long long multi_compress_db_original_size(UDF_INIT *initid, UDF_ARGS *args, char *is_null,
+                                          char *error) {
+    mcdb_header header;
+    mcdb_status st;
+    (void)initid;
+    *error = 0;
+    if (!args->args[0]) {
+        *is_null = 1;
+        return 0;
+    }
+    st = mcdb_parse_header((const unsigned char *)args->args[0], (size_t)args->lengths[0], &header);
+    if (st != MCDB_OK) {
+        *is_null = 1;
+        return 0;
+    }
+    *is_null = 0;
+    return (long long)header.original_size;
+}
+
+mcdb_udf_init_result_t multi_compress_db_dictionary_ref_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                             char *message) {
+    return mcdb_one_blob_init(initid, args, message, "multi_compress_db_dictionary_ref(blob)", 20);
+}
+
+long long multi_compress_db_dictionary_ref(UDF_INIT *initid, UDF_ARGS *args, char *is_null,
+                                           char *error) {
+    uint64_t ref;
+    mcdb_status st;
+    (void)initid;
+    *error = 0;
+    if (!args->args[0]) {
+        *is_null = 1;
+        return 0;
+    }
+    ref = mcdb_dictionary_ref((const unsigned char *)args->args[0], (size_t)args->lengths[0], &st);
+    if (st != MCDB_OK) {
+        *is_null = 1;
+        return 0;
+    }
+    *is_null = 0;
+    return (long long)ref;
+}
+
+mcdb_udf_init_result_t multi_compress_db_dictionary_zstd_id_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                                 char *message) {
+    return mcdb_one_blob_init(initid, args, message,
+                              "multi_compress_db_dictionary_zstd_id(dictionary)", 20);
+}
+
+long long multi_compress_db_dictionary_zstd_id(UDF_INIT *initid, UDF_ARGS *args, char *is_null,
+                                               char *error) {
+    uint32_t id;
+    (void)initid;
+    *error = 0;
+    if (!args->args[0] || mcdb_validate_dictionary((const unsigned char *)args->args[0],
+                                                   (size_t)args->lengths[0], &id) != MCDB_OK) {
+        *is_null = 1;
+        return 0;
+    }
+    *is_null = 0;
+    return (long long)id;
+}
+
+mcdb_udf_init_result_t multi_compress_db_dictionary_sha256_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                                char *message) {
+    return mcdb_one_blob_init(initid, args, message,
+                              "multi_compress_db_dictionary_sha256(dictionary)", MCDB_SHA256_BYTES);
+}
+
+char *multi_compress_db_dictionary_sha256(UDF_INIT *initid, UDF_ARGS *args, char *result,
+                                          unsigned long *length, char *is_null, char *error) {
+    (void)initid;
+    *error = 0;
+    if (!args->args[0]) {
+        *is_null = 1;
+        return NULL;
+    }
+    mcdb_sha256((const unsigned char *)args->args[0], (size_t)args->lengths[0],
+                (unsigned char *)result);
+    *length = MCDB_SHA256_BYTES;
+    *is_null = 0;
+    return result;
+}
+
+static mcdb_udf_init_result_t mcdb_dict_init(UDF_INIT *initid, UDF_ARGS *args, char *message,
+                                             const char *signature, unsigned long max_length) {
+    mcdb_mysql_dictionary_state *state;
+    if (args->arg_count != 4) {
+        snprintf(message, 255, "%s takes exactly four arguments", signature);
+        return 1;
+    }
+    args->arg_type[0] = STRING_RESULT;
+    args->arg_type[1] = INT_RESULT;
+    args->arg_type[2] = STRING_RESULT;
+    args->arg_type[3] = STRING_RESULT;
+    initid->maybe_null = 1;
+    initid->const_item = 0;
+    initid->max_length = max_length;
+    state = mcdb_mysql_state_new();
+    if (!state) {
+        strcpy(message, "multi_compress: could not allocate zstd dictionary cache");
+        return 1;
+    }
+    initid->ptr = (char *)state;
+    return 0;
+}
+
+mcdb_udf_init_result_t multi_compress_db_is_valid_dict_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                            char *message) {
+    return mcdb_dict_init(
+        initid, args, message,
+        "multi_compress_db_is_valid_dict(blob, dictionary_id, dictionary_sha256, dictionary)", 1);
+}
+
+mcdb_udf_init_result_t multi_compress_db_decompress_dict_init(UDF_INIT *initid, UDF_ARGS *args,
+                                                              char *message) {
+    return mcdb_dict_init(
+        initid, args, message,
+        "multi_compress_db_decompress_dict(blob, dictionary_id, dictionary_sha256, dictionary)",
+        MCDB_MAX_OUTPUT);
+}
+
+void multi_compress_db_is_valid_dict_deinit(UDF_INIT *initid) {
+    mcdb_mysql_state_free((mcdb_mysql_dictionary_state *)initid->ptr);
+    initid->ptr = NULL;
+}
+
+void multi_compress_db_decompress_dict_deinit(UDF_INIT *initid) {
+    mcdb_mysql_state_free((mcdb_mysql_dictionary_state *)initid->ptr);
+    initid->ptr = NULL;
+}
+
+static mcdb_status mcdb_mysql_decode_v2(UDF_INIT *initid, UDF_ARGS *args, unsigned char **out,
+                                        size_t *out_len) {
+    mcdb_mysql_dictionary_state *state = (mcdb_mysql_dictionary_state *)initid->ptr;
+    uint64_t dictionary_ref;
+    ZSTD_DDict *ddict;
+    uint64_t original_size;
+    unsigned char *buffer;
     char err[MCDB_ERRLEN];
-    mcdb_status st = mcdb_decode((const unsigned char *)args->args[0], (size_t)args->lengths[0],
-                                 &out, &out_len, err);
+    mcdb_status st;
+
+    if (!args->args[0] || !args->args[1] || !args->args[2] || !args->args[3])
+        return MCDB_ERR_DICTIONARY_REQUIRED;
+    if (args->lengths[2] != MCDB_SHA256_BYTES || !mcdb_mysql_parse_ref(args, &dictionary_ref))
+        return MCDB_ERR_DICTIONARY_REF;
+    ddict = mcdb_mysql_cached_ddict(state, dictionary_ref, (const unsigned char *)args->args[2],
+                                    (const unsigned char *)args->args[3], (size_t)args->lengths[3]);
+    if (!ddict)
+        return MCDB_ERR_DICTIONARY_ID;
+    st = mcdb_validate_v2_header((const unsigned char *)args->args[0], (size_t)args->lengths[0],
+                                 &original_size, NULL);
+    if (st != MCDB_OK)
+        return st;
+    buffer = (unsigned char *)malloc(original_size ? (size_t)original_size : 1);
+    if (!buffer)
+        return MCDB_ERR_ALLOC;
+    st = mcdb_decode_v2_into((const unsigned char *)args->args[0], (size_t)args->lengths[0],
+                             dictionary_ref, (const unsigned char *)args->args[3],
+                             (size_t)args->lengths[3], state->dctx, ddict, buffer,
+                             (size_t)original_size, out_len, err);
     if (st != MCDB_OK) {
+        free(buffer);
+        return st;
+    }
+    *out = buffer;
+    return MCDB_OK;
+}
+
+long long multi_compress_db_is_valid_dict(UDF_INIT *initid, UDF_ARGS *args, char *is_null,
+                                          char *error) {
+    unsigned char *out = NULL;
+    size_t out_len = 0;
+    mcdb_status st;
+    *error = 0;
+    if (!args->args[0] || !args->args[1] || !args->args[2] || !args->args[3]) {
+        *is_null = 1;
+        return 0;
+    }
+    st = mcdb_mysql_decode_v2(initid, args, &out, &out_len);
+    free(out);
+    *is_null = 0;
+    return st == MCDB_OK ? 1 : 0;
+}
+
+char *multi_compress_db_decompress_dict(UDF_INIT *initid, UDF_ARGS *args, char *result,
+                                        unsigned long *length, char *is_null, char *error) {
+    mcdb_mysql_dictionary_state *state = (mcdb_mysql_dictionary_state *)initid->ptr;
+    unsigned char *out = NULL;
+    size_t out_len = 0;
+    mcdb_status st;
+    (void)result;
+    *error = 0;
+    if (!args->args[0] || !args->args[1] || !args->args[2] || !args->args[3]) {
         *is_null = 1;
+        return NULL;
+    }
+    free(state->output);
+    state->output = NULL;
+    st = mcdb_mysql_decode_v2(initid, args, &out, &out_len);
+    if (st != MCDB_OK) {
         free(out);
+        *is_null = 1;
         return NULL;
     }
-
-    initid->ptr = (char *)out;
+    state->output = (char *)out;
     *length = (unsigned long)out_len;
     *is_null = 0;
-    return (char *)out;
+    return state->output;
 }

data/mysql_udf/test/run_e2e.sh

@@ -74,6 +74,23 @@ compress_hex() {
     -e 'print MultiCompress::Database.compress(ENV.fetch("MCDB_TEXT")).unpack1("H*")'
 }
 
+mcdb2_fixture() {
+  MCDB_TEXT="$1" ruby -Ilib -r multi_compress -r multi_compress/database <<'RUBY'
+    samples = 256.times.map do |i|
+      %({"kind":"event","tenant":#{i % 8},"metadata":{"source":"worker","version":1,"name":"same-shape-#{i % 16}"},"payload":"#{"x" * (48 + i % 64)}"})
+    end
+
+    dictionary = MultiCompress::Database::Dictionary.train(samples, id: 42, size: 4096)
+    blob = MultiCompress::Database.compress(ENV.fetch("MCDB_TEXT"), dictionary: dictionary)
+
+    puts dictionary.id
+    puts dictionary.zstd_id
+    puts dictionary.sha256.unpack1("H*")
+    puts dictionary.bytes.unpack1("H*")
+    puts blob.unpack1("H*")
+RUBY
+}
+
 bundle_name() {
   ruby -I "$PACKAGED_GEM_ROOT/lib" -r multi_compress/version \
     -e 'print "multi_compress-mysql-#{MultiCompress::VERSION}"'
@@ -138,6 +155,26 @@ apply_readable_view() {
   docker exec -i "$CONTAINER" mysql --default-character-set=utf8mb4 < "$path"
 }
 
+apply_dictionary_readable_view() {
+  local path="$1"
+  local warnings
+  db_cli view mysql \
+    --table app.dictionary_events \
+    --column payload_compressed \
+    --dictionary-table app.mcdb_dictionary_versions \
+    --dictionary-id-column payload_dictionary_id \
+    --view admin.dictionary_events_readable \
+    --columns id \
+    --output "$path"
+  warnings=$( { cat "$path"; printf '\nSHOW WARNINGS;\n'; } | docker exec -i "$CONTAINER" mysql \
+    --default-character-set=utf8mb4 --batch --skip-column-names --raw)
+  if printf '%s\n' "$warnings" | grep -Eqi '(undefined|merge)'; then
+    echo "MySQL refused the required MERGE view algorithm:" >&2
+    printf '%s\n' "$warnings" >&2
+    exit 1
+  fi
+}
+
 ensure_ruby_extension
 prepare_packaged_gem
 build_deployment_bundle
@@ -151,9 +188,23 @@ BIG=$(ruby -e 'print "a" * 300')
 VALID_HEX=$(compress_hex "$TEXT")
 BIG_HEX=$(compress_hex "$BIG")
 CORRUPT_HEX=$(fixture_hex test/fixtures/database_v1/corrupt_crc.mcdb)
+MCDB2_FIXTURE_OUTPUT="$(mcdb2_fixture "$TEXT")"
+readarray -t MCDB2_FIXTURE <<< "$MCDB2_FIXTURE_OUTPUT"
+if [ "${#MCDB2_FIXTURE[@]}" -ne 5 ] || [ -z "${MCDB2_FIXTURE[0]}" ] || [ -z "${MCDB2_FIXTURE[4]}" ]; then
+  echo "MCDB2 fixture generator returned an invalid payload" >&2
+  exit 2
+fi
+DICT_REF="${MCDB2_FIXTURE[0]}"
+DICT_ZSTD_ID="${MCDB2_FIXTURE[1]}"
+DICT_SHA_HEX="${MCDB2_FIXTURE[2]}"
+DICT_HEX="${MCDB2_FIXTURE[3]}"
+DICT_BLOB_HEX="${MCDB2_FIXTURE[4]}"
+TEXT_BYTES=$(MCDB_TEXT="$TEXT" ruby -e 'print ENV.fetch("MCDB_TEXT").bytesize')
 BUNDLE_NAME="$(bundle_name)"
 VIEW_SQL="$(mktemp -t multi-compress-mysql-view.XXXXXX.sql)"
-trap 'rm -f "$VIEW_SQL"; cleanup' EXIT
+DICT_VIEW_SQL="$(mktemp -t multi-compress-mysql-dictionary-view.XXXXXX.sql)"
+REGISTRY_SQL="$(mktemp -t multi-compress-mysql-registry.XXXXXX.sql)"
+trap 'rm -f "$VIEW_SQL" "$DICT_VIEW_SQL" "$REGISTRY_SQL"; cleanup' EXIT
 
 docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
 docker run --platform "$PLATFORM" -d --name "$CONTAINER" \
@@ -202,6 +253,35 @@ q "DROP DATABASE IF EXISTS app;
      (6, UNHEX('$VALID_HEX'));"
 apply_readable_view "$VIEW_SQL"
 
+# MCDB2: registry DDL lives in app and the generated view supplies dictionary bytes through an INNER JOIN.
+q "USE app;
+   CREATE TABLE dictionary_events (
+     id INT PRIMARY KEY,
+     payload_compressed LONGBLOB NOT NULL,
+     payload_dictionary_id BIGINT UNSIGNED NOT NULL
+   ) ENGINE=InnoDB;"
+db_cli registry mysql --database app \
+  --payload-table dictionary_events --payload-column payload_compressed \
+  --payload-dictionary-id-column payload_dictionary_id \
+  --output "$REGISTRY_SQL"
+docker exec -i "$CONTAINER" mysql --default-character-set=utf8mb4 < "$REGISTRY_SQL"
+q "USE app;
+   INSERT INTO mcdb_dictionary_versions (id, family, zstd_dict_id, sha256, bytes)
+     VALUES ($DICT_REF, 'events_payload_v1', $DICT_ZSTD_ID, UNHEX('$DICT_SHA_HEX'), UNHEX('$DICT_HEX'));
+   INSERT INTO dictionary_events (id, payload_compressed, payload_dictionary_id)
+   SELECT seq.id, UNHEX('$DICT_BLOB_HEX'), $DICT_REF
+   FROM (
+     SELECT ones.n + tens.n * 10 + hundreds.n * 100 + 1 AS id
+     FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+           UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS ones
+     CROSS JOIN (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+                 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS tens
+     CROSS JOIN (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4
+                 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS hundreds
+   ) AS seq
+   WHERE seq.id <= 1000;"
+apply_dictionary_readable_view "$DICT_VIEW_SQL"
+
 fails=0
 check() {
   if [ "$2" = "$3" ]; then echo "  PASS  $1"; else echo "  FAIL  $1 (got [$2] want [$3])"; fails=$((fails + 1)); fi
@@ -238,18 +318,56 @@ done
 check "is_valid false (crc)" \
   "$(q "SELECT multi_compress_db_is_valid(UNHEX('$(fixture_hex test/fixtures/database_v1/corrupt_crc.mcdb)'));")" \
   "0"
+check "MCDB2 dictionary reference" "$(q 'USE app; SELECT multi_compress_db_dictionary_ref(payload_compressed) FROM dictionary_events WHERE id=1;')" "$DICT_REF"
+check "MCDB2 original size" "$(q 'USE app; SELECT multi_compress_db_original_size(payload_compressed) FROM dictionary_events WHERE id=1;')" "$TEXT_BYTES"
+check "MCDB2 generated readable view" "$(q 'SELECT payload FROM admin.dictionary_events_readable WHERE id=1;')" "$TEXT"
+check "MCDB2 dictionary is valid" \
+  "$(q 'USE app; SELECT multi_compress_db_is_valid_dict(e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes) FROM dictionary_events e JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id WHERE e.id=1;')" \
+  "1"
+check_err "MCDB2 payload/FK mismatch rejected" "USE app; INSERT INTO dictionary_events VALUES (1001, UNHEX('$DICT_BLOB_HEX'), $((DICT_REF + 1)));"
+check "MCDB2 rejects wrong dictionary digest" \
+  "$(q 'USE app; SELECT multi_compress_db_is_valid_dict(e.payload_compressed, e.payload_dictionary_id, UNHEX(REPEAT("00", 32)), d.bytes) FROM dictionary_events e JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id WHERE e.id=1;')" \
+  "0"
+MYSQL_DICT_PLAN="$(q 'EXPLAIN SELECT id, payload FROM admin.dictionary_events_readable WHERE id BETWEEN 100 AND 900 ORDER BY id LIMIT 100;')"
+if printf '%s\n' "$MYSQL_DICT_PLAN" | grep -Fq '<derived'; then
+  echo "  FAIL  MCDB2 view materialized as a derived table"
+  printf '%s\n' "$MYSQL_DICT_PLAN" >&2
+  fails=$((fails + 1))
+elif ! printf '%s\n' "$MYSQL_DICT_PLAN" | awk -F '\t' '$3 == "source" && $5 == "range" && $7 == "PRIMARY" { found = 1 } END { exit(found ? 0 : 1) }'; then
+  echo "  FAIL  MCDB2 view did not use the source PRIMARY range scan"
+  printf '%s\n' "$MYSQL_DICT_PLAN" >&2
+  fails=$((fails + 1))
+else
+  # `Using temporary` / `Using filesort` here belongs to the outer ORDER BY
+  # over a joined projection. It is not evidence that the view was materialized:
+  # MySQL reports materialized views as <derived...>. The important contract is
+  # that the source predicate is merged and reaches the PRIMARY range scan.
+  echo "  PASS  MCDB2 view is MERGE-able and keeps the source PRIMARY range scan"
+fi
 check "max_allowed_packet is reported" "$(q 'SELECT @@GLOBAL.max_allowed_packet > 0;')" "1"
 
 MYSQL_SOCKET_PATH="$(q 'SELECT @@socket;' | tr -d '\r\n')"
+# Simulate an installed 0.5.x server: only the three MCDB1 UDF registrations
+# remain. 0.6 must recognise this as upgradeable, not partial, then use the
+# documented DROP -> replace library -> CREATE lifecycle to register all nine.
 docker exec -u 0 \
   -e "MCDB_BUNDLE_ROOT=${BUNDLE_PARENT}/${BUNDLE_NAME}" \
   -e "MCDB_MYSQL_SOCKET=$MYSQL_SOCKET_PATH" \
   "$CONTAINER" sh -ceu '
+    mysql --protocol=SOCKET --socket="$MCDB_MYSQL_SOCKET" -e "
+      DROP FUNCTION multi_compress_db_decompress_dict;
+      DROP FUNCTION multi_compress_db_is_valid_dict;
+      DROP FUNCTION multi_compress_db_dictionary_sha256;
+      DROP FUNCTION multi_compress_db_dictionary_zstd_id;
+      DROP FUNCTION multi_compress_db_dictionary_ref;
+      DROP FUNCTION multi_compress_db_original_size;"
     cd "$MCDB_BUNDLE_ROOT"
+    make status MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "state: MCDB1-only"
     make doctor MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "server socket: $MCDB_MYSQL_SOCKET"
     make upgrade CONFIRM=UPGRADE_MULTI_COMPRESS MYSQL_SOCKET="$MCDB_MYSQL_SOCKET"
     make status MYSQL_SOCKET="$MCDB_MYSQL_SOCKET" | grep -F "state: enabled"
   '
+check "upgrade from MCDB1-only restores MCDB2 functions" "$(q 'SELECT multi_compress_db_original_size(payload_compressed) FROM app.dictionary_events WHERE id=1;')" "$TEXT_BYTES"
 check "upgrade keeps version() available" "$(q 'SELECT LOCATE(CHAR(77,67,68,66,49), multi_compress_db_version()) > 0;')" "1"
 
 # Register one expected name to a different SONAME and prove enable refuses without

data/postgres_extension/Makefile

@@ -1,7 +1,7 @@
 EXTENSION   = multi_compress
 MODULE_big  = multi_compress_pg
 OBJS        = src/multi_compress_pg.o src/mcdb_format.o
-DATA        = sql/multi_compress--0.5.0.sql
+DATA        = sql/multi_compress--0.5.0.sql sql/multi_compress--0.6.0.sql sql/multi_compress--0.5.0--0.6.0.sql
 
 PG_CONFIG   ?= pg_config
 ZSTD_DIR    ?= ../ext/multi_compress/vendor/zstd/lib
@@ -56,7 +56,7 @@ $(ZSTD_OBJ_DIR)/%.o: $(ZSTD_DIR)/%.S
 $(ZSTD_LIB): $(ZSTD_OBJS) | $(BUILD)
 	$(AR) rcs $@ $^
 
-# The source wrapper includes the common MCDB1 decoder; it is not a fork.
+# The source wrapper includes the shared MCDB1/MCDB2 decoder; it is not a fork.
 src/mcdb_format.o: src/mcdb_format.c $(CORE_DIR)/src/mcdb_format.c $(CORE_DIR)/include/mcdb_format.h
 
 # Golden fixtures are committed. Regeneration is a development action only.