multi_compress 0.5.0 → 0.6.0

data/lib/multi_compress/db_deployment.rb

@@ -14,8 +14,9 @@ require "zlib"
 module MultiCompress
   module DBDeployment
     ROOT = File.expand_path("../..", __dir__)
-    FORMAT = "MCDB1"
-    BUNDLE_FORMAT_VERSION = 1
+    FORMAT = "MCDB1 + MCDB2"
+    BUNDLE_FORMAT_VERSION = 2
+    MAX_DICTIONARY_BYTES = 256 * 1024
 
     class Error < StandardError; end
 
@@ -32,6 +33,8 @@ module MultiCompress
           PackageCommand.new.run(argv)
         when "view"
           ViewCommand.new.run(argv)
+        when "registry"
+          RegistryCommand.new.run(argv)
         when "help", "--help", "-h", nil
           puts help
           0
@@ -66,7 +69,14 @@ module MultiCompress
           install`, then `make enable DB=...`.
 
           `view` emits a safe readable-view definition so DBeaver users query
-          decoded text instead of the compressed bytea/BLOB column.
+          decoded text instead of the compressed bytea/BLOB column. Supply
+          --dictionary-table and --dictionary-id-column for MCDB2.
+
+          `registry` emits append-only MCDB2 dictionary-registry DDL. Dictionary
+          bytes are application data and are never placed in a DBA reader bundle.
+          Supplying --payload-table, --payload-column and
+          --payload-dictionary-id-column also emits the MCDB2 header/FK
+          consistency trigger for that source table.
         TEXT
       end
     end
@@ -105,7 +115,12 @@ module MultiCompress
         raise OptionParser::MissingArgument, "TARGET must be postgres or mysql" if target.nil?
         raise OptionParser::InvalidArgument, "TARGET must be postgres or mysql" unless TARGETS.include?(target)
 
-        options = { columns: [], as: "payload", output: nil, extension_schema: "multi_compress" }
+        options = {
+          columns: [], as: "payload", output: nil, extension_schema: "multi_compress",
+          dictionary_table: nil, dictionary_id_column: nil,
+          registry_id_column: "id", dictionary_sha256_column: "sha256",
+          dictionary_bytes_column: "bytes"
+        }
         parser = OptionParser.new do |opts|
           opts.banner = "Usage: multi_compress db view #{target} --table SCHEMA.TABLE --column COLUMN --view SCHEMA.VIEW --columns ID,CREATED_AT [OPTIONS]"
           opts.on("--table NAME", "source table, optionally schema-qualified") { |value| options[:table] = value }
@@ -113,6 +128,11 @@ module MultiCompress
           opts.on("--view NAME", "destination view, optionally schema-qualified") { |value| options[:view] = value }
           opts.on("--columns LIST", "comma-separated plain columns to expose") { |value| options[:columns] = value.split(",").map(&:strip) }
           opts.on("--as NAME", "decoded column name (default: payload)") { |value| options[:as] = value }
+          opts.on("--dictionary-table NAME", "MCDB2 immutable dictionary registry table") { |value| options[:dictionary_table] = value }
+          opts.on("--dictionary-id-column NAME", "MCDB2 FK column on source table") { |value| options[:dictionary_id_column] = value }
+          opts.on("--registry-id-column NAME", "registry PK (default: id)") { |value| options[:registry_id_column] = value }
+          opts.on("--dictionary-sha256-column NAME", "registry SHA-256 column (default: sha256)") { |value| options[:dictionary_sha256_column] = value }
+          opts.on("--dictionary-bytes-column NAME", "registry dictionary bytes column (default: bytes)") { |value| options[:dictionary_bytes_column] = value }
           if target == "postgres"
             opts.on("--extension-schema NAME", "extension schema (default: multi_compress)") { |value| options[:extension_schema] = value }
           end
@@ -145,6 +165,275 @@ module MultiCompress
       end
     end
 
+    class RegistryCommand
+      TARGETS = %w[postgres mysql].freeze
+      IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/.freeze
+
+      def run(argv)
+        target = argv.shift
+        raise OptionParser::MissingArgument, "TARGET must be postgres or mysql" if target.nil?
+        raise OptionParser::InvalidArgument, "TARGET must be postgres or mysql" unless TARGETS.include?(target)
+
+        options = { output: nil, extension_schema: "multi_compress", table: "mcdb_dictionary_versions", heads_table: "mcdb_dictionary_heads" }
+        parser = OptionParser.new do |opts|
+          opts.banner = "Usage: multi_compress db registry #{target} [OPTIONS]"
+          if target == "postgres"
+            opts.on("--schema NAME", "application schema") { |value| options[:schema] = value }
+            opts.on("--owner ROLE", "NOLOGIN owner for append-only dictionary versions") { |value| options[:owner] = value }
+            opts.on("--migration-role ROLE", "role allowed to insert dictionary versions") { |value| options[:migration_role] = value }
+            opts.on("--extension-schema NAME", "extension schema (default: multi_compress)") { |value| options[:extension_schema] = value }
+          else
+            opts.on("--database NAME", "application database") { |value| options[:schema] = value }
+          end
+          opts.on("--table NAME", "versions table (default: mcdb_dictionary_versions)") { |value| options[:table] = value }
+          opts.on("--heads-table NAME", "heads table (default: mcdb_dictionary_heads)") { |value| options[:heads_table] = value }
+          opts.on("--payload-table NAME", "MCDB2 source table in this schema/database") { |value| options[:payload_table] = value }
+          opts.on("--payload-column NAME", "MCDB2 compressed envelope column on --payload-table") { |value| options[:payload_column] = value }
+          opts.on("--payload-dictionary-id-column NAME", "MCDB2 dictionary FK column on --payload-table") { |value| options[:payload_dictionary_id_column] = value }
+          opts.on("-o", "--output PATH", "write SQL to PATH instead of stdout") { |value| options[:output] = value }
+          opts.on("-h", "--help", "show this help") { puts opts; return 0 }
+        end
+        parser.parse!(argv)
+        raise OptionParser::InvalidOption, argv.join(" ") unless argv.empty?
+        sql = DictionaryRegistry.new(target, options).to_sql
+        if options[:output]
+          ViewCommand.new.send(:write_text_atomically, options[:output], sql)
+          puts File.expand_path(options[:output])
+        else
+          $stdout.write(sql)
+        end
+        0
+      end
+    end
+
+    class DictionaryRegistry
+      IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/.freeze
+
+      def initialize(target, options)
+        @target = target
+        @schema = identifier!(options.fetch(:schema), target == "postgres" ? "--schema" : "--database")
+        @table = identifier!(options.fetch(:table), "--table")
+        @heads_table = identifier!(options.fetch(:heads_table), "--heads-table")
+        @owner = options[:owner] && identifier!(options[:owner], "--owner")
+        @migration_role = options[:migration_role] && identifier!(options[:migration_role], "--migration-role")
+        @extension_schema = identifier!(options.fetch(:extension_schema, "multi_compress"), "--extension-schema")
+        @payload_table = options[:payload_table] && identifier!(options[:payload_table], "--payload-table")
+        @payload_column = options[:payload_column] && identifier!(options[:payload_column], "--payload-column")
+        @payload_dictionary_id_column = options[:payload_dictionary_id_column] && identifier!(options[:payload_dictionary_id_column], "--payload-dictionary-id-column")
+        payload_options = [@payload_table, @payload_column, @payload_dictionary_id_column]
+        unless payload_options.all? || payload_options.none?
+          raise Error, "--payload-table, --payload-column and --payload-dictionary-id-column must be used together"
+        end
+        if @target == "postgres" && (!@owner || !@migration_role)
+          raise Error, "--owner and --migration-role are required for postgres registry DDL"
+        end
+      rescue KeyError
+        raise Error, @target == "postgres" ? "--schema, --owner and --migration-role are required" : "--database is required"
+      end
+
+      def to_sql
+        @target == "postgres" ? postgres_sql : mysql_sql
+      end
+
+      private
+
+      def postgres_sql
+        q = method(:pg_quote)
+        schema = q.call(@schema)
+        table = q.call(@table)
+        heads = q.call(@heads_table)
+        ext = q.call(@extension_schema)
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. Apply as a privileged migration/DBA role.
+          -- The owner role must be a pre-existing NOLOGIN role. Dictionary bytes are append-only application data.
+          CREATE TABLE #{schema}.#{table} (
+            id bigint PRIMARY KEY CHECK (id > 0),
+            family text NOT NULL,
+            zstd_dict_id bigint NOT NULL CHECK (zstd_dict_id > 0),
+            sha256 bytea NOT NULL UNIQUE CHECK (octet_length(sha256) = 32),
+            bytes bytea NOT NULL CHECK (octet_length(bytes) BETWEEN 1 AND #{MAX_DICTIONARY_BYTES}),
+            created_at timestamptz NOT NULL DEFAULT now()
+          );
+          ALTER TABLE #{schema}.#{table} OWNER TO #{q.call(@owner)};
+          -- PostgreSQL RI triggers execute the FK lookup under the relation owner.
+          -- A NOLOGIN owner therefore still needs schema USAGE for the payload FK.
+          GRANT USAGE ON SCHEMA #{schema} TO #{q.call(@owner)};
+
+          CREATE TABLE #{schema}.#{heads} (
+            family text PRIMARY KEY,
+            dictionary_id bigint NOT NULL REFERENCES #{schema}.#{table}(id) ON DELETE RESTRICT
+          );
+          ALTER TABLE #{schema}.#{heads} OWNER TO #{q.call(@owner)};
+
+          CREATE FUNCTION #{schema}.mcdb_dictionary_version_validate() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF NEW.id <= 0 OR NEW.zstd_dict_id <= 0 OR octet_length(NEW.sha256) <> 32 THEN
+              RAISE EXCEPTION 'invalid MCDB2 dictionary metadata';
+            END IF;
+            IF NEW.sha256 IS DISTINCT FROM #{ext}.multi_compress_db_dictionary_sha256(NEW.bytes) THEN
+              RAISE EXCEPTION 'MCDB2 dictionary sha256 does not match bytes';
+            END IF;
+            IF NEW.zstd_dict_id IS DISTINCT FROM #{ext}.multi_compress_db_dictionary_zstd_id(NEW.bytes) THEN
+              RAISE EXCEPTION 'MCDB2 zstd dictionary id does not match bytes';
+            END IF;
+            RETURN NEW;
+          END $$;
+          ALTER FUNCTION #{schema}.mcdb_dictionary_version_validate() OWNER TO #{q.call(@owner)};
+
+          CREATE FUNCTION #{schema}.mcdb_dictionary_version_frozen() RETURNS trigger
+          LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'MCDB dictionary versions are append-only'; END $$;
+          ALTER FUNCTION #{schema}.mcdb_dictionary_version_frozen() OWNER TO #{q.call(@owner)};
+
+          CREATE TRIGGER mcdb_dictionary_version_validate
+            BEFORE INSERT ON #{schema}.#{table}
+            FOR EACH ROW EXECUTE FUNCTION #{schema}.mcdb_dictionary_version_validate();
+          CREATE TRIGGER mcdb_dictionary_version_frozen
+            BEFORE UPDATE OR DELETE ON #{schema}.#{table}
+            FOR EACH ROW EXECUTE FUNCTION #{schema}.mcdb_dictionary_version_frozen();
+
+          REVOKE ALL ON #{schema}.#{table}, #{schema}.#{heads} FROM PUBLIC;
+          GRANT SELECT, INSERT ON #{schema}.#{table} TO #{q.call(@migration_role)};
+          GRANT SELECT, INSERT, UPDATE, DELETE ON #{schema}.#{heads} TO #{q.call(@migration_role)};
+          #{postgres_payload_consistency_sql}
+        SQL
+      end
+
+      def mysql_sql
+        schema = mysql_quote(@schema)
+        table = mysql_quote(@table)
+        heads = mysql_quote(@heads_table)
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. MySQL 5.7 registry DDL.
+          -- Apply this in the application database. Dictionary versions are append-only.
+          USE #{schema};
+          CREATE TABLE #{table} (
+            id BIGINT UNSIGNED NOT NULL,
+            family VARCHAR(191) NOT NULL,
+            zstd_dict_id BIGINT UNSIGNED NOT NULL,
+            sha256 BINARY(32) NOT NULL,
+            bytes LONGBLOB NOT NULL,
+            created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+            PRIMARY KEY (id),
+            UNIQUE KEY mcdb_dictionary_versions_sha256 (sha256)
+          ) ENGINE=InnoDB;
+          CREATE TABLE #{heads} (
+            family VARCHAR(191) NOT NULL,
+            dictionary_id BIGINT UNSIGNED NOT NULL,
+            PRIMARY KEY (family),
+            CONSTRAINT mcdb_dictionary_heads_dictionary_fk
+              FOREIGN KEY (dictionary_id) REFERENCES #{table}(id) ON DELETE RESTRICT
+          ) ENGINE=InnoDB;
+
+          DELIMITER //
+          CREATE TRIGGER #{mysql_quote("#{@table}_validate")}
+          BEFORE INSERT ON #{table} FOR EACH ROW
+          BEGIN
+            IF NEW.id = 0 OR NEW.id > 9223372036854775807 OR NEW.zstd_dict_id = 0 OR OCTET_LENGTH(NEW.sha256) <> 32 OR
+               OCTET_LENGTH(NEW.bytes) = 0 OR OCTET_LENGTH(NEW.bytes) > #{MAX_DICTIONARY_BYTES} OR
+               NOT (NEW.sha256 <=> multi_compress_db_dictionary_sha256(NEW.bytes)) OR
+               NOT (NEW.zstd_dict_id <=> multi_compress_db_dictionary_zstd_id(NEW.bytes)) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'invalid MCDB2 dictionary metadata';
+            END IF;
+          END//
+          CREATE TRIGGER #{mysql_quote("#{@table}_frozen_update")}
+          BEFORE UPDATE ON #{table} FOR EACH ROW
+          BEGIN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB dictionary versions are append-only'; END//
+          CREATE TRIGGER #{mysql_quote("#{@table}_frozen_delete")}
+          BEFORE DELETE ON #{table} FOR EACH ROW
+          BEGIN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB dictionary versions are append-only'; END//
+          #{mysql_payload_consistency_sql}
+          DELIMITER ;
+        SQL
+      end
+
+      def payload_consistency?
+        !@payload_table.nil?
+      end
+
+      def payload_consistency_token
+        Digest::SHA256.hexdigest([@schema, @payload_table, @payload_column, @payload_dictionary_id_column].join("\0"))[0, 12]
+      end
+
+      def postgres_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        q = method(:pg_quote)
+        token = payload_consistency_token
+        function_name = "mcdb_payload_ref_#{token}"
+        trigger_name = "mcdb_payload_ref_#{token}_check"
+        <<~SQL
+
+          ALTER TABLE #{q.call(@schema)}.#{q.call(@payload_table)}
+            ADD CONSTRAINT #{q.call("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{q.call(@payload_dictionary_id_column)})
+            REFERENCES #{q.call(@schema)}.#{q.call(@table)}(id) ON DELETE RESTRICT;
+
+          CREATE FUNCTION #{q.call(@schema)}.#{q.call(function_name)}() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF NEW.#{q.call(@payload_column)} IS NULL OR NEW.#{q.call(@payload_dictionary_id_column)} IS NULL OR
+               #{q.call(@extension_schema)}.multi_compress_db_dictionary_ref(NEW.#{q.call(@payload_column)}) IS DISTINCT FROM NEW.#{q.call(@payload_dictionary_id_column)} THEN
+              RAISE EXCEPTION 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+            RETURN NEW;
+          END $$;
+          ALTER FUNCTION #{q.call(@schema)}.#{q.call(function_name)}() OWNER TO #{q.call(@owner)};
+          CREATE TRIGGER #{q.call(trigger_name)}
+            BEFORE INSERT OR UPDATE OF #{q.call(@payload_column)}, #{q.call(@payload_dictionary_id_column)}
+            ON #{q.call(@schema)}.#{q.call(@payload_table)}
+            FOR EACH ROW EXECUTE FUNCTION #{q.call(@schema)}.#{q.call(function_name)}();
+        SQL
+      end
+
+      def mysql_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        token = payload_consistency_token
+        insert_trigger = mysql_quote("mcdb_payload_ref_#{token}_insert")
+        update_trigger = mysql_quote("mcdb_payload_ref_#{token}_update")
+        payload_table = mysql_quote(@payload_table)
+        payload_column = mysql_quote(@payload_column)
+        payload_ref_column = mysql_quote(@payload_dictionary_id_column)
+        <<~SQL
+
+          ALTER TABLE #{payload_table}
+            ADD CONSTRAINT #{mysql_quote("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{payload_ref_column}) REFERENCES #{mysql_quote(@table)}(id) ON DELETE RESTRICT;
+          CREATE TRIGGER #{insert_trigger}
+          BEFORE INSERT ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+          CREATE TRIGGER #{update_trigger}
+          BEFORE UPDATE ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+        SQL
+      end
+
+      def identifier!(value, option)
+        string = value.to_s
+        raise Error, "#{option} must use letters, digits and underscores and cannot start with a digit" unless IDENTIFIER.match?(string)
+        string
+      end
+
+      def pg_quote(identifier)
+        %Q("#{identifier}")
+      end
+
+      def mysql_quote(identifier)
+        "`#{identifier}`"
+      end
+    end
+
     class ReadableView
       IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/.freeze
 
@@ -156,6 +445,14 @@ module MultiCompress
         @columns = Array(options.fetch(:columns)).reject(&:empty?).map { |value| identifier!(value, "--columns") }
         @decoded_name = identifier!(options.fetch(:as), "--as")
         @extension_schema = identifier!(options.fetch(:extension_schema, "multi_compress"), "--extension-schema")
+        @dictionary_table = options[:dictionary_table] && qualified_identifier!(options[:dictionary_table], "--dictionary-table")
+        @dictionary_id_column = options[:dictionary_id_column] && identifier!(options[:dictionary_id_column], "--dictionary-id-column")
+        @registry_id_column = identifier!(options.fetch(:registry_id_column, "id"), "--registry-id-column")
+        @dictionary_sha256_column = identifier!(options.fetch(:dictionary_sha256_column, "sha256"), "--dictionary-sha256-column")
+        @dictionary_bytes_column = identifier!(options.fetch(:dictionary_bytes_column, "bytes"), "--dictionary-bytes-column")
+        if !!@dictionary_table != !!@dictionary_id_column
+          raise Error, "--dictionary-table and --dictionary-id-column must be used together for MCDB2"
+        end
         raise Error, "--columns must contain at least one plain column" if @columns.empty?
         raise Error, "--columns must not contain duplicate names" unless @columns.uniq.length == @columns.length
         raise Error, "--as must not duplicate a selected column" if @columns.include?(@decoded_name)
@@ -175,32 +472,71 @@ module MultiCompress
 
       def postgres_sql
         selections = @columns.map { |column| "  source.#{pg_quote(column)}" }
-        selections << "  #{pg_quote(@extension_schema)}.multi_compress_db_decompress(source.#{pg_quote(@column)}) AS #{pg_quote(@decoded_name)}"
+        selections << "  #{postgres_decoder_sql} AS #{pg_quote(@decoded_name)}"
 
         <<~SQL
           -- Generated by multi_compress #{MultiCompress::VERSION}. Keep this view in your schema migrations.
-          -- Query it in DBeaver; do not expose the compressed column for ad-hoc reading.
+          -- Query it in DBeaver; do not expose the compressed column or dictionary registry directly.
           CREATE OR REPLACE VIEW #{pg_qualified(@view)} AS
           SELECT
           #{selections.join(",\n")}
-          FROM #{pg_qualified(@table)} AS source;
+          FROM #{pg_qualified(@table)} AS source#{postgres_dictionary_join};
         SQL
       end
 
       def mysql_sql
         selections = @columns.map { |column| "  source.#{mysql_quote(column)}" }
-        selections << "  CONVERT(multi_compress_db_decompress(source.#{mysql_quote(@column)}) USING utf8mb4) AS #{mysql_quote(@decoded_name)}"
+        selections << "  CONVERT(#{mysql_decoder_sql} USING utf8mb4) AS #{mysql_quote(@decoded_name)}"
+        algorithm = dictionary? ? " ALGORITHM=MERGE SQL SECURITY DEFINER" : ""
 
         <<~SQL
           -- Generated by multi_compress #{MultiCompress::VERSION}. Keep this view in your schema migrations.
-          -- Query it in DBeaver; do not expose the compressed column for ad-hoc reading.
-          CREATE OR REPLACE VIEW #{mysql_qualified(@view)} AS
+          -- Query it in DBeaver; do not expose the compressed column or dictionary registry directly.
+          -- MCDB2 requires this view to remain MERGE-able: no DISTINCT/GROUP BY/UNION/aggregate/subquery/LIMIT.
+          CREATE OR REPLACE#{algorithm} VIEW #{mysql_qualified(@view)} AS
           SELECT
           #{selections.join(",\n")}
-          FROM #{mysql_qualified(@table)} AS source;
+          FROM #{mysql_qualified(@table)} AS source#{mysql_dictionary_join};
         SQL
       end
 
+      def dictionary?
+        !@dictionary_table.nil?
+      end
+
+      def postgres_decoder_sql
+        return "#{pg_quote(@extension_schema)}.multi_compress_db_decompress(source.#{pg_quote(@column)})" unless dictionary?
+
+        "#{pg_quote(@extension_schema)}.multi_compress_db_decompress_dict(" \
+          "source.#{pg_quote(@column)}, source.#{pg_quote(@dictionary_id_column)}, " \
+          "dictionary.#{pg_quote(@dictionary_sha256_column)}, dictionary.#{pg_quote(@dictionary_bytes_column)})"
+      end
+
+      def mysql_decoder_sql
+        return "multi_compress_db_decompress(source.#{mysql_quote(@column)})" unless dictionary?
+
+        "multi_compress_db_decompress_dict(" \
+          "source.#{mysql_quote(@column)}, source.#{mysql_quote(@dictionary_id_column)}, " \
+          "dictionary.#{mysql_quote(@dictionary_sha256_column)}, dictionary.#{mysql_quote(@dictionary_bytes_column)})"
+      end
+
+      def postgres_dictionary_join
+        return "" unless dictionary?
+
+        "\nJOIN #{pg_qualified(@dictionary_table)} AS dictionary ON dictionary.#{pg_quote(@registry_id_column)} = source.#{pg_quote(@dictionary_id_column)}"
+      end
+
+      def mysql_dictionary_join
+        return "" unless dictionary?
+
+        # MySQL 5.7 may otherwise start from the tiny registry and then examine a
+        # broad source set. The generated view always places source on the left, so
+        # STRAIGHT_JOIN preserves source-first predicate pushdown before dictionary
+        # lookup. An outer ORDER BY can still legitimately report filesort; that is
+        # distinct from a materialized view (<derived...> in EXPLAIN).
+        "\nSTRAIGHT_JOIN #{mysql_qualified(@dictionary_table)} AS dictionary ON dictionary.#{mysql_quote(@registry_id_column)} = source.#{mysql_quote(@dictionary_id_column)}"
+      end
+
       def qualified_identifier!(value, option)
         pieces = value.to_s.split(".", -1)
         unless pieces.length.between?(1, 2) && pieces.all? { |piece| IDENTIFIER.match?(piece) }
@@ -209,6 +545,78 @@ module MultiCompress
         pieces
       end
 
+      def payload_consistency?
+        !@payload_table.nil?
+      end
+
+      def payload_consistency_token
+        Digest::SHA256.hexdigest([@schema, @payload_table, @payload_column, @payload_dictionary_id_column].join("\0"))[0, 12]
+      end
+
+      def postgres_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        q = method(:pg_quote)
+        token = payload_consistency_token
+        function_name = "mcdb_payload_ref_#{token}"
+        trigger_name = "mcdb_payload_ref_#{token}_check"
+        <<~SQL
+
+          ALTER TABLE #{q.call(@schema)}.#{q.call(@payload_table)}
+            ADD CONSTRAINT #{q.call("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{q.call(@payload_dictionary_id_column)})
+            REFERENCES #{q.call(@schema)}.#{q.call(@table)}(id) ON DELETE RESTRICT;
+
+          CREATE FUNCTION #{q.call(@schema)}.#{q.call(function_name)}() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF NEW.#{q.call(@payload_column)} IS NULL OR NEW.#{q.call(@payload_dictionary_id_column)} IS NULL OR
+               #{q.call(@extension_schema)}.multi_compress_db_dictionary_ref(NEW.#{q.call(@payload_column)}) IS DISTINCT FROM NEW.#{q.call(@payload_dictionary_id_column)} THEN
+              RAISE EXCEPTION 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+            RETURN NEW;
+          END $$;
+          ALTER FUNCTION #{q.call(@schema)}.#{q.call(function_name)}() OWNER TO #{q.call(@owner)};
+          CREATE TRIGGER #{q.call(trigger_name)}
+            BEFORE INSERT OR UPDATE OF #{q.call(@payload_column)}, #{q.call(@payload_dictionary_id_column)}
+            ON #{q.call(@schema)}.#{q.call(@payload_table)}
+            FOR EACH ROW EXECUTE FUNCTION #{q.call(@schema)}.#{q.call(function_name)}();
+        SQL
+      end
+
+      def mysql_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        token = payload_consistency_token
+        insert_trigger = mysql_quote("mcdb_payload_ref_#{token}_insert")
+        update_trigger = mysql_quote("mcdb_payload_ref_#{token}_update")
+        payload_table = mysql_quote(@payload_table)
+        payload_column = mysql_quote(@payload_column)
+        payload_ref_column = mysql_quote(@payload_dictionary_id_column)
+        <<~SQL
+
+          ALTER TABLE #{payload_table}
+            ADD CONSTRAINT #{mysql_quote("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{payload_ref_column}) REFERENCES #{mysql_quote(@table)}(id) ON DELETE RESTRICT;
+          CREATE TRIGGER #{insert_trigger}
+          BEFORE INSERT ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+          CREATE TRIGGER #{update_trigger}
+          BEFORE UPDATE ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+        SQL
+      end
+
       def identifier!(value, option)
         string = value.to_s
         raise Error, "#{option} must use letters, digits and underscores and cannot start with a digit" unless IDENTIFIER.match?(string)
@@ -299,6 +707,8 @@ module MultiCompress
             postgres_extension
             ext/multi_compress/vendor/zstd
             docs/database-envelope-v1.md
+            docs/database-envelope-v2.md
+            docs/rfcs/0002-mcdb2-dictionaries.md
             LICENSE.txt
             THIRD_PARTY_NOTICES.md
           ],
@@ -310,6 +720,8 @@ module MultiCompress
             mysql_udf
             ext/multi_compress/vendor/zstd
             docs/database-envelope-v1.md
+            docs/database-envelope-v2.md
+            docs/rfcs/0002-mcdb2-dictionaries.md
             LICENSE.txt
             THIRD_PARTY_NOTICES.md
           ],

data/lib/multi_compress/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MultiCompress
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/mysql_udf/README.md

@@ -93,3 +93,48 @@ invokes it.
 `make verify` catches accidental corruption after extraction; it does not
 authenticate an archive. Use an out-of-band SHA-256, minisign/GPG signature, or
 signed release provenance for a real trust chain.
+
+## MCDB2 dictionary-backed columns
+
+MCDB2 is opt-in for one homogeneous column with many similar small JSON/text
+values. Upgrade the UDF library first; the controlled `make upgrade` flow keeps
+all MCDB1 and MCDB2 functions registered.
+
+Dictionary bytes are application data. Generate the append-only registry in the
+application database, register immutable versions, and retain all referenced
+versions in backups and replicas:
+
+```bash
+bundle exec multi_compress db registry mysql --database app \
+  --output db/mcdb_dictionary_registry.sql
+```
+
+For a new MCDB2 column add `payload_dictionary_id BIGINT UNSIGNED NOT NULL` with
+a foreign key to `mcdb_dictionary_versions(id)`, then generate the view:
+
+```bash
+bundle exec multi_compress db view mysql \
+  --table app.events \
+  --column payload_compressed \
+  --dictionary-table app.mcdb_dictionary_versions \
+  --dictionary-id-column payload_dictionary_id \
+  --view admin.events_readable \
+  --columns id,created_at,status \
+  --as payload \
+  --output db/views/events_readable.sql
+```
+
+The generated MCDB2 view requests `ALGORITHM=MERGE SQL SECURITY DEFINER` and uses
+a source-first `STRAIGHT_JOIN` to the registry. That keeps an indexed outer source
+predicate from starting at the registry. It passes payload, registry id, SHA-256
+and dictionary bytes to the UDF. Keep the definition free of `DISTINCT`, aggregates,
+`GROUP BY`, `UNION`, subqueries and inner `LIMIT`; validate real queries with
+`EXPLAIN`: source must be a selective `range`/`ref` access and no `<derived>` row
+may appear. MySQL 5.7 can still report `Using temporary; Using filesort` for an
+outer `ORDER BY` over the joined projection; that is sorting, not view
+materialization. A stable definer account must remain part of backup/restore
+procedures.
+
+Details are in [`docs/database-envelope-v2.md`](../docs/database-envelope-v2.md).
+
+For MCDB2 registry DDL, pass `--payload-table`, `--payload-column`, and `--payload-dictionary-id-column` to `multi_compress db registry`; this creates the payload FK and enforces header dictionary-reference consistency.

data/mysql_udf/sql/examples.sql

@@ -9,3 +9,8 @@ FROM events
 WHERE multi_compress_db_is_valid(payload_compressed) = 0;
 
 SELECT multi_compress_db_version();
+
+-- MCDB2: use the generated ALGORITHM=MERGE readable view. Avoid filtering on
+-- decoded payload text; filter/order by indexed source columns before LIMIT.
+--
+-- SELECT id, payload FROM admin.events_readable WHERE id >= 100 ORDER BY id LIMIT 100;

data/mysql_udf/sql/install.sql

@@ -1,4 +1,4 @@
--- Install the MultiCompress MCDB1 reader UDFs (MySQL 5.7).
+-- Install the MultiCompress MCDB1/MCDB2 reader UDFs (MySQL 5.7).
 
 CREATE FUNCTION multi_compress_db_version
   RETURNS STRING
@@ -12,4 +12,28 @@ CREATE FUNCTION multi_compress_db_decompress
   RETURNS STRING
   SONAME 'multi_compress_mysql.so';
 
+CREATE FUNCTION multi_compress_db_original_size
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_ref
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_zstd_id
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_dictionary_sha256
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_is_valid_dict
+  RETURNS INTEGER
+  SONAME 'multi_compress_mysql.so';
+
+CREATE FUNCTION multi_compress_db_decompress_dict
+  RETURNS STRING
+  SONAME 'multi_compress_mysql.so';
+
 SELECT multi_compress_db_version();

data/mysql_udf/sql/uninstall.sql

@@ -1,4 +1,10 @@
--- Remove the MultiCompress MCDB1 reader UDFs.
+-- Run only after verifying no generated readable view depends on these UDFs.
+DROP FUNCTION IF EXISTS multi_compress_db_decompress_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_is_valid_dict;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_sha256;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_zstd_id;
+DROP FUNCTION IF EXISTS multi_compress_db_original_size;
+DROP FUNCTION IF EXISTS multi_compress_db_dictionary_ref;
 DROP FUNCTION IF EXISTS multi_compress_db_decompress;
 DROP FUNCTION IF EXISTS multi_compress_db_is_valid;
 DROP FUNCTION IF EXISTS multi_compress_db_version;

data/mysql_udf/sql/views.sql

@@ -1,8 +1,20 @@
--- Example read-side view for DBeaver / DataGrip / mysql console.
-
+-- MCDB1 view example. Generate production SQL with:
+--   multi_compress db view mysql ...
 CREATE OR REPLACE VIEW events_readable AS
 SELECT
   id,
-  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload,
-  created_at
+  CONVERT(multi_compress_db_decompress(payload_compressed) USING utf8mb4) AS payload
 FROM events;
+
+-- MCDB2 dictionary-backed shape. The generated view uses ALGORITHM=MERGE and
+-- a source-first STRAIGHT_JOIN to the append-only registry, so an indexed outer
+-- WHERE predicate is pushed into the source scan in MySQL 5.7. An outer ORDER BY
+-- may still use filesort; that is not view materialization.
+--
+-- CREATE ALGORITHM=MERGE SQL SECURITY DEFINER VIEW events_readable AS
+-- SELECT e.id,
+--   CONVERT(multi_compress_db_decompress_dict(
+--     e.payload_compressed, e.payload_dictionary_id, d.sha256, d.bytes
+--   ) USING utf8mb4) AS payload
+-- FROM events e
+-- STRAIGHT_JOIN mcdb_dictionary_versions d ON d.id = e.payload_dictionary_id;