multi_compress 0.6.0 → 0.6.1

data/lib/multi_compress/db_deployment/dictionary_registry.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    class DictionaryRegistry
+      include SqlIdentifiers
+
+      attr_reader :target, :schema, :table, :heads_table, :owner, :migration_role,
+                  :extension_schema, :payload_table, :payload_column, :payload_dictionary_id_column
+      private :target, :schema, :table, :heads_table, :owner, :migration_role,
+              :extension_schema, :payload_table, :payload_column, :payload_dictionary_id_column
+
+      def initialize(target, options)
+        @target = target
+        load_options(options)
+        validate_payload_options!
+      rescue KeyError
+        raise Error,
+              target == "postgres" ? "--schema, --owner and --migration-role are required" : "--database is required"
+      end
+
+      private
+
+      def load_options(options)
+        @schema = identifier!(options.fetch(:schema), target == "postgres" ? "--schema" : "--database")
+        @table = identifier!(options.fetch(:table), "--table")
+        @heads_table = identifier!(options.fetch(:heads_table), "--heads-table")
+        @owner = optional_identifier(options, :owner, "--owner")
+        @migration_role = optional_identifier(options, :migration_role, "--migration-role")
+        @extension_schema = identifier!(options.fetch(:extension_schema, "multi_compress"), "--extension-schema")
+        load_payload_options(options)
+      end
+
+      def load_payload_options(options)
+        @payload_table = optional_identifier(options, :payload_table, "--payload-table")
+        @payload_column = optional_identifier(options, :payload_column, "--payload-column")
+        @payload_dictionary_id_column = optional_identifier(
+          options, :payload_dictionary_id_column, "--payload-dictionary-id-column"
+        )
+      end
+
+      def validate_payload_options!
+        payload_options = [payload_table, payload_column, payload_dictionary_id_column]
+        unless payload_options.all? || payload_options.none?
+          raise Error, "--payload-table, --payload-column and --payload-dictionary-id-column must be used together"
+        end
+        return unless target == "postgres" && (!owner || !migration_role)
+
+        raise Error, "--owner and --migration-role are required for postgres registry DDL"
+      end
+
+      public
+
+      def to_sql
+        target == "postgres" ? postgres_sql : mysql_sql
+      end
+
+      private
+
+      def postgres_sql
+        q = method(:pg_quote)
+        schema = q.call(schema())
+        table = q.call(table())
+        heads = q.call(heads_table)
+        ext = q.call(extension_schema)
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. Apply as a privileged migration/DBA role.
+          -- The owner role must be a pre-existing NOLOGIN role. Dictionary bytes are append-only application data.
+          CREATE TABLE #{schema}.#{table} (
+            id bigint PRIMARY KEY CHECK (id > 0),
+            family text NOT NULL,
+            zstd_dict_id bigint NOT NULL CHECK (zstd_dict_id > 0),
+            sha256 bytea NOT NULL UNIQUE CHECK (octet_length(sha256) = 32),
+            bytes bytea NOT NULL CHECK (octet_length(bytes) BETWEEN 1 AND #{MAX_DICTIONARY_BYTES}),
+            created_at timestamptz NOT NULL DEFAULT now()
+          );
+          ALTER TABLE #{schema}.#{table} OWNER TO #{q.call(owner)};
+          -- PostgreSQL RI triggers execute the FK lookup under the relation owner.
+          -- A NOLOGIN owner therefore still needs schema USAGE for the payload FK.
+          GRANT USAGE ON SCHEMA #{schema} TO #{q.call(owner)};
+
+          CREATE TABLE #{schema}.#{heads} (
+            family text PRIMARY KEY,
+            dictionary_id bigint NOT NULL REFERENCES #{schema}.#{table}(id) ON DELETE RESTRICT
+          );
+          ALTER TABLE #{schema}.#{heads} OWNER TO #{q.call(owner)};
+
+          CREATE FUNCTION #{schema}.mcdb_dictionary_version_validate() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF NEW.id <= 0 OR NEW.zstd_dict_id <= 0 OR octet_length(NEW.sha256) <> 32 THEN
+              RAISE EXCEPTION 'invalid MCDB2 dictionary metadata';
+            END IF;
+            IF NEW.sha256 IS DISTINCT FROM #{ext}.multi_compress_db_dictionary_sha256(NEW.bytes) THEN
+              RAISE EXCEPTION 'MCDB2 dictionary sha256 does not match bytes';
+            END IF;
+            IF NEW.zstd_dict_id IS DISTINCT FROM #{ext}.multi_compress_db_dictionary_zstd_id(NEW.bytes) THEN
+              RAISE EXCEPTION 'MCDB2 zstd dictionary id does not match bytes';
+            END IF;
+            RETURN NEW;
+          END $$;
+          ALTER FUNCTION #{schema}.mcdb_dictionary_version_validate() OWNER TO #{q.call(owner)};
+
+          CREATE FUNCTION #{schema}.mcdb_dictionary_version_frozen() RETURNS trigger
+          LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'MCDB dictionary versions are append-only'; END $$;
+          ALTER FUNCTION #{schema}.mcdb_dictionary_version_frozen() OWNER TO #{q.call(owner)};
+
+          CREATE TRIGGER mcdb_dictionary_version_validate
+            BEFORE INSERT ON #{schema}.#{table}
+            FOR EACH ROW EXECUTE FUNCTION #{schema}.mcdb_dictionary_version_validate();
+          CREATE TRIGGER mcdb_dictionary_version_frozen
+            BEFORE UPDATE OR DELETE ON #{schema}.#{table}
+            FOR EACH ROW EXECUTE FUNCTION #{schema}.mcdb_dictionary_version_frozen();
+
+          REVOKE ALL ON #{schema}.#{table}, #{schema}.#{heads} FROM PUBLIC;
+          GRANT SELECT, INSERT ON #{schema}.#{table} TO #{q.call(migration_role)};
+          GRANT SELECT, INSERT, UPDATE, DELETE ON #{schema}.#{heads} TO #{q.call(migration_role)};
+          #{postgres_payload_consistency_sql}
+        SQL
+      end
+
+      def mysql_sql
+        schema = mysql_quote(schema())
+        table = mysql_quote(table())
+        heads = mysql_quote(heads_table)
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. MySQL 5.7 registry DDL.
+          -- Apply this in the application database. Dictionary versions are append-only.
+          USE #{schema};
+          CREATE TABLE #{table} (
+            id BIGINT UNSIGNED NOT NULL,
+            family VARCHAR(191) NOT NULL,
+            zstd_dict_id BIGINT UNSIGNED NOT NULL,
+            sha256 BINARY(32) NOT NULL,
+            bytes LONGBLOB NOT NULL,
+            created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+            PRIMARY KEY (id),
+            UNIQUE KEY mcdb_dictionary_versions_sha256 (sha256)
+          ) ENGINE=InnoDB;
+          CREATE TABLE #{heads} (
+            family VARCHAR(191) NOT NULL,
+            dictionary_id BIGINT UNSIGNED NOT NULL,
+            PRIMARY KEY (family),
+            CONSTRAINT mcdb_dictionary_heads_dictionary_fk
+              FOREIGN KEY (dictionary_id) REFERENCES #{table}(id) ON DELETE RESTRICT
+          ) ENGINE=InnoDB;
+
+          DELIMITER //
+          CREATE TRIGGER #{mysql_quote("#{table()}_validate")}
+          BEFORE INSERT ON #{table} FOR EACH ROW
+          BEGIN
+            IF NEW.id = 0 OR NEW.id > 9223372036854775807 OR NEW.zstd_dict_id = 0 OR OCTET_LENGTH(NEW.sha256) <> 32 OR
+               OCTET_LENGTH(NEW.bytes) = 0 OR OCTET_LENGTH(NEW.bytes) > #{MAX_DICTIONARY_BYTES} OR
+               NOT (NEW.sha256 <=> multi_compress_db_dictionary_sha256(NEW.bytes)) OR
+               NOT (NEW.zstd_dict_id <=> multi_compress_db_dictionary_zstd_id(NEW.bytes)) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'invalid MCDB2 dictionary metadata';
+            END IF;
+          END//
+          CREATE TRIGGER #{mysql_quote("#{table()}_frozen_update")}
+          BEFORE UPDATE ON #{table} FOR EACH ROW
+          BEGIN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB dictionary versions are append-only'; END//
+          CREATE TRIGGER #{mysql_quote("#{table()}_frozen_delete")}
+          BEFORE DELETE ON #{table} FOR EACH ROW
+          BEGIN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB dictionary versions are append-only'; END//
+          #{mysql_payload_consistency_sql}
+          DELIMITER ;
+        SQL
+      end
+
+      def payload_consistency?
+        !payload_table.nil?
+      end
+
+      def payload_consistency_token
+        values = [schema, payload_table, payload_column, payload_dictionary_id_column]
+        Digest::SHA256.hexdigest(values.join("\0"))[0, 12]
+      end
+
+      def postgres_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        q = method(:pg_quote)
+        token = payload_consistency_token
+        function_name = "mcdb_payload_ref_#{token}"
+        trigger_name = "mcdb_payload_ref_#{token}_check"
+        <<~SQL
+
+          ALTER TABLE #{q.call(schema)}.#{q.call(payload_table)}
+            ADD CONSTRAINT #{q.call("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{q.call(payload_dictionary_id_column)})
+            REFERENCES #{q.call(schema)}.#{q.call(table)}(id) ON DELETE RESTRICT;
+
+          CREATE FUNCTION #{q.call(schema)}.#{q.call(function_name)}() RETURNS trigger
+          LANGUAGE plpgsql AS $$
+          BEGIN
+            IF NEW.#{q.call(payload_column)} IS NULL OR NEW.#{q.call(payload_dictionary_id_column)} IS NULL OR
+               #{q.call(extension_schema)}.multi_compress_db_dictionary_ref(NEW.#{q.call(payload_column)}) IS DISTINCT FROM NEW.#{q.call(payload_dictionary_id_column)} THEN
+              RAISE EXCEPTION 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+            RETURN NEW;
+          END $$;
+          ALTER FUNCTION #{q.call(schema)}.#{q.call(function_name)}() OWNER TO #{q.call(owner)};
+          CREATE TRIGGER #{q.call(trigger_name)}
+            BEFORE INSERT OR UPDATE OF #{q.call(payload_column)}, #{q.call(payload_dictionary_id_column)}
+            ON #{q.call(schema)}.#{q.call(payload_table)}
+            FOR EACH ROW EXECUTE FUNCTION #{q.call(schema)}.#{q.call(function_name)}();
+        SQL
+      end
+
+      def mysql_payload_consistency_sql
+        return "" unless payload_consistency?
+
+        token = payload_consistency_token
+        insert_trigger = mysql_quote("mcdb_payload_ref_#{token}_insert")
+        update_trigger = mysql_quote("mcdb_payload_ref_#{token}_update")
+        payload_table = mysql_quote(payload_table())
+        payload_column = mysql_quote(payload_column())
+        payload_ref_column = mysql_quote(payload_dictionary_id_column)
+        <<~SQL
+
+          ALTER TABLE #{payload_table}
+            ADD CONSTRAINT #{mysql_quote("mcdb_payload_dictionary_#{token}_fk")}
+            FOREIGN KEY (#{payload_ref_column}) REFERENCES #{mysql_quote(table)}(id) ON DELETE RESTRICT;
+          CREATE TRIGGER #{insert_trigger}
+          BEFORE INSERT ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+          CREATE TRIGGER #{update_trigger}
+          BEFORE UPDATE ON #{payload_table} FOR EACH ROW
+          BEGIN
+            IF NEW.#{payload_column} IS NULL OR NEW.#{payload_ref_column} IS NULL OR
+               NOT (multi_compress_db_dictionary_ref(NEW.#{payload_column}) <=> NEW.#{payload_ref_column}) THEN
+              SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'MCDB2 payload dictionary reference must match its dictionary FK';
+            END IF;
+          END//
+        SQL
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/readable_view.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "multi_compress/db_deployment/readable_view_options"
+
+module MultiCompress
+  module DBDeployment
+    class ReadableView
+      include SqlIdentifiers
+      include ReadableViewOptions
+
+      attr_reader :target, :table, :view, :column, :columns, :decoded_name, :extension_schema,
+                  :dictionary_table, :dictionary_id_column, :registry_id_column,
+                  :dictionary_sha256_column, :dictionary_bytes_column
+      private :target, :table, :view, :column, :columns, :decoded_name, :extension_schema,
+              :dictionary_table, :dictionary_id_column, :registry_id_column,
+              :dictionary_sha256_column, :dictionary_bytes_column
+
+      def initialize(target, options)
+        @target = target
+        load_options(options)
+        validate_options!
+      rescue KeyError
+        raise Error, "--table, --column, --view and --columns are required"
+      end
+
+      def to_sql
+        case target
+        when "postgres" then postgres_sql
+        when "mysql" then mysql_sql
+        else raise Error, "unsupported database target: #{target.inspect}"
+        end
+      end
+
+      private
+
+      def postgres_sql
+        selections = columns.map { |column| "  source.#{pg_quote(column)}" }
+        selections << "  #{postgres_decoder_sql} AS #{pg_quote(decoded_name)}"
+
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. Keep this view in your schema migrations.
+          -- Query it in DBeaver; do not expose the compressed column or dictionary registry directly.
+          CREATE OR REPLACE VIEW #{pg_qualified(view)} AS
+          SELECT
+          #{selections.join(",\n")}
+          FROM #{pg_qualified(table)} AS source#{postgres_dictionary_join};
+        SQL
+      end
+
+      def mysql_sql
+        selections = columns.map { |column| "  source.#{mysql_quote(column)}" }
+        selections << "  CONVERT(#{mysql_decoder_sql} USING utf8mb4) AS #{mysql_quote(decoded_name)}"
+
+        <<~SQL
+          -- Generated by multi_compress #{MultiCompress::VERSION}. Keep this view in your schema migrations.
+          -- Query it in DBeaver; do not expose the compressed column or dictionary registry directly.
+          -- MCDB2 requires this view to remain MERGE-able: no DISTINCT/GROUP BY/UNION/aggregate/subquery/LIMIT.
+          CREATE OR REPLACE#{mysql_algorithm} VIEW #{mysql_qualified(view)} AS
+          SELECT
+          #{selections.join(",\n")}
+          FROM #{mysql_qualified(table)} AS source#{mysql_dictionary_join};
+        SQL
+      end
+
+      def dictionary?
+        !dictionary_table.nil?
+      end
+
+      def postgres_decoder_sql
+        unless dictionary?
+          return "#{pg_quote(extension_schema)}.multi_compress_db_decompress(source.#{pg_quote(column)})"
+        end
+
+        "#{pg_quote(extension_schema)}.multi_compress_db_decompress_dict(" \
+          "source.#{pg_quote(column)}, source.#{pg_quote(dictionary_id_column)}, " \
+          "dictionary.#{pg_quote(dictionary_sha256_column)}, dictionary.#{pg_quote(dictionary_bytes_column)})"
+      end
+
+      def mysql_decoder_sql
+        return "multi_compress_db_decompress(source.#{mysql_quote(column)})" unless dictionary?
+
+        "multi_compress_db_decompress_dict(" \
+          "source.#{mysql_quote(column)}, source.#{mysql_quote(dictionary_id_column)}, " \
+          "dictionary.#{mysql_quote(dictionary_sha256_column)}, dictionary.#{mysql_quote(dictionary_bytes_column)})"
+      end
+
+      def postgres_dictionary_join
+        return "" unless dictionary?
+
+        join = "\nJOIN #{pg_qualified(dictionary_table)} AS dictionary"
+        join << " ON dictionary.#{pg_quote(registry_id_column)} = source.#{pg_quote(dictionary_id_column)}"
+      end
+
+      def mysql_dictionary_join
+        return "" unless dictionary?
+
+        # MySQL 5.7 may otherwise start from the tiny registry and then examine a
+        # broad source set. The generated view always places source on the left, so
+        # STRAIGHT_JOIN preserves source-first predicate pushdown before dictionary
+        # lookup. An outer ORDER BY can still legitimately report filesort; that is
+        # distinct from a materialized view (<derived...> in EXPLAIN).
+        join = "\nSTRAIGHT_JOIN #{mysql_qualified(dictionary_table)} AS dictionary"
+        join << " ON dictionary.#{mysql_quote(registry_id_column)}"
+        join << " = source.#{mysql_quote(dictionary_id_column)}"
+      end
+
+      def mysql_algorithm
+        dictionary? ? " ALGORITHM=MERGE SQL SECURITY DEFINER" : ""
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/readable_view_options.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    module ReadableViewOptions
+      private
+
+      def load_options(options)
+        load_source_options(options)
+        load_dictionary_options(options)
+      end
+
+      def load_source_options(options)
+        @table = qualified_identifier!(options.fetch(:table), "--table")
+        @view = qualified_identifier!(options.fetch(:view), "--view")
+        @column = identifier!(options.fetch(:column), "--column")
+        @columns = Array(options.fetch(:columns)).reject(&:empty?).map { |value| identifier!(value, "--columns") }
+        @decoded_name = identifier!(options.fetch(:as), "--as")
+        @extension_schema = identifier!(options.fetch(:extension_schema, "multi_compress"), "--extension-schema")
+      end
+
+      def load_dictionary_options(options)
+        @dictionary_table = qualified_identifier!(options.fetch(:dictionary_table), "--dictionary-table") if
+          options[:dictionary_table]
+        @dictionary_id_column = optional_identifier(options, :dictionary_id_column, "--dictionary-id-column")
+        @registry_id_column = identifier!(options.fetch(:registry_id_column, "id"), "--registry-id-column")
+        @dictionary_sha256_column = identifier!(options.fetch(:dictionary_sha256_column, "sha256"),
+                                                "--dictionary-sha256-column")
+        @dictionary_bytes_column = identifier!(options.fetch(:dictionary_bytes_column, "bytes"),
+                                               "--dictionary-bytes-column")
+      end
+
+      def validate_options!
+        validate_dictionary_options!
+        validate_columns!
+      end
+
+      def validate_dictionary_options!
+        return if dictionary_table.nil? == dictionary_id_column.nil?
+
+        raise Error, "--dictionary-table and --dictionary-id-column must be used together for MCDB2"
+      end
+
+      def validate_columns!
+        raise Error, "--columns must contain at least one plain column" if columns.empty?
+        raise Error, "--columns must not contain duplicate names" unless columns.uniq.length == columns.length
+        raise Error, "--as must not duplicate a selected column" if columns.include?(decoded_name)
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/sql_identifiers.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    module SqlIdentifiers
+      IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/.freeze
+
+      private
+
+      def identifier!(value, option)
+        string = value.to_s
+        return string if IDENTIFIER.match?(string)
+
+        raise Error, "#{option} must use letters, digits and underscores and cannot start with a digit"
+      end
+
+      def optional_identifier(options, key, label)
+        identifier!(options.fetch(key), label) if options.key?(key) && options[key]
+      end
+
+      def qualified_identifier!(value, option)
+        pieces = value.to_s.split(".", -1)
+        return pieces if pieces.length.between?(1, 2) && pieces.all? { |piece| IDENTIFIER.match?(piece) }
+
+        raise Error, "#{option} must be NAME or SCHEMA.NAME using letters, digits and underscores"
+      end
+
+      def pg_quote(identifier)
+        %("#{identifier}")
+      end
+
+      def mysql_quote(identifier)
+        "`#{identifier}`"
+      end
+
+      def pg_qualified(pieces)
+        pieces.map { |piece| pg_quote(piece) }.join(".")
+      end
+
+      def mysql_qualified(pieces)
+        pieces.map { |piece| mysql_quote(piece) }.join(".")
+      end
+    end
+  end
+end