multi_compress 0.6.0 → 0.6.1

data/lib/multi_compress/db_deployment/bundle_archiver.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    class BundleArchiver
+      DEFAULT_EPOCH = 315_619_200
+      attr_reader :output, :root_name
+      private :output, :root_name
+
+      def initialize(output, root_name)
+        @output = output
+        @root_name = root_name
+      end
+
+      def write!(temporary_directory)
+        temporary_output = temporary_path
+        FileUtils.mkdir_p(File.dirname(output))
+        write_archive(temporary_output, temporary_directory)
+        File.rename(temporary_output, output)
+      ensure
+        File.delete(temporary_output) if defined?(temporary_output) && File.exist?(temporary_output)
+      end
+
+      private
+
+      def temporary_path
+        File.join(File.dirname(output), ".#{File.basename(output)}.#{Process.pid}.#{SecureRandom.hex(8)}.tmp")
+      end
+
+      def write_archive(path, temporary_directory)
+        File.open(path, "wb", 0o644) do |file|
+          Zlib::GzipWriter.wrap(file) do |gzip|
+            gzip.mtime = archive_epoch
+            DeterministicTarWriter.new(gzip, archive_epoch) do |tar|
+              add_tree(tar, File.join(temporary_directory, root_name), root_name)
+            end
+          end
+        end
+      end
+
+      def archive_epoch
+        value = ENV.fetch("SOURCE_DATE_EPOCH", nil)
+        value&.match?(/\A\d+\z/) ? value.to_i : DEFAULT_EPOCH
+      end
+
+      def add_tree(tar, directory, archive_path)
+        tar.mkdir(archive_path, mode(directory))
+        Dir.children(directory).sort.each { |entry| add_entry(tar, directory, archive_path, entry) }
+      end
+
+      def add_entry(tar, directory, archive_path, entry)
+        source = File.join(directory, entry)
+        destination = File.join(archive_path, entry)
+        return add_tree(tar, source, destination) if File.directory?(source)
+
+        add_file(tar, source, destination)
+      end
+
+      def add_file(tar, source, destination)
+        tar.add_file_simple(destination, mode(source), File.size(source)) do |destination_io|
+          File.open(source, "rb") { |source_io| IO.copy_stream(source_io, destination_io) }
+        end
+      end
+
+      def mode(path)
+        File.stat(path).mode & 0o777
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/bundle_metadata.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    class BundleMetadata
+      attr_reader :target
+      private :target
+
+      def initialize(target)
+        @target = target
+      end
+
+      def write!(bundle_root)
+        write_manifest(bundle_root, manifest_files(bundle_root))
+        write_checksums(bundle_root)
+      end
+
+      private
+
+      def manifest_files(root)
+        regular_files(root).reject { |path| %w[manifest.json SHA256SUMS].include?(File.basename(path)) }
+                           .sort.map { |path| file_record(root, path) }
+      end
+
+      def file_record(root, path)
+        { "path" => relative_to(root, path), "sha256" => Digest::SHA256.file(path).hexdigest }
+      end
+
+      def write_manifest(root, files)
+        manifest = {
+          "bundle_format_version" => BUNDLE_FORMAT_VERSION,
+          "gem" => { "name" => "multi_compress", "version" => MultiCompress::VERSION },
+          "database_target" => target,
+          "database_envelope" => FORMAT,
+          "files" => files
+        }
+        File.write(File.join(root, "manifest.json"), "#{JSON.pretty_generate(manifest)}\n")
+      end
+
+      def write_checksums(root)
+        checksums = regular_files(root).reject { |path| File.basename(path) == "SHA256SUMS" }.sort.map do |path|
+          "#{Digest::SHA256.file(path).hexdigest}  #{relative_to(root, path)}"
+        end
+        File.write(File.join(root, "SHA256SUMS"), "#{checksums.join("\n")}\n")
+      end
+
+      def regular_files(root)
+        Dir.glob(File.join(root, "**", "*"), File::FNM_DOTMATCH).select { |path| File.file?(path) }
+      end
+
+      def relative_to(root, path)
+        Pathname.new(path).relative_path_from(Pathname.new(root)).to_s
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/bundle_stager.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    class BundleStager
+      attr_reader :root, :target
+      private :root, :target
+
+      IGNORED_ENTRIES = [".DS_Store", ".git", "build", "test"].freeze
+
+      def initialize(root, target)
+        @root = root
+        @target = target
+      end
+
+      def stage!(bundle_root)
+        target.fetch(:source_paths).each { |relative| copy_path!(relative, bundle_root) }
+        stage_template(bundle_root)
+      end
+
+      private
+
+      def stage_template(bundle_root)
+        template = target.fetch(:template_path)
+        Dir.children(File.join(root, template)).sort.each do |entry|
+          next if ignored?(entry)
+
+          copy_path!(File.join(template, entry), bundle_root, destination: entry)
+        end
+      end
+
+      def copy_path!(relative, bundle_root, destination: relative)
+        source = File.join(root, relative)
+        raise Error, "package source is missing: #{relative}" unless File.exist?(source)
+
+        destination_path = File.join(bundle_root, destination)
+        return copy_directory(source, relative, destination_path, destination, bundle_root) if File.directory?(source)
+
+        copy_file(source, destination_path)
+      end
+
+      def copy_directory(source, relative, destination_path, destination, bundle_root)
+        FileUtils.mkdir_p(destination_path)
+        Dir.children(source).sort.each do |entry|
+          next if ignored?(entry)
+
+          copy_path!(File.join(relative, entry), bundle_root, destination: File.join(destination, entry))
+        end
+      end
+
+      def copy_file(source, destination)
+        FileUtils.mkdir_p(File.dirname(destination))
+        FileUtils.copy_file(source, destination)
+        File.chmod(File.stat(source).mode & 0o777, destination)
+      end
+
+      def ignored?(entry)
+        IGNORED_ENTRIES.include?(entry)
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/commands.rb

@@ -0,0 +1,309 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    module CommandHelpers
+      private
+
+      def validated_target!(argv, targets)
+        target = argv.shift
+        raise OptionParser::MissingArgument, "TARGET must be postgres or mysql" if target.nil?
+        raise OptionParser::InvalidArgument, "TARGET must be postgres or mysql" unless targets.include?(target)
+
+        target
+      end
+
+      def show_help(parser)
+        puts parser
+        0
+      end
+
+      def write_sql(sql, output)
+        if output
+          write_text_atomically(output, sql)
+          puts File.expand_path(output)
+        else
+          $stdout.write(sql)
+        end
+      end
+
+      def write_text_atomically(path, contents)
+        directory = File.dirname(File.expand_path(path))
+        FileUtils.mkdir_p(directory)
+        temporary = File.join(directory, ".multi-compress-view-#{Process.pid}-#{SecureRandom.hex(8)}")
+        File.open(temporary, "wb", 0o644) { |file| file.write(contents) }
+        File.rename(temporary, path)
+      ensure
+        File.delete(temporary) if defined?(temporary) && File.exist?(temporary)
+      end
+
+      def add_value_options(parser, options, definitions)
+        definitions.each do |key, flag, description|
+          parser.on("--#{flag} NAME", description) { |value| options[key] = value }
+        end
+      end
+    end
+
+    class CLI
+      COMMANDS = {
+        "package" => "PackageCommand",
+        "view" => "ViewCommand",
+        "registry" => "RegistryCommand"
+      }.freeze
+      HELP_COMMANDS = ["help", "--help", "-h", nil].freeze
+
+      def self.run(argv)
+        new.run(argv)
+      end
+
+      def run(argv)
+        dispatch(argv)
+      rescue OptionParser::ParseError, Error => e
+        report_error(e, 2)
+      rescue SystemCallError, IOError => e
+        report_error(e, 1)
+      end
+
+      private
+
+      def dispatch(argv)
+        command = argv.shift
+        return show_help if HELP_COMMANDS.include?(command)
+
+        command_class = DBDeployment.const_get(
+          COMMANDS.fetch(command) { raise OptionParser::InvalidOption, "unknown db command: #{command.inspect}" },
+          false
+        )
+        command_class.new.run(argv)
+      end
+
+      def report_error(error, status)
+        warn "multi_compress db: #{error.message}"
+        status
+      end
+
+      def show_help
+        puts help
+        0
+      end
+
+      def help
+        <<~TEXT
+          Usage:
+            multi_compress db package postgres [--output PATH] [--force]
+            multi_compress db package mysql    [--output PATH] [--force]
+
+            multi_compress db view postgres --table SCHEMA.TABLE --column COLUMN \\
+              --view SCHEMA.VIEW --columns ID,CREATED_AT [--as PAYLOAD] \\
+              [--extension-schema multi_compress] [--output PATH]
+
+            multi_compress db view mysql --table SCHEMA.TABLE --column COLUMN \\
+              --view SCHEMA.VIEW --columns ID,CREATED_AT [--as PAYLOAD] [--output PATH]
+
+          `package` creates a self-contained source bundle for the DBA. The DBA
+          extracts it on the database host and runs `make doctor`, `sudo make
+          install`, then `make enable DB=...`.
+
+          `view` emits a safe readable-view definition so DBeaver users query
+          decoded text instead of the compressed bytea/BLOB column. Supply
+          --dictionary-table and --dictionary-id-column for MCDB2.
+
+          `registry` emits append-only MCDB2 dictionary-registry DDL. Dictionary
+          bytes are application data and are never placed in a DBA reader bundle.
+          Supplying --payload-table, --payload-column and
+          --payload-dictionary-id-column also emits the MCDB2 header/FK
+          consistency trigger for that source table.
+        TEXT
+      end
+    end
+
+    class PackageCommand
+      include CommandHelpers
+
+      TARGETS = %w[postgres mysql].freeze
+
+      def run(argv)
+        target = validated_target!(argv, TARGETS)
+        options = { output: nil, force: false }
+        parser = option_parser(target, options)
+        parser.parse!(argv)
+        return show_help(options[:help]) if options[:help]
+
+        raise OptionParser::InvalidOption, argv.join(" ") unless argv.empty?
+
+        write_bundle(target, options)
+        0
+      end
+
+      private
+
+      def option_parser(target, options)
+        OptionParser.new do |parser|
+          parser.banner = "Usage: multi_compress db package #{target} [OPTIONS]"
+          parser.on("-o", "--output PATH", "write the deployment archive to PATH") do |value|
+            options[:output] = value
+          end
+          parser.on("-f", "--force", "overwrite an existing archive") { options[:force] = true }
+          parser.on("-h", "--help", "show this help") { options[:help] = parser }
+        end
+      end
+
+      def write_bundle(target, options)
+        output = options[:output] || "multi_compress-#{target}-#{MultiCompress::VERSION}.tar.gz"
+        DeploymentBundle.new(target, output, force: options[:force]).write!
+        puts File.expand_path(output)
+      end
+    end
+
+    class SqlCommand
+      include CommandHelpers
+
+      def run(argv)
+        target = validated_target!(argv, self.class::TARGETS)
+        options = default_options
+        parser = option_parser(target, options)
+        parser.parse!(argv)
+        return show_help(options[:help]) if options[:help]
+
+        raise OptionParser::InvalidOption, argv.join(" ") unless argv.empty?
+
+        write_sql(build_sql(target, options), options[:output])
+        0
+      end
+
+      private
+
+      def build_sql(_target, _options)
+        raise NotImplementedError
+      end
+    end
+
+    class ViewCommand < SqlCommand
+      TARGETS = %w[postgres mysql].freeze
+      IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/.freeze
+      DICTIONARY_OPTION_DEFINITIONS = [
+        [:dictionary_table, "dictionary-table", "MCDB2 immutable dictionary registry table"],
+        [:dictionary_id_column, "dictionary-id-column", "MCDB2 FK column on source table"],
+        [:registry_id_column, "registry-id-column", "registry PK (default: id)"],
+        [:dictionary_sha256_column, "dictionary-sha256-column", "registry SHA-256 column (default: sha256)"],
+        [:dictionary_bytes_column, "dictionary-bytes-column", "registry dictionary bytes column (default: bytes)"]
+      ].map(&:freeze).freeze
+
+      private_constant :DICTIONARY_OPTION_DEFINITIONS
+
+      private
+
+      def build_sql(target, options)
+        ReadableView.new(target, options).to_sql
+      end
+
+      def default_options
+        {
+          columns: [], as: "payload", output: nil, extension_schema: "multi_compress",
+          dictionary_table: nil, dictionary_id_column: nil, registry_id_column: "id",
+          dictionary_sha256_column: "sha256", dictionary_bytes_column: "bytes"
+        }
+      end
+
+      def option_parser(target, options)
+        OptionParser.new do |parser|
+          parser.banner = view_usage(target)
+          add_source_options(parser, options)
+          add_dictionary_options(parser, options)
+          add_view_output_options(parser, target, options)
+        end
+      end
+
+      def view_usage(target)
+        "Usage: multi_compress db view #{target} --table SCHEMA.TABLE --column COLUMN " \
+          "--view SCHEMA.VIEW --columns ID,CREATED_AT [OPTIONS]"
+      end
+
+      def add_source_options(parser, options)
+        parser.on("--table NAME", "source table, optionally schema-qualified") { |value| options[:table] = value }
+        parser.on("--column NAME", "compressed bytea/BLOB column") { |value| options[:column] = value }
+        parser.on("--view NAME", "destination view, optionally schema-qualified") { |value| options[:view] = value }
+        parser.on("--columns LIST", "comma-separated plain columns to expose") do |value|
+          options[:columns] = value.split(",").map(&:strip)
+        end
+        parser.on("--as NAME", "decoded column name (default: payload)") { |value| options[:as] = value }
+      end
+
+      def add_dictionary_options(parser, options)
+        add_value_options(parser, options, DICTIONARY_OPTION_DEFINITIONS)
+      end
+
+      def add_view_output_options(parser, target, options)
+        if target == "postgres"
+          parser.on("--extension-schema NAME", "extension schema (default: multi_compress)") do |value|
+            options[:extension_schema] = value
+          end
+        end
+        parser.on("-o", "--output PATH", "write SQL to PATH instead of stdout") { |value| options[:output] = value }
+        parser.on("-h", "--help", "show this help") { options[:help] = parser }
+      end
+    end
+
+    class RegistryCommand < SqlCommand
+      TARGETS = %w[postgres mysql].freeze
+      TABLE_OPTION_DEFINITIONS = [
+        [:table, "table", "versions table (default: mcdb_dictionary_versions)"],
+        [:heads_table, "heads-table", "heads table (default: mcdb_dictionary_heads)"],
+        [:payload_table, "payload-table", "MCDB2 source table in this schema/database"],
+        [:payload_column, "payload-column", "MCDB2 compressed envelope column on --payload-table"],
+        [:payload_dictionary_id_column, "payload-dictionary-id-column",
+         "MCDB2 dictionary FK column on --payload-table"]
+      ].map(&:freeze).freeze
+
+      private_constant :TABLE_OPTION_DEFINITIONS
+
+      private
+
+      def build_sql(target, options)
+        DictionaryRegistry.new(target, options).to_sql
+      end
+
+      def default_options
+        { output: nil, extension_schema: "multi_compress", table: "mcdb_dictionary_versions",
+          heads_table: "mcdb_dictionary_heads" }
+      end
+
+      def option_parser(target, options)
+        OptionParser.new do |parser|
+          parser.banner = "Usage: multi_compress db registry #{target} [OPTIONS]"
+          add_target_options(parser, target, options)
+          add_registry_table_options(parser, options)
+          add_registry_output_options(parser, options)
+        end
+      end
+
+      def add_target_options(parser, target, options)
+        return add_postgres_options(parser, options) if target == "postgres"
+
+        parser.on("--database NAME", "application database") { |value| options[:schema] = value }
+      end
+
+      def add_postgres_options(parser, options)
+        parser.on("--schema NAME", "application schema") { |value| options[:schema] = value }
+        parser.on("--owner ROLE", "NOLOGIN owner for append-only dictionary versions") do |value|
+          options[:owner] = value
+        end
+        parser.on("--migration-role ROLE", "role allowed to insert dictionary versions") do |value|
+          options[:migration_role] = value
+        end
+        parser.on("--extension-schema NAME", "extension schema (default: multi_compress)") do |value|
+          options[:extension_schema] = value
+        end
+      end
+
+      def add_registry_table_options(parser, options)
+        add_value_options(parser, options, TABLE_OPTION_DEFINITIONS)
+      end
+
+      def add_registry_output_options(parser, options)
+        parser.on("-o", "--output PATH", "write SQL to PATH instead of stdout") { |value| options[:output] = value }
+        parser.on("-h", "--help", "show this help") { options[:help] = parser }
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/deployment_bundle.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "multi_compress/db_deployment/bundle_stager"
+require "multi_compress/db_deployment/bundle_metadata"
+require "multi_compress/db_deployment/bundle_archiver"
+
+module MultiCompress
+  module DBDeployment
+    class DeploymentBundle
+      attr_reader :target, :output, :force, :root_name
+      private :target, :output, :force, :root_name
+
+      DEFAULT_ARCHIVE_EPOCH = 315_619_200
+
+      TARGETS = {
+        "postgres" => {
+          source_paths: %w[
+            db_core
+            postgres_extension
+            ext/multi_compress/vendor/zstd
+            docs/database-envelope-v1.md
+            docs/database-envelope-v2.md
+            docs/rfcs/0002-mcdb2-dictionaries.md
+            LICENSE.txt
+            THIRD_PARTY_NOTICES.md
+          ],
+          template_path: "db_deployment/postgres"
+        },
+        "mysql" => {
+          source_paths: %w[
+            db_core
+            mysql_udf
+            ext/multi_compress/vendor/zstd
+            docs/database-envelope-v1.md
+            docs/database-envelope-v2.md
+            docs/rfcs/0002-mcdb2-dictionaries.md
+            LICENSE.txt
+            THIRD_PARTY_NOTICES.md
+          ],
+          template_path: "db_deployment/mysql"
+        }
+      }.freeze
+
+      def initialize(target, output, force:)
+        @target = target
+        @output = File.expand_path(output)
+        @force = force
+        @root_name = "multi_compress-#{target}-#{MultiCompress::VERSION}"
+      end
+
+      def write!
+        validate_sources!
+        validate_output!
+        Dir.mktmpdir("multi-compress-db-package") { |directory| build_bundle(directory) }
+      end
+
+      private
+
+      def validate_sources!
+        return if File.directory?(ROOT)
+
+        raise Error, "deployment sources are unavailable from #{ROOT}"
+      end
+
+      def validate_output!
+        return unless File.exist?(output) && !force
+
+        raise Error, "#{output} already exists (use --force to overwrite)"
+      end
+
+      def build_bundle(directory)
+        bundle_root = File.join(directory, root_name)
+        FileUtils.mkdir_p(bundle_root)
+        BundleStager.new(ROOT, TARGETS.fetch(target)).stage!(bundle_root)
+        BundleMetadata.new(target).write!(bundle_root)
+        BundleArchiver.new(output, root_name).write!(directory)
+      end
+    end
+  end
+end

data/lib/multi_compress/db_deployment/deterministic_tar_writer.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module MultiCompress
+  module DBDeployment
+    class DeterministicTarWriter < Gem::Package::TarWriter
+      attr_reader :archive_epoch
+      private :archive_epoch
+
+      def self.new(io, epoch)
+        writer = allocate
+        writer.send(:initialize, io, epoch)
+        return writer unless block_given?
+
+        begin
+          yield writer
+        ensure
+          writer.close
+        end
+
+        nil
+      end
+
+      def initialize(io, epoch)
+        @archive_epoch = epoch
+        super(io)
+      end
+
+      def add_file_simple(name, mode, size)
+        check_closed
+        name, prefix = split_name(name)
+        write_file_header(name, prefix, mode, size)
+        write_file_body(size) { |output| yield output if block_given? }
+        self
+      end
+
+      def mkdir(name, mode)
+        check_closed
+        name, prefix = split_name(name)
+        @io.write(directory_header(name, prefix, mode))
+        self
+      end
+
+      private
+
+      def write_file_header(name, prefix, mode, size)
+        header = Gem::Package::TarHeader.new(
+          name: name,
+          mode: mode,
+          size: size,
+          prefix: prefix,
+          mtime: archive_epoch
+        ).to_s
+        @io.write(header)
+      end
+
+      def write_file_body(size)
+        output = BoundedStream.new(@io, size)
+        yield output
+        @io.write("\0" * (size - output.written))
+        padding = (512 - (size % 512)) % 512
+        @io.write("\0" * padding)
+      end
+
+      def directory_header(name, prefix, mode)
+        Gem::Package::TarHeader.new(
+          name: name,
+          mode: mode,
+          typeflag: "5",
+          size: 0,
+          prefix: prefix,
+          mtime: archive_epoch
+        )
+      end
+    end
+  end
+end