compare_compressors 0.0.1

data/lib/compare_compressors/compressor.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tmpdir'
+
+module CompareCompressors
+  #
+  # Base class for compressors. Subclasses provide compressor-specific
+  # configuration and logic.
+  #
+  class Compressor
+    #
+    # Run the compressor at the given level on the given target and measure
+    # its running time and memory usage.
+    #
+    # @param [String] target original pathname of the target (read only)
+    # @param [String] work_target temporary path of the target (read/write)
+    # @param [Numeric] level the compression level
+    # @return [Result]
+    #
+    def evaluate(target, work_target, level)
+      compression_times = time(compression_command(work_target, level))
+      size = output_size(work_target)
+      remove_if_exists(work_target)
+
+      decompression_times = time(decompression_command(work_target))
+      remove_if_exists(output_name(work_target))
+
+      Result.new(
+        target, name, level, *compression_times, size, *decompression_times
+      )
+    end
+
+    #
+    # @abstract
+    # @return [String] name that can be a ruby symbol
+    #
+    def name
+      raise NotImplementedError
+    end
+
+    #
+    # @abstract
+    # @return [String] extension added to the compressed file
+    #
+    def extension
+      raise NotImplementedError
+    end
+
+    #
+    # @abstract
+    # @return [Array<Integer>] the levels supported by the compressor
+    #
+    def levels
+      raise NotImplementedError
+    end
+
+    #
+    # @abstract
+    # @return [String?] version string (for information only)
+    #
+    def version
+      nil
+    end
+
+    #
+    # @return [String] display name (need not be safe to intern as a symbol)
+    #
+    def display_name
+      name
+    end
+
+    #
+    # @abstract
+    # @return [Array<String>] command to run the compressor
+    #
+    def compression_command
+      raise NotImplementedError
+    end
+
+    #
+    # @abstract
+    # @return [Array<String>] command to run the compressor in decompress mode
+    #
+    def decompression_command
+      raise NotImplementedError
+    end
+
+    private
+
+    def output_size(target)
+      File.stat(output_name(target)).size
+    end
+
+    def output_name(target)
+      "#{target}.#{extension}"
+    end
+
+    def time(command)
+      status, times, out, err = run(*command)
+      return times if status.zero?
+      raise format(
+        "%s: %s exited with %d:\n%s\n%s",
+        name, command.join(' '), status, out, err
+      )
+    end
+
+    def run(*command, **options)
+      Dir.mktmpdir do |tmp|
+        out_pathname = File.join(tmp, 'out')
+        err_pathname = File.join(tmp, 'err')
+        options[:out] = out_pathname
+        options[:err] = err_pathname
+        options[:in] = '/dev/null'
+
+        # Note: this is not the shell builtin but rather /usr/bin/time; at least
+        # on Ubuntu, the latter reports both time and max RSS (memory usage)
+        # metrics, which is what we want here. Write the time output to a
+        # temporary file to avoid conflicting with the child's stderr output.
+        time_pathname = File.join(tmp, 'time')
+        timed_command = [
+          'time', '--format=%e %S %U %M', "--output=#{time_pathname}"
+        ] + command
+
+        Process.waitpid(Process.spawn(*timed_command, **options))
+
+        [
+          $CHILD_STATUS.exitstatus,
+          parse_time(time_pathname),
+          File.read(out_pathname),
+          File.read(err_pathname)
+        ]
+      end
+    end
+
+    # Returns elapsed time in seconds, total (system plus user) CPU time in
+    # seconds, and maximum resident set size (memory usage) in Kilobytes, which
+    # I think means KiB.
+    def parse_time(time_pathname)
+      elapsed, sys, user, max_rss = File.read(time_pathname).split
+      [elapsed.to_f, sys.to_f + user.to_f, max_rss.to_i]
+    end
+
+    def remove_if_exists(pathname)
+      FileUtils.rm pathname
+    rescue Errno::ENOENT
+      nil # not a problem
+    end
+  end
+end

data/lib/compare_compressors/compressors/brotli_compressor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with Brotli.
+  #
+  # Note: At present, the command does not seem to have anything that prints a
+  # version, so we can't implement `version`.
+  #
+  class BrotliCompressor < Compressor
+    def name
+      'brotli'
+    end
+
+    def extension
+      'bro'
+    end
+
+    # Can't find any documentation about this, so this is based on
+    # https://github.com/google/brotli/blob/cdca91b6f59dd7632985667d2cd585ab68937b48/enc/quality.h
+    def levels
+      (0..11).to_a
+    end
+
+    def compression_command(target, level)
+      [
+        'brotli',
+        '--input', target,
+        '--output', output_name(target),
+        '--quality', level.to_s
+      ]
+    end
+
+    def decompression_command(target)
+      [
+        'brotli',
+        '--decompress',
+        '--input', output_name(target),
+        '--output', target
+      ]
+    end
+  end
+end

data/lib/compare_compressors/compressors/bzip2_compressor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with bzip2.
+  #
+  class Bzip2Compressor < Compressor
+    def name
+      'bzip2'
+    end
+
+    def extension
+      'bz2'
+    end
+
+    def levels
+      (1..9).to_a
+    end
+
+    def version
+      status, _times, _out, err = run(name, '--version')
+      return nil unless status.zero?
+      version_line = err.lines.first.chomp
+      raise "bad #{name} version line: #{version_line.inspect}" unless
+        version_line =~ /Version (.+)\.\z/
+      Regexp.last_match(1)
+    end
+
+    def compression_command(target, level)
+      ['bzip2', "-#{level}", target]
+    end
+
+    def decompression_command(target)
+      ['bunzip2', output_name(target)]
+    end
+  end
+end

data/lib/compare_compressors/compressors/gzip_compressor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with gzip.
+  #
+  class GzipCompressor < Compressor
+    def name
+      'gzip'
+    end
+
+    def extension
+      'gz'
+    end
+
+    def levels
+      (1..9).to_a
+    end
+
+    def version
+      status, _times, out, _err = run(name, '--version')
+      return nil unless status.zero?
+      out.lines.first.chomp
+    end
+
+    def compression_command(target, level)
+      ['gzip', "-#{level}", target]
+    end
+
+    def decompression_command(target)
+      ['gunzip', output_name(target)]
+    end
+  end
+end

data/lib/compare_compressors/compressors/seven_zip_compressor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with `7z`.
+  #
+  class SevenZipCompressor < Compressor
+    def name
+      'seven_zip'
+    end
+
+    def display_name
+      '7z'
+    end
+
+    def extension
+      '7z'
+    end
+
+    # Based on share/doc/p7zip/DOC/MANUAL/cmdline/switches/method.htm
+    # Level 0 is no compression, so we exclude it.
+    def levels
+      [1, 3, 5, 7, 9]
+    end
+
+    def version
+      status, _times, out, _err = run('7zr', '--help')
+      return nil unless status.zero?
+      version_line = out.strip.lines.first.chomp
+      raise "bad version line #{version_line}" unless
+        version_line =~ /([0-9.]+)[\s:]+Copyright/
+      Regexp.last_match(1)
+    end
+
+    def compression_command(target, level)
+      ['7zr', 'a', "-mx=#{level}", output_name(target), target]
+    end
+
+    def decompression_command(target)
+      ['7zr', 'x', "-o#{File.dirname(target)}", output_name(target)]
+    end
+  end
+end

data/lib/compare_compressors/compressors/xz_compressor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with `xz` (LZMA).
+  #
+  class XzCompressor < Compressor
+    def name
+      'xz'
+    end
+
+    def extension
+      'xz'
+    end
+
+    def levels
+      (0..9).to_a
+    end
+
+    def version
+      status, _times, out, _err = run(name, '--version')
+      return nil unless status.zero?
+      version_line = out.lines.first.chomp
+      raise "bad version line #{version_line}" unless
+        version_line =~ /([0-9.a-z]+)$/
+      Regexp.last_match(1)
+    end
+
+    def compression_command(target, level)
+      ['xz', "-#{level}", target]
+    end
+
+    def decompression_command(target)
+      ['unxz', output_name(target)]
+    end
+  end
+end

data/lib/compare_compressors/compressors/zstd_compressor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Compress with Zstandard.
+  #
+  class ZstdCompressor < Compressor
+    def name
+      'zstd'
+    end
+
+    def extension
+      'zst'
+    end
+
+    def levels
+      (1..19).to_a
+    end
+
+    def version
+      status, _times, out, _err = run(name, '-V')
+      return nil unless status.zero?
+      version_line = out.lines.first.chomp
+      raise "bad version line #{version_line}" unless
+        version_line =~ /([0-9.]+),/
+      Regexp.last_match(1)
+    end
+
+    def compression_command(target, level)
+      ['zstd', "-#{level}", target]
+    end
+
+    def decompression_command(target)
+      ['unzstd', output_name(target)]
+    end
+  end
+end

data/lib/compare_compressors/cost_model.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Define costs for comparing grouped results.
+  #
+  class CostModel
+    # Default to current Amazon S3 storage cost per GiB*month ($).
+    DEFAULT_GIBYTE_COST = 0.023
+
+    # Default to on-demand cost for an Amazon EC2 m3.medium ($).
+    DEFAULT_HOUR_COST = 0.073
+
+    # Default to elapsed time rather than CPU time.
+    DEFAULT_USE_CPU_TIME = false
+
+    # Default to dollars.
+    DEFAULT_CURRENCY = '$'
+
+    # Default to dollars.
+    DEFAULT_SUMMARIZE_TOP = 5
+
+    def initialize(
+      gibyte_cost: DEFAULT_GIBYTE_COST,
+      compression_hour_cost: DEFAULT_HOUR_COST,
+      decompression_hour_cost: DEFAULT_HOUR_COST,
+      use_cpu_time: DEFAULT_USE_CPU_TIME,
+      currency: DEFAULT_CURRENCY
+    )
+      @gibyte_cost = gibyte_cost
+      @compression_hour_cost = compression_hour_cost
+      @decompression_hour_cost = decompression_hour_cost
+      @use_cpu_time = use_cpu_time
+      @currency = currency
+    end
+
+    attr_reader :gibyte_cost
+    attr_reader :compression_hour_cost
+    attr_reader :decompression_hour_cost
+    attr_reader :use_cpu_time
+    attr_reader :currency
+
+    def cost(grouped_results)
+      grouped_results.map do |group_result|
+        CostedGroupResult.new(self, group_result)
+      end
+    end
+
+    def summarize(costed_grouped_results, top = DEFAULT_SUMMARIZE_TOP)
+      costed_grouped_results.sort_by(&:total_cost).take(top).map do |result|
+        result.to_s(currency)
+      end
+    end
+  end
+end

data/lib/compare_compressors/costed_group_result.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module CompareCompressors
+  #
+  # Grouped result with costs calculated.
+  #
+  CostedGroupResult = Struct.new(
+    :compressor_name,
+    :compressor_level,
+    :mean_compression_elapsed_hours,
+    :mean_compression_cpu_hours,
+    :max_compression_max_rss,
+    :mean_compressed_gibytes,
+    :mean_compression_delta_gibytes,
+    :geomean_compression_ratio,
+    :mean_decompression_elapsed_hours,
+    :mean_decompression_cpu_hours,
+    :max_decompression_max_rss,
+    :compression_hour_cost,
+    :decompression_hour_cost,
+    :hour_cost,
+    :gibyte_cost,
+    :total_cost
+  ) do
+    def self.new_from_group_result(cost_model, group_result)
+      if cost_model.use_cpu_time
+        compression_hours = group_result.mean_compression_cpu_hours
+        decompression_hours = group_result.mean_decompression_cpu_hours
+      else
+        compression_hours = group_result.mean_compression_elapsed_hours
+        decompression_hours = group_result.mean_decompression_elapsed_hours
+      end
+      compression_hour_cost =
+        cost_model.compression_hour_cost * compression_hours
+      decompression_hour_cost =
+        cost_model.decompression_hour_cost * decompression_hours
+      hour_cost = compression_hour_cost + decompression_hour_cost
+      gibyte_cost =
+        cost_model.gibyte_cost *
+        group_result.mean_compressed_gibytes
+      new(
+        group_result.compressor_name,
+        group_result.compressor_level,
+        group_result.mean_compression_cpu_hours,
+        group_result.mean_compression_elapsed_hours,
+        group_result.max_compression_max_rss,
+        group_result.mean_compressed_gibytes,
+        group_result.mean_compression_delta_gibytes,
+        group_result.geomean_compression_ratio,
+        group_result.mean_decompression_elapsed_hours,
+        group_result.mean_decompression_cpu_hours,
+        group_result.max_decompression_max_rss,
+        compression_hour_cost,
+        decompression_hour_cost,
+        hour_cost,
+        gibyte_cost,
+        hour_cost + gibyte_cost
+      )
+    end
+
+    def self.from_group_results(cost_model, group_results)
+      group_results.map do |group_result|
+        new_from_group_result(cost_model, group_result)
+      end
+    end
+
+    def to_s(currency = CostModel::DEFAULT_CURRENCY)
+      gib_saved = mean_compression_delta_gibytes
+      <<-STRING
+#{compressor_name} level #{compressor_level}:
+  compression ratio           : #{format('%.2f', geomean_compression_ratio)}
+  compression elapsed hours   : #{format('%.4f', mean_compression_elapsed_hours)}
+  compression CPU hours       : #{format('%.4f', mean_compression_cpu_hours)}
+  compression max RSS (KiB)   : #{format('%d', max_compression_max_rss)}
+  compressed GiB              : #{format('%.4f', mean_compressed_gibytes)}
+  GiB saved                   : #{format('%.2f', gib_saved)}
+  decompression elapsed hours : #{format('%.4f', mean_decompression_elapsed_hours)}
+  decompression CPU hours     : #{format('%.4f', mean_decompression_cpu_hours)}
+  decompression max RSS (KiB) : #{format('%d', max_decompression_max_rss)}
+  ------------------
+  storage cost                : #{format('%s%0.02f', currency, gibyte_cost)}
+  compute cost                : #{format('%s%0.02f', currency, hour_cost)}
+  total cost                  : #{format('%s%0.02f', currency, total_cost)}
+      STRING
+    end
+  end
+end