adsp 1.0.0

data/lib/adsp.rb

@@ -0,0 +1,8 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require_relative "adsp/stream/reader"
+require_relative "adsp/stream/writer"
+require_relative "adsp/file"
+require_relative "adsp/string"
+require_relative "adsp/version"

data/test/common.rb

@@ -0,0 +1,108 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "parallel"
+require "securerandom"
+require "tempfile"
+
+require_relative "coverage_helper"
+
+module ADSP
+  module Test
+    module Common
+      BASE_PATH = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..")).freeze
+      TEMP_PATH = ::File.join(BASE_PATH, "tmp").freeze
+
+      SOURCE_PATH  = ::File.join(TEMP_PATH, "source").freeze
+      ARCHIVE_PATH = ::File.join(TEMP_PATH, "archive").freeze
+
+      [
+        SOURCE_PATH,
+        ARCHIVE_PATH
+      ]
+      .each { |path| FileUtils.touch path }
+
+      def self.get_path(path, suffix)
+        "#{path}_#{suffix}"
+      end
+
+      ENCODINGS = %w[
+        binary
+        UTF-8
+        UTF-16LE
+      ]
+      .map { |encoding_name| ::Encoding.find encoding_name }
+      .freeze
+
+      TRANSCODE_OPTIONS = {
+        :invalid => :replace,
+        :undef   => :replace,
+        :replace => "?"
+      }
+      .freeze
+
+      def self.generate_texts(*sources)
+        sources.flat_map do |source|
+          ENCODINGS.map do |encoding|
+            source.encode encoding, **TRANSCODE_OPTIONS
+          end
+        end
+      end
+
+      TEXTS = generate_texts(
+        "",
+        ::SecureRandom.random_bytes(1 << 8) # 256 B
+      )
+      .freeze
+
+      LARGE_TEXTS = generate_texts(
+        ::SecureRandom.random_bytes(1 << 21) # 2 MB
+      )
+      .freeze
+
+      # It is better to have text lengths not divisible by portion lengths.
+      PORTION_LENGTHS = [
+        10**2,
+        5 * (10**2)
+      ]
+      .freeze
+
+      # It is better to have large text lengths not divisible by large portion lengths.
+      LARGE_PORTION_LENGTHS = [
+        10**6,
+        5 * (10**6)
+      ]
+      .freeze
+
+      # We need at least 2 threads for testing multiple threads support.
+      THREADS_COUNT = [Parallel.processor_count, 2].max.freeze
+
+      def self.parallel(producer, &_block)
+        Parallel.each producer, :in_threads => THREADS_COUNT do |item|
+          yield item, Parallel.worker_number
+        end
+      end
+
+      def self.parallel_options(generator, &block)
+        producer = proc do
+          next Parallel::Stop if generator.finished?
+
+          generator.next
+        end
+
+        parallel producer, &block
+      end
+
+      def self.file_can_be_used_nonblock?
+        ::File.open(::Tempfile.new, "w") do |file|
+          file.write_nonblock "text"
+        end
+      rescue Errno::EBADF
+        # Nonblock operations with files are not available on Windows.
+        false
+      else
+        true
+      end
+    end
+  end
+end

data/test/coverage_helper.rb

@@ -0,0 +1,18 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+if ENV["CI"]
+  require "codecov"
+  require "simplecov"
+
+  SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new(
+    [
+      SimpleCov::Formatter::HTMLFormatter,
+      SimpleCov::Formatter::Codecov
+    ]
+  )
+
+  SimpleCov.start do
+    add_filter %r{^/test/}
+  end
+end

data/test/file.test.rb

@@ -0,0 +1,120 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/file"
+
+require_relative "common"
+require_relative "minitest"
+require_relative "mock/file"
+require_relative "option"
+require_relative "validation"
+
+module ADSP
+  module Test
+    class File < Minitest::Test
+      Option = Test::Option
+      Target = Mock::File
+
+      SOURCE_PATH  = Common::SOURCE_PATH
+      ARCHIVE_PATH = Common::ARCHIVE_PATH
+      TEXTS        = Common::TEXTS
+      LARGE_TEXTS  = Common::LARGE_TEXTS
+
+      BUFFER_LENGTH_NAMES   = %i[source_buffer_length destination_buffer_length].freeze
+      BUFFER_LENGTH_MAPPING = {
+        :source_buffer_length      => :destination_buffer_length,
+        :destination_buffer_length => :source_buffer_length
+      }
+      .freeze
+
+      def test_invalid_arguments
+        Validation::INVALID_STRINGS.each do |invalid_path|
+          assert_raises ValidateError do
+            Target.compress invalid_path, ARCHIVE_PATH
+          end
+
+          assert_raises ValidateError do
+            Target.compress SOURCE_PATH, invalid_path
+          end
+
+          assert_raises ValidateError do
+            Target.decompress invalid_path, SOURCE_PATH
+          end
+
+          assert_raises ValidateError do
+            Target.decompress ARCHIVE_PATH, invalid_path
+          end
+        end
+
+        get_invalid_compressor_options do |invalid_options|
+          assert_raises ValidateError do
+            Target.compress SOURCE_PATH, ARCHIVE_PATH, invalid_options
+          end
+        end
+
+        get_invalid_decompressor_options do |invalid_options|
+          assert_raises ValidateError do
+            Target.decompress ARCHIVE_PATH, SOURCE_PATH, invalid_options
+          end
+        end
+      end
+
+      def test_texts
+        parallel_compressor_options do |compressor_options, worker_index|
+          source_path  = Common.get_path SOURCE_PATH, worker_index
+          archive_path = Common.get_path ARCHIVE_PATH, worker_index
+
+          TEXTS.each do |text|
+            ::File.write source_path, text, :mode => "wb"
+            Target.compress source_path, archive_path, compressor_options
+
+            get_compatible_decompressor_options compressor_options do |decompressor_options|
+              Target.decompress archive_path, source_path, decompressor_options
+
+              decompressed_text = ::File.read source_path, :mode => "rb"
+              decompressed_text.force_encoding text.encoding
+
+              assert_equal text, decompressed_text
+            end
+          end
+        end
+      end
+
+      def test_large_texts
+        Common.parallel LARGE_TEXTS do |text, worker_index|
+          source_path  = Common.get_path SOURCE_PATH, worker_index
+          archive_path = Common.get_path ARCHIVE_PATH, worker_index
+
+          ::File.write source_path, text, :mode => "wb"
+          Target.compress source_path, archive_path
+          Target.decompress archive_path, source_path
+
+          decompressed_text = ::File.read source_path, :mode => "rb"
+          decompressed_text.force_encoding text.encoding
+
+          assert_equal text, decompressed_text
+        end
+      end
+
+      # -----
+
+      def get_invalid_compressor_options(&block)
+        self.class::Option.get_invalid_compressor_options BUFFER_LENGTH_NAMES, &block
+      end
+
+      def get_invalid_decompressor_options(&block)
+        self.class::Option.get_invalid_decompressor_options BUFFER_LENGTH_NAMES, &block
+      end
+
+      def parallel_compressor_options(&block)
+        Common.parallel_options self.class::Option.get_compressor_options_generator(BUFFER_LENGTH_NAMES), &block
+      end
+
+      def get_compatible_decompressor_options(compressor_options, &block)
+        self.class::Option.get_compatible_decompressor_options compressor_options, BUFFER_LENGTH_MAPPING, &block
+      end
+    end
+
+    Minitest << File
+  end
+end

data/test/minitest.rb

@@ -0,0 +1,20 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "minitest"
+require "minitest/autorun"
+
+module Minitest
+  Minitest.module_eval do
+    class << self
+      def <<(klass)
+        Runnable.runnables << klass unless Runnable.runnables.include? klass
+        nil
+      end
+    end
+  end
+
+  Runnable.instance_eval do
+    def self.inherited(_klass); end # rubocop:disable Lint/MissingSuper
+  end
+end

data/test/mock/common.rb

@@ -0,0 +1,57 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/string"
+
+require_relative "../common"
+
+module ADSP
+  module Test
+    module Mock
+      module Common
+        DEFAULT_DESTINATION_BUFFER_LENGTH = 1 << 18 # 256 KB
+
+        # Flipping bytes and packing them as 16-bit values in network byte order.
+        # This method will make compressed data bytesize more than original bytesize.
+        # It will help to improve testing coverage.
+        def self.native_compress(source, destination_buffer_length)
+          source_length             = source.bytesize
+          destination_buffer_length = DEFAULT_DESTINATION_BUFFER_LENGTH if destination_buffer_length.zero?
+
+          bytes_read =
+            if destination_buffer_length.nil?
+              source_length
+            else
+              [source_length, destination_buffer_length / 2].min
+            end
+
+          result = source
+            .byteslice(0, bytes_read)
+            .unpack("C*")
+            .map { |byte| byte ^ 0xFF }
+            .pack "n*"
+          [result, bytes_read]
+        end
+
+        def self.native_decompress(source, destination_buffer_length)
+          source_length             = source.bytesize / 2 * 2
+          destination_buffer_length = DEFAULT_DESTINATION_BUFFER_LENGTH if destination_buffer_length.zero?
+
+          bytes_read =
+            if destination_buffer_length.nil?
+              source_length
+            else
+              [source_length, destination_buffer_length * 2].min
+            end
+
+          result = source
+            .byteslice(0, bytes_read)
+            .unpack("n*")
+            .map { |byte| byte ^ 0xFF }
+            .pack "C*"
+          [result, bytes_read]
+        end
+      end
+    end
+  end
+end

data/test/mock/file.rb

@@ -0,0 +1,60 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/file"
+
+require_relative "./common"
+
+module ADSP
+  module Test
+    module Mock
+      class File < ADSP::File
+        PORTION_LENGTH = 10**6
+
+        def self.native_compress_io(source_io, destination_io, options)
+          destination_buffer_length = options[:destination_buffer_length]
+          source                    = "".b
+
+          loop do
+            begin
+              source << source_io.readpartial(PORTION_LENGTH)
+            rescue ::EOFError
+              break
+            end
+
+            until source.empty?
+              data, bytes_read = Common.native_compress source, destination_buffer_length
+              source = source.byteslice bytes_read, source.bytesize - bytes_read
+
+              destination_io.write data
+            end
+          end
+
+          nil
+        end
+
+        def self.native_decompress_io(source_io, destination_io, options)
+          destination_buffer_length = options[:destination_buffer_length]
+          source                    = "".b
+
+          loop do
+            begin
+              source << source_io.readpartial(PORTION_LENGTH)
+            rescue ::EOFError
+              break
+            end
+
+            until source.empty?
+              data, bytes_read = Common.native_decompress source, destination_buffer_length
+              source = source.byteslice bytes_read, source.bytesize - bytes_read
+
+              destination_io.write data
+            end
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/raw/compressor.rb

@@ -0,0 +1,20 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/stream/raw/compressor"
+
+require_relative "native_compressor"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        module Raw
+          class Compressor < ADSP::Stream::Raw::Compressor
+            NativeCompressor = Raw::NativeCompressor
+          end
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/raw/decompressor.rb

@@ -0,0 +1,20 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/stream/raw/decompressor"
+
+require_relative "native_decompressor"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        module Raw
+          class Decompressor < ADSP::Stream::Raw::Decompressor
+            NativeDecompressor = Raw::NativeDecompressor
+          end
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/raw/native_compressor.rb

@@ -0,0 +1,82 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/error"
+require "adsp/validation"
+
+require_relative "../../common"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        module Raw
+          class NativeCompressor
+            def initialize(options)
+              ADSP::Validation.validate_hash options
+
+              destination_buffer_length = options[:destination_buffer_length]
+              ADSP::Validation.validate_not_negative_integer destination_buffer_length
+
+              destination_buffer_length = Common::DEFAULT_DESTINATION_BUFFER_LENGTH \
+                if destination_buffer_length.zero?
+
+              @destination_buffer        = "".b
+              @destination_buffer_length = destination_buffer_length
+
+              @is_closed = false
+            end
+
+            def write(source)
+              ADSP::Validation.validate_string source
+
+              do_not_use_after_close
+
+              remaining_destination_buffer_length = @destination_buffer_length - @destination_buffer.bytesize
+              return [0, true] if remaining_destination_buffer_length.zero?
+
+              data, bytes_read = Common.native_compress source, remaining_destination_buffer_length
+              needs_more_destination = bytes_read < source.bytesize
+              @destination_buffer << data
+
+              [bytes_read, needs_more_destination]
+            end
+
+            def read_result
+              do_not_use_after_close
+
+              result = @destination_buffer
+              @destination_buffer = "".b
+
+              result
+            end
+
+            def flush
+              do_not_use_after_close
+
+              nil
+            end
+
+            def finish
+              do_not_use_after_close
+
+              nil
+            end
+
+            def close
+              do_not_use_after_close
+
+              @is_closed = true
+
+              nil
+            end
+
+            protected def do_not_use_after_close
+              raise UsedAfterCloseError if @is_closed
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/raw/native_decompressor.rb

@@ -0,0 +1,70 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/error"
+require "adsp/validation"
+
+require_relative "../../common"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        module Raw
+          class NativeDecompressor
+            def initialize(options)
+              ADSP::Validation.validate_hash options
+
+              destination_buffer_length = options[:destination_buffer_length]
+              ADSP::Validation.validate_not_negative_integer destination_buffer_length
+
+              destination_buffer_length = Common::DEFAULT_DESTINATION_BUFFER_LENGTH \
+                if destination_buffer_length.zero?
+
+              @destination_buffer        = "".b
+              @destination_buffer_length = destination_buffer_length
+
+              @is_closed = false
+            end
+
+            def read(source)
+              ADSP::Validation.validate_string source
+
+              do_not_use_after_close
+
+              remaining_destination_buffer_length = @destination_buffer_length - @destination_buffer.bytesize
+              return [0, true] if remaining_destination_buffer_length.zero?
+
+              data, bytes_read = Common.native_decompress source, remaining_destination_buffer_length
+              needs_more_destination = bytes_read < source.bytesize
+              @destination_buffer << data
+
+              [bytes_read, needs_more_destination]
+            end
+
+            def read_result
+              do_not_use_after_close
+
+              result = @destination_buffer
+              @destination_buffer = "".b
+
+              result
+            end
+
+            def close
+              do_not_use_after_close
+
+              @is_closed = true
+
+              nil
+            end
+
+            protected def do_not_use_after_close
+              raise UsedAfterCloseError if @is_closed
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/reader.rb

@@ -0,0 +1,18 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/stream/reader"
+
+require_relative "raw/decompressor"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        class Reader < ADSP::Stream::Reader
+          RawDecompressor = Mock::Stream::Raw::Decompressor
+        end
+      end
+    end
+  end
+end

data/test/mock/stream/writer.rb

@@ -0,0 +1,18 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/stream/writer"
+
+require_relative "raw/compressor"
+
+module ADSP
+  module Test
+    module Mock
+      module Stream
+        class Writer < ADSP::Stream::Writer
+          RawCompressor = Mock::Stream::Raw::Compressor
+        end
+      end
+    end
+  end
+end

data/test/mock/string.rb

@@ -0,0 +1,44 @@
+# Abstract data stream processor.
+# Copyright (c) 2021 AUTHORS, MIT License.
+
+require "adsp/string"
+
+require_relative "./common"
+
+module ADSP
+  module Test
+    module Mock
+      class String < ADSP::String
+        def self.native_compress_string(source, options)
+          destination_buffer_length = options[:destination_buffer_length]
+          result                    = "".b
+
+          loop do
+            data, bytes_read = Common.native_compress source, destination_buffer_length
+            source = source.byteslice bytes_read, source.bytesize - bytes_read
+            result << data
+
+            break if source.bytesize.zero?
+          end
+
+          result
+        end
+
+        def self.native_decompress_string(source, options)
+          destination_buffer_length = options[:destination_buffer_length]
+          result                    = "".b
+
+          loop do
+            data, bytes_read = Common.native_decompress source, destination_buffer_length
+            source = source.byteslice bytes_read, source.bytesize - bytes_read
+            result << data
+
+            break if source.bytesize.zero?
+          end
+
+          result
+        end
+      end
+    end
+  end
+end