ruby-zstds 1.0.0

data/ext/extconf.rb

@@ -0,0 +1,82 @@
+# Ruby bindings for zstd library.
+# Copyright (c) 2019 AUTHORS, MIT License.
+
+require "mkmf"
+
+$LDFLAGS << " -pthread" # rubocop:disable Style/GlobalVars
+
+def require_header(name, types = [])
+  abort "Can't find #{name} header" unless find_header name
+
+  types.each do |type|
+    abort "Can't find #{type} type in #{name} header" unless find_type type, nil, name
+  end
+end
+
+require_header "zstd_errors.h", %w[
+  ZSTD_ErrorCode
+]
+require_header "zstd.h", [
+  "ZSTD_CCtx *",
+  "ZSTD_DCtx *",
+  "ZSTD_strategy",
+  "ZSTD_bounds",
+  "ZSTD_inBuffer",
+  "ZSTD_outBuffer"
+]
+require_header "zdict.h"
+
+def require_library(name, functions)
+  functions.each do |function|
+    abort "Can't find #{function} function in #{name} library" unless find_library name, function
+  end
+end
+
+require_library(
+  "zstd",
+  %w[
+    ZSTD_isError
+    ZSTD_getErrorCode
+    ZSTD_createCCtx
+    ZSTD_createDCtx
+    ZSTD_freeCCtx
+    ZSTD_freeDCtx
+    ZSTD_CCtx_setParameter
+    ZSTD_DCtx_setParameter
+    ZSTD_CCtx_setPledgedSrcSize
+    ZSTD_cParam_getBounds
+    ZSTD_dParam_getBounds
+    ZSTD_CStreamInSize
+    ZSTD_CStreamOutSize
+    ZSTD_DStreamInSize
+    ZSTD_DStreamOutSize
+    ZSTD_compressStream2
+    ZSTD_decompressStream
+    ZDICT_getDictID
+    ZDICT_trainFromBuffer
+  ]
+)
+
+extension_name = "zstds_ext".freeze
+dir_config extension_name
+
+# rubocop:disable Style/GlobalVars
+$srcs = %w[
+  stream/compressor
+  stream/decompressor
+  buffer
+  dictionary
+  error
+  io
+  main
+  option
+  string
+]
+.map { |name| "src/#{extension_name}/#{name}.c" }
+.freeze
+
+$CFLAGS << " -Wno-declaration-after-statement"
+$VPATH << "$(srcdir)/#{extension_name}:$(srcdir)/#{extension_name}/stream"
+# rubocop:enable Style/GlobalVars
+
+create_makefile extension_name

data/ext/zstds_ext/buffer.c

@@ -0,0 +1,30 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#include "zstds_ext/buffer.h"
+
+#include <zstd.h>
+
+#include "ruby.h"
+
+VALUE zstds_ext_create_string_buffer(VALUE length)
+{
+  return rb_str_new(NULL, NUM2SIZET(length));
+}
+
+VALUE zstds_ext_resize_string_buffer(VALUE args)
+{
+  VALUE buffer = rb_ary_entry(args, 0);
+  VALUE length = rb_ary_entry(args, 1);
+  return rb_str_resize(buffer, NUM2SIZET(length));
+}
+
+void zstds_ext_buffer_exports(VALUE root_module)
+{
+  VALUE module = rb_define_module_under(root_module, "Buffer");
+
+  rb_define_const(module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(ZSTD_CStreamInSize()));
+  rb_define_const(module, "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_COMPRESSOR", SIZET2NUM(ZSTD_CStreamOutSize()));
+  rb_define_const(module, "DEFAULT_SOURCE_BUFFER_LENGTH_FOR_DECOMPRESSOR", SIZET2NUM(ZSTD_DStreamInSize()));
+  rb_define_const(module, "DEFAULT_DESTINATION_BUFFER_LENGTH_FOR_DECOMPRESSOR", SIZET2NUM(ZSTD_DStreamOutSize()));
+}

data/ext/zstds_ext/buffer.h

@@ -0,0 +1,23 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_BUFFER_H)
+#define ZSTDS_EXT_BUFFER_H
+
+#include "ruby.h"
+
+VALUE zstds_ext_create_string_buffer(VALUE length);
+
+#define ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, length, exception) \
+  VALUE buffer = rb_protect(zstds_ext_create_string_buffer, SIZET2NUM(length), &exception);
+
+VALUE zstds_ext_resize_string_buffer(VALUE args);
+
+#define ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, length, exception)            \
+  VALUE args = rb_ary_new_from_args(2, buffer, SIZET2NUM(length));           \
+  buffer     = rb_protect(zstds_ext_resize_string_buffer, args, &exception); \
+  RB_GC_GUARD(args);
+
+void zstds_ext_buffer_exports(VALUE root_module);
+
+#endif // ZSTDS_EXT_BUFFER_H

data/ext/zstds_ext/common.h

@@ -0,0 +1,16 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_COMMON_H)
+#define ZSTDS_EXT_COMMON_H
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ZSTDS_EXT_MODULE_NAME "ZSTDS"
+
+// WARNING: zstd library are mixing size and error codes inside size_t, dangerous.
+typedef size_t       zstds_result_t;
+typedef uint_fast8_t zstds_ext_result_t;
+
+#endif // ZSTDS_EXT_COMMON_H

data/ext/zstds_ext/dictionary.c

@@ -0,0 +1,106 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#include "zstds_ext/dictionary.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zdict.h>
+
+#include "ruby.h"
+#include "zstds_ext/buffer.h"
+#include "zstds_ext/common.h"
+#include "zstds_ext/error.h"
+#include "zstds_ext/macro.h"
+#include "zstds_ext/option.h"
+
+VALUE zstds_ext_get_dictionary_buffer_id(VALUE ZSTDS_EXT_UNUSED(self), VALUE buffer)
+{
+  unsigned int id = ZDICT_getDictID(RSTRING_PTR(buffer), RSTRING_LEN(buffer));
+  if (id == 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_VALIDATE_FAILED);
+  }
+
+  return UINT2NUM(id);
+}
+
+VALUE zstds_ext_train_dictionary_buffer(VALUE ZSTDS_EXT_UNUSED(self), VALUE samples, VALUE options)
+{
+  Check_Type(samples, T_ARRAY);
+
+  size_t       sample_index;
+  unsigned int samples_length = (unsigned int)RARRAY_LEN(samples);
+  size_t       samples_size   = 0;
+
+  for (sample_index = 0; sample_index < samples_length; sample_index++) {
+    VALUE sample = rb_ary_entry(samples, sample_index);
+    Check_Type(sample, T_STRING);
+
+    samples_size += RSTRING_LEN(sample);
+  }
+
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, capacity);
+
+  if (capacity == 0) {
+    capacity = ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY;
+  }
+
+  int exception;
+
+  ZSTDS_EXT_CREATE_STRING_BUFFER(buffer, capacity, exception);
+  if (exception != 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  uint8_t* samples_buffer = malloc(samples_size);
+  if (samples_buffer == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  size_t* samples_sizes = malloc(samples_length * sizeof(size_t));
+  if (samples_sizes == NULL) {
+    free(samples_buffer);
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  size_t sample_offset = 0;
+
+  for (sample_index = 0; sample_index < samples_length; sample_index++) {
+    VALUE       sample      = rb_ary_entry(samples, sample_index);
+    const char* sample_data = RSTRING_PTR(sample);
+    size_t      sample_size = RSTRING_LEN(sample);
+
+    memmove(samples_buffer + sample_offset, sample_data, sample_size);
+    sample_offset += sample_size;
+
+    samples_sizes[sample_index] = sample_size;
+  }
+
+  zstds_result_t result = ZDICT_trainFromBuffer(
+    RSTRING_PTR(buffer), capacity,
+    samples_buffer, samples_sizes, samples_length);
+
+  free(samples_buffer);
+  free(samples_sizes);
+
+  if (ZSTD_isError(result)) {
+    zstds_ext_raise_error(zstds_ext_get_error(ZSTD_getErrorCode(result)));
+  }
+
+  ZSTDS_EXT_RESIZE_STRING_BUFFER(buffer, result, exception);
+  if (exception != 0) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  return buffer;
+}
+
+void zstds_ext_dictionary_exports(VALUE root_module)
+{
+  VALUE dictionary = rb_define_class_under(root_module, "Dictionary", rb_cObject);
+
+  rb_define_singleton_method(dictionary, "get_buffer_id", zstds_ext_get_dictionary_buffer_id, 1);
+  rb_define_singleton_method(dictionary, "train_buffer", zstds_ext_train_dictionary_buffer, 2);
+}

data/ext/zstds_ext/dictionary.h

@@ -0,0 +1,16 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_DICTIONARY_H)
+#define ZSTDS_EXT_DICTIONARY_H
+
+#include "ruby.h"
+
+#define ZSTDS_EXT_DEFAULT_DICTIONARY_CAPACITY (1 << 17); // 128 KB
+
+VALUE zstds_ext_get_dictionary_buffer_id(VALUE self, VALUE buffer);
+VALUE zstds_ext_train_dictionary_buffer(VALUE self, VALUE samples, VALUE options);
+
+void zstds_ext_dictionary_exports(VALUE root_module);
+
+#endif // ZSTDS_EXT_DICTIONARY_H

data/ext/zstds_ext/error.c

@@ -0,0 +1,81 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#include "zstds_ext/error.h"
+
+#include <zstd_errors.h>
+
+#include "ruby.h"
+#include "zstds_ext/common.h"
+
+zstds_ext_result_t zstds_ext_get_error(ZSTD_ErrorCode error_code)
+{
+  switch (error_code) {
+    case ZSTD_error_memory_allocation:
+      return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+    case ZSTD_error_parameter_unsupported:
+    case ZSTD_error_parameter_outOfBound:
+    case ZSTD_error_tableLog_tooLarge:
+    case ZSTD_error_maxSymbolValue_tooLarge:
+    case ZSTD_error_maxSymbolValue_tooSmall:
+    case ZSTD_error_stage_wrong:
+    case ZSTD_error_init_missing:
+    case ZSTD_error_workSpace_tooSmall:
+    case ZSTD_error_srcSize_wrong:
+    case ZSTD_error_dstSize_tooSmall:
+    case ZSTD_error_dstBuffer_null:
+      return ZSTDS_EXT_ERROR_VALIDATE_FAILED;
+    case ZSTD_error_prefix_unknown:
+    case ZSTD_error_version_unsupported:
+    case ZSTD_error_frameParameter_unsupported:
+    case ZSTD_error_frameParameter_windowTooLarge:
+    case ZSTD_error_corruption_detected:
+    case ZSTD_error_checksum_wrong:
+      return ZSTDS_EXT_ERROR_DECOMPRESSOR_CORRUPTED_SOURCE;
+    case ZSTD_error_dictionary_corrupted:
+    case ZSTD_error_dictionary_wrong:
+    case ZSTD_error_dictionaryCreation_failed:
+      return ZSTDS_EXT_ERROR_CORRUPTED_DICTIONARY;
+    default:
+      return ZSTDS_EXT_ERROR_UNEXPECTED;
+  }
+}
+
+static inline NORETURN(void raise(const char *name, const char *description))
+{
+  VALUE module = rb_define_module(ZSTDS_EXT_MODULE_NAME);
+  VALUE error  = rb_const_get(module, rb_intern(name));
+  rb_raise(error, "%s", description);
+}
+
+void zstds_ext_raise_error(zstds_ext_result_t ext_result)
+{
+  switch (ext_result) {
+    case ZSTDS_EXT_ERROR_ALLOCATE_FAILED:
+      raise("AllocateError", "allocate error");
+    case ZSTDS_EXT_ERROR_VALIDATE_FAILED:
+      raise("ValidateError", "validate error");
+
+    case ZSTDS_EXT_ERROR_USED_AFTER_CLOSE:
+      raise("UsedAfterCloseError", "used after closed");
+    case ZSTDS_EXT_ERROR_NOT_ENOUGH_SOURCE_BUFFER:
+      raise("NotEnoughSourceBufferError", "not enough source buffer");
+    case ZSTDS_EXT_ERROR_NOT_ENOUGH_DESTINATION_BUFFER:
+      raise("NotEnoughDestinationBufferError", "not enough destination buffer");
+    case ZSTDS_EXT_ERROR_DECOMPRESSOR_CORRUPTED_SOURCE:
+      raise("DecompressorCorruptedSourceError", "decompressor received corrupted source");
+    case ZSTDS_EXT_ERROR_CORRUPTED_DICTIONARY:
+      raise("CorruptedDictionaryError", "corrupted dictionary");
+
+    case ZSTDS_EXT_ERROR_ACCESS_IO:
+      raise("AccessIOError", "failed to access IO");
+    case ZSTDS_EXT_ERROR_READ_IO:
+      raise("ReadIOError", "failed to read IO");
+    case ZSTDS_EXT_ERROR_WRITE_IO:
+      raise("WriteIOError", "failed to write IO");
+
+    default:
+      // ZSTDS_EXT_ERROR_UNEXPECTED
+      raise("UnexpectedError", "unexpected error");
+  }
+}

data/ext/zstds_ext/error.h

@@ -0,0 +1,35 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#if !defined(ZSTDS_EXT_ERROR_H)
+#define ZSTDS_EXT_ERROR_H
+
+#include <zstd_errors.h>
+
+#include "ruby.h"
+#include "zstds_ext/common.h"
+
+// Results for errors listed in "lib/zstds/error" used in c extension.
+
+enum {
+  ZSTDS_EXT_ERROR_ALLOCATE_FAILED = 1,
+  ZSTDS_EXT_ERROR_VALIDATE_FAILED,
+
+  ZSTDS_EXT_ERROR_USED_AFTER_CLOSE,
+  ZSTDS_EXT_ERROR_NOT_ENOUGH_SOURCE_BUFFER,
+  ZSTDS_EXT_ERROR_NOT_ENOUGH_DESTINATION_BUFFER,
+  ZSTDS_EXT_ERROR_DECOMPRESSOR_CORRUPTED_SOURCE,
+  ZSTDS_EXT_ERROR_CORRUPTED_DICTIONARY,
+
+  ZSTDS_EXT_ERROR_ACCESS_IO,
+  ZSTDS_EXT_ERROR_READ_IO,
+  ZSTDS_EXT_ERROR_WRITE_IO,
+
+  ZSTDS_EXT_ERROR_UNEXPECTED
+};
+
+zstds_ext_result_t zstds_ext_get_error(ZSTD_ErrorCode error_code);
+
+NORETURN(void zstds_ext_raise_error(zstds_ext_result_t ext_result));
+
+#endif // ZSTDS_EXT_ERROR_H

data/ext/zstds_ext/io.c

@@ -0,0 +1,512 @@
+// Ruby bindings for zstd library.
+// Copyright (c) 2019 AUTHORS, MIT License.
+
+#include "ruby/io.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zstd.h>
+
+#include "ruby.h"
+#include "zstds_ext/common.h"
+#include "zstds_ext/error.h"
+#include "zstds_ext/io.h"
+#include "zstds_ext/macro.h"
+#include "zstds_ext/option.h"
+
+// Additional possible results:
+enum {
+  ZSTDS_EXT_FILE_READ_FINISHED = 128
+};
+
+// -- file --
+
+static inline zstds_ext_result_t read_file(FILE* source_file, uint8_t* source_buffer, size_t* source_length_ptr, size_t source_buffer_length)
+{
+  size_t read_length = fread(source_buffer, 1, source_buffer_length, source_file);
+  if (read_length == 0 && feof(source_file)) {
+    return ZSTDS_EXT_FILE_READ_FINISHED;
+  }
+
+  if (read_length != source_buffer_length && ferror(source_file)) {
+    return ZSTDS_EXT_ERROR_READ_IO;
+  }
+
+  *source_length_ptr = read_length;
+
+  return 0;
+}
+
+static inline zstds_ext_result_t write_file(FILE* destination_file, uint8_t* destination_buffer, size_t destination_length)
+{
+  size_t written_length = fwrite(destination_buffer, 1, destination_length, destination_file);
+  if (written_length != destination_length) {
+    return ZSTDS_EXT_ERROR_WRITE_IO;
+  }
+
+  return 0;
+}
+
+// -- buffer --
+
+static inline zstds_ext_result_t create_buffers(
+  uint8_t** source_buffer_ptr, size_t source_buffer_length,
+  uint8_t** destination_buffer_ptr, size_t destination_buffer_length)
+{
+  uint8_t* source_buffer = malloc(source_buffer_length);
+  if (source_buffer == NULL) {
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+  }
+
+  uint8_t* destination_buffer = malloc(destination_buffer_length);
+  if (destination_buffer == NULL) {
+    free(source_buffer);
+    return ZSTDS_EXT_ERROR_ALLOCATE_FAILED;
+  }
+
+  *source_buffer_ptr      = source_buffer;
+  *destination_buffer_ptr = destination_buffer;
+
+  return 0;
+}
+
+// We have read some source from file into source buffer.
+// Than algorithm has read part of this source.
+// We need to move remaining source to the top of source buffer.
+// Than we can read more source from file.
+// Algorithm can use same buffer again.
+
+static inline zstds_ext_result_t read_more_source(
+  FILE*           source_file,
+  const uint8_t** source_ptr, size_t* source_length_ptr,
+  uint8_t* source_buffer, size_t source_buffer_length)
+{
+  const uint8_t* source        = *source_ptr;
+  size_t         source_length = *source_length_ptr;
+
+  if (source != source_buffer) {
+    if (source_length != 0) {
+      memmove(source_buffer, source, source_length);
+    }
+
+    // Source can be accessed even if next code will fail.
+    *source_ptr = source_buffer;
+  }
+
+  size_t remaining_source_buffer_length = source_buffer_length - source_length;
+  if (remaining_source_buffer_length == 0) {
+    // We want to read more data at once, than buffer has.
+    return ZSTDS_EXT_ERROR_NOT_ENOUGH_SOURCE_BUFFER;
+  }
+
+  uint8_t* remaining_source_buffer = source_buffer + source_length;
+  size_t   new_source_length;
+
+  zstds_ext_result_t ext_result = read_file(source_file, remaining_source_buffer, &new_source_length, remaining_source_buffer_length);
+  if (ext_result != 0) {
+    return ext_result;
+  }
+
+  *source_length_ptr = source_length + new_source_length;
+
+  return 0;
+}
+
+#define BUFFERED_READ_SOURCE(function, ...)                 \
+  do {                                                      \
+    bool is_function_called = false;                        \
+                                                            \
+    while (true) {                                          \
+      ext_result = read_more_source(                        \
+        source_file,                                        \
+        &source, &source_length,                            \
+        source_buffer, source_buffer_length);               \
+                                                            \
+      if (ext_result == ZSTDS_EXT_FILE_READ_FINISHED) {     \
+        if (source_length != 0) {                           \
+          /* ZSTD won't provide any remainder by design. */ \
+          return ZSTDS_EXT_ERROR_READ_IO;                   \
+        }                                                   \
+        break;                                              \
+      }                                                     \
+      else if (ext_result != 0) {                           \
+        return ext_result;                                  \
+      }                                                     \
+                                                            \
+      ext_result = function(__VA_ARGS__);                   \
+      if (ext_result != 0) {                                \
+        return ext_result;                                  \
+      }                                                     \
+                                                            \
+      is_function_called = true;                            \
+    }                                                       \
+                                                            \
+    if (!is_function_called) {                              \
+      /* Function should be called at least once. */        \
+      ext_result = function(__VA_ARGS__);                   \
+      if (ext_result != 0) {                                \
+        return ext_result;                                  \
+      }                                                     \
+    }                                                       \
+  } while (false);
+
+// Algorithm has written data into destination buffer.
+// We need to write this data into file.
+// Than algorithm can use same buffer again.
+
+static inline zstds_ext_result_t flush_destination_buffer(
+  FILE*    destination_file,
+  uint8_t* destination_buffer, size_t* destination_length_ptr, size_t destination_buffer_length)
+{
+  if (*destination_length_ptr == 0) {
+    // We want to write more data at once, than buffer has.
+    return ZSTDS_EXT_ERROR_NOT_ENOUGH_DESTINATION_BUFFER;
+  }
+
+  zstds_ext_result_t ext_result = write_file(destination_file, destination_buffer, *destination_length_ptr);
+  if (ext_result != 0) {
+    return ext_result;
+  }
+
+  *destination_length_ptr = 0;
+
+  return 0;
+}
+
+static inline zstds_ext_result_t write_remaining_destination(FILE* destination_file, uint8_t* destination_buffer, size_t destination_length)
+{
+  if (destination_length == 0) {
+    return 0;
+  }
+
+  return write_file(destination_file, destination_buffer, destination_length);
+}
+
+// -- utils --
+
+#define GET_FILE(target)                               \
+  Check_Type(target, T_FILE);                          \
+                                                       \
+  rb_io_t* target##_io;                                \
+  GetOpenFile(target, target##_io);                    \
+                                                       \
+  FILE* target##_file = rb_io_stdio_file(target##_io); \
+  if (target##_file == NULL) {                         \
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ACCESS_IO);  \
+  }
+
+// -- compress --
+
+static inline zstds_ext_result_t buffered_compress(
+  ZSTD_CCtx*      ctx,
+  const uint8_t** source_ptr, size_t* source_length_ptr,
+  FILE* destination_file, uint8_t* destination_buffer, size_t* destination_length_ptr, size_t destination_buffer_length)
+{
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+
+  ZSTD_inBuffer in_buffer;
+  in_buffer.src  = *source_ptr;
+  in_buffer.size = *source_length_ptr;
+  in_buffer.pos  = 0;
+
+  ZSTD_outBuffer out_buffer;
+
+  while (true) {
+    out_buffer.dst  = destination_buffer + *destination_length_ptr;
+    out_buffer.size = destination_buffer_length - *destination_length_ptr;
+    out_buffer.pos  = 0;
+
+    result = ZSTD_compressStream2(ctx, &out_buffer, &in_buffer, ZSTD_e_continue);
+    if (ZSTD_isError(result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    }
+
+    *destination_length_ptr += out_buffer.pos;
+
+    if (*destination_length_ptr == destination_buffer_length) {
+      ext_result = flush_destination_buffer(
+        destination_file,
+        destination_buffer, destination_length_ptr, destination_buffer_length);
+
+      if (ext_result != 0) {
+        return ext_result;
+      }
+
+      continue;
+    }
+
+    break;
+  }
+
+  *source_ptr += in_buffer.pos;
+  *source_length_ptr -= in_buffer.pos;
+
+  return 0;
+}
+
+static inline zstds_ext_result_t buffered_compressor_finish(
+  ZSTD_CCtx* ctx,
+  FILE* destination_file, uint8_t* destination_buffer, size_t* destination_length_ptr, size_t destination_buffer_length)
+{
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+
+  ZSTD_inBuffer in_buffer;
+  in_buffer.src  = NULL;
+  in_buffer.size = 0;
+  in_buffer.pos  = 0;
+
+  ZSTD_outBuffer out_buffer;
+
+  while (true) {
+    out_buffer.dst  = destination_buffer + *destination_length_ptr;
+    out_buffer.size = destination_buffer_length - *destination_length_ptr;
+    out_buffer.pos  = 0;
+
+    result = ZSTD_compressStream2(ctx, &out_buffer, &in_buffer, ZSTD_e_end);
+    if (ZSTD_isError(result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    }
+
+    *destination_length_ptr += out_buffer.pos;
+
+    if (result != 0) {
+      ext_result = flush_destination_buffer(
+        destination_file,
+        destination_buffer, destination_length_ptr, destination_buffer_length);
+
+      if (ext_result != 0) {
+        return ext_result;
+      }
+
+      continue;
+    }
+
+    break;
+  }
+
+  return 0;
+}
+
+static inline zstds_ext_result_t compress(
+  ZSTD_CCtx* ctx,
+  FILE* source_file, uint8_t* source_buffer, size_t source_buffer_length,
+  FILE* destination_file, uint8_t* destination_buffer, size_t destination_buffer_length)
+{
+  zstds_ext_result_t ext_result;
+
+  const uint8_t* source             = source_buffer;
+  size_t         source_length      = 0;
+  size_t         destination_length = 0;
+
+  BUFFERED_READ_SOURCE(
+    buffered_compress,
+    ctx,
+    &source, &source_length,
+    destination_file, destination_buffer, &destination_length, destination_buffer_length);
+
+  ext_result = buffered_compressor_finish(
+    ctx,
+    destination_file, destination_buffer, &destination_length, destination_buffer_length);
+
+  if (ext_result != 0) {
+    return ext_result;
+  }
+
+  return write_remaining_destination(destination_file, destination_buffer, destination_length);
+}
+
+VALUE zstds_ext_compress_io(VALUE ZSTDS_EXT_UNUSED(self), VALUE source, VALUE destination, VALUE options)
+{
+  GET_FILE(source);
+  GET_FILE(destination);
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_COMPRESSOR_OPTIONS(options);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
+
+  ZSTD_CCtx* ctx = ZSTD_createCCtx();
+  if (ctx == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  zstds_ext_result_t ext_result = zstds_ext_set_compressor_options(ctx, &compressor_options);
+  if (ext_result != 0) {
+    ZSTD_freeCCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  if (source_buffer_length == 0) {
+    source_buffer_length = ZSTD_CStreamInSize();
+  }
+  if (destination_buffer_length == 0) {
+    destination_buffer_length = ZSTD_CStreamOutSize();
+  }
+
+  uint8_t* source_buffer;
+  uint8_t* destination_buffer;
+
+  ext_result = create_buffers(
+    &source_buffer, source_buffer_length,
+    &destination_buffer, destination_buffer_length);
+
+  if (ext_result != 0) {
+    ZSTD_freeCCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  ext_result = compress(
+    ctx,
+    source_file, source_buffer, source_buffer_length,
+    destination_file, destination_buffer, destination_buffer_length);
+
+  free(source_buffer);
+  free(destination_buffer);
+  ZSTD_freeCCtx(ctx);
+
+  if (ext_result != 0) {
+    zstds_ext_raise_error(ext_result);
+  }
+
+  // Ruby itself won't flush stdio file before closing fd, flush is required.
+  fflush(destination_file);
+
+  return Qnil;
+}
+
+// -- decompress --
+
+static inline zstds_ext_result_t buffered_decompress(
+  ZSTD_DCtx*      ctx,
+  const uint8_t** source_ptr, size_t* source_length_ptr,
+  FILE* destination_file, uint8_t* destination_buffer, size_t* destination_length_ptr, size_t destination_buffer_length)
+{
+  zstds_result_t     result;
+  zstds_ext_result_t ext_result;
+
+  ZSTD_inBuffer in_buffer;
+  in_buffer.src  = *source_ptr;
+  in_buffer.size = *source_length_ptr;
+  in_buffer.pos  = 0;
+
+  ZSTD_outBuffer out_buffer;
+
+  while (true) {
+    out_buffer.dst  = destination_buffer + *destination_length_ptr;
+    out_buffer.size = destination_buffer_length - *destination_length_ptr;
+    out_buffer.pos  = 0;
+
+    result = ZSTD_decompressStream(ctx, &out_buffer, &in_buffer);
+    if (ZSTD_isError(result)) {
+      return zstds_ext_get_error(ZSTD_getErrorCode(result));
+    }
+
+    *destination_length_ptr += out_buffer.pos;
+
+    if (*destination_length_ptr == destination_buffer_length) {
+      ext_result = flush_destination_buffer(
+        destination_file,
+        destination_buffer, destination_length_ptr, destination_buffer_length);
+
+      if (ext_result != 0) {
+        return ext_result;
+      }
+
+      continue;
+    }
+
+    break;
+  }
+
+  *source_ptr += in_buffer.pos;
+  *source_length_ptr -= in_buffer.pos;
+
+  return 0;
+}
+
+static inline zstds_ext_result_t decompress(
+  ZSTD_DCtx* ctx,
+  FILE* source_file, uint8_t* source_buffer, size_t source_buffer_length,
+  FILE* destination_file, uint8_t* destination_buffer, size_t destination_buffer_length)
+{
+  zstds_ext_result_t ext_result;
+
+  const uint8_t* source             = source_buffer;
+  size_t         source_length      = 0;
+  size_t         destination_length = 0;
+
+  BUFFERED_READ_SOURCE(
+    buffered_decompress,
+    ctx,
+    &source, &source_length,
+    destination_file, destination_buffer, &destination_length, destination_buffer_length);
+
+  return write_remaining_destination(destination_file, destination_buffer, destination_length);
+}
+
+VALUE zstds_ext_decompress_io(VALUE ZSTDS_EXT_UNUSED(self), VALUE source, VALUE destination, VALUE options)
+{
+  GET_FILE(source);
+  GET_FILE(destination);
+  Check_Type(options, T_HASH);
+  ZSTDS_EXT_GET_DECOMPRESSOR_OPTIONS(options);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, source_buffer_length);
+  ZSTDS_EXT_GET_BUFFER_LENGTH_OPTION(options, destination_buffer_length);
+
+  ZSTD_DCtx* ctx = ZSTD_createDCtx();
+  if (ctx == NULL) {
+    zstds_ext_raise_error(ZSTDS_EXT_ERROR_ALLOCATE_FAILED);
+  }
+
+  zstds_ext_result_t ext_result = zstds_ext_set_decompressor_options(ctx, &decompressor_options);
+  if (ext_result != 0) {
+    ZSTD_freeDCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  if (source_buffer_length == 0) {
+    source_buffer_length = ZSTD_DStreamInSize();
+  }
+  if (destination_buffer_length == 0) {
+    destination_buffer_length = ZSTD_DStreamOutSize();
+  }
+
+  uint8_t* source_buffer;
+  uint8_t* destination_buffer;
+
+  ext_result = create_buffers(
+    &source_buffer, source_buffer_length,
+    &destination_buffer, destination_buffer_length);
+
+  if (ext_result != 0) {
+    ZSTD_freeDCtx(ctx);
+    zstds_ext_raise_error(ext_result);
+  }
+
+  ext_result = decompress(
+    ctx,
+    source_file, source_buffer, source_buffer_length,
+    destination_file, destination_buffer, destination_buffer_length);
+
+  free(source_buffer);
+  free(destination_buffer);
+  ZSTD_freeDCtx(ctx);
+
+  if (ext_result != 0) {
+    zstds_ext_raise_error(ext_result);
+  }
+
+  // Ruby itself won't flush stdio file before closing fd, flush is required.
+  fflush(destination_file);
+
+  return Qnil;
+}
+
+void zstds_ext_io_exports(VALUE root_module)
+{
+  rb_define_module_function(root_module, "_native_compress_io", RUBY_METHOD_FUNC(zstds_ext_compress_io), 3);
+  rb_define_module_function(root_module, "_native_decompress_io", RUBY_METHOD_FUNC(zstds_ext_decompress_io), 3);
+}