ndav 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 443c009b1751ab850206b12f6ae175989fef5bb3e9dfd6094d069b338262474a
-  data.tar.gz: fc5ef147dcbe89690add67f9aca9ed11a71a56ed2d801284936753066da2be2c
+  metadata.gz: '0648fef27ca47ec51a45cacb90330e5fb2de53071d4223604752c04746ad9c2d'
+  data.tar.gz: 4af1dacda9bfa1258552bc271d773136b06ea414cce61c75c194c590f1361850
 SHA512:
-  metadata.gz: 7ce73fa38eba65d0c1933cef3e787a6a770854847f7946098ca4ff7e26ba0e588ee6d5a1d79e469abb27a0c303722f69e9030943a73b5295e7a422c73dee96f5
-  data.tar.gz: 917725d70d4a3269da345d389b93562d54bec9b0911e88229a469a98d79203026ab37bbe834039aded9b1635c827d6723b5b82f23dcacd3864e9df0908dcab20
+  metadata.gz: b81a17db0cc3814963b47e590e2f3e021e86792326a98ad1f246df8f887c4f1e49aec42808c82aeceb522e28f05f7dac9d3a6eb65417fbfc9ff785cb75284723
+  data.tar.gz: 798ed5c4f2f84fa73b262af460e0054454a78d6f8a9eeacbb191f0b866c6c4e98ae72ed296af23b7a67d98d6894a988cc22dfc23484267d1773bd91ed7043a63

data/.gitignore

@@ -6,3 +6,5 @@ ext/Makefile
 *.o
 *.bundle.dSYM
 pkg/
+doc/
+.yardoc/

data/.gitlab-ci.yml

@@ -12,3 +12,15 @@ test:
       - /root/.local/share/gem/ruby
   script:
     - rake test
+
+pages:
+  stage: deploy
+  image: ruby:4.0
+  script:
+    - bundle exec rake yard
+    - mv doc public
+  artifacts:
+    paths:
+      - public
+  only:
+    - main

data/.yardopts

@@ -0,0 +1,4 @@
+--title="NDAV - N-Dimensional Array View" -M commonmarker
+-
+LICENSE.txt
+lib/ndav/ffi.rb

data/README.md

@@ -1,9 +1,183 @@
 NDAV - N-Dimensional Array View
 ===============================
 
-Wrapper for MemoryView and pointer.
+[![Gem Version](https://badge.fury.io/rb/ndav.svg)](https://badge.fury.io/rb/ndav)
+
+A thin wrapper around [MemoryView][] ("buffer protocol" for Ruby).
+
+It provides an interoperability layer for multi-dimensional arrays which can be shared between libraries.
+
+![NDAV converts library data each other](https://i.gyazo.com/905f541f15c0d57db5fc7ea3f6edaf07.png)
+
+SYNOPSIS
+--------
+
+    waveform, sample_rate = TorchAudio.load("path/to/audio.wav")
+    
+    # Convert Torch::Tensor to NDAV
+    # so that you can convert it to OrtValue,
+    # a data format for ONNX Runtime
+    input = waveform
+              .to_ndav
+              .to_ort_value
+    
+    # Make ONNX Runtime return result as OrtValue
+    outputs = OnnxRuntime::Session.new("path/to/model.onnx")
+                .run(
+                  [:output_name],
+                  {input_name: input},
+                  output_type: :ort_value
+                )
+    
+    # You may convert OrtValue to Torch::Tensor via NDAV
+    output_tensor = outputs[0]
+                      .to_ndav
+                      .to_torch_tensor # converts back to Torch::Tensor
+    
+    TorchAudio.save("path/to/output.wav", output_tensor, sample_rate)
+
+ABSTRACT
+--------
+
+NDAV acts as an interoperability layer between multi-dimensional arrays including images, audio and tensors such as [Numo::NArray][], [Torch.rb][]'s `Torch::Tensor`, [ONNX Runtime Ruby][]'s `OnnxRuntime::OrtValue`, [Red Arrow][]'s `Arrow::Array` and so on.
+
+It allows data to be shared without copying.
+
+BACKGROUND
+----------
+
+In the modern Ruby community, [Numo::NArray][] is often used for data conversion. But, data are copied when converting to and from Numo::NArray. In addition, Numo::NArray neither exports nor accepts MemoryView.
+
+[Red Arrow][] is also used and it can export MemoryView from `Arrow::Array` (not from `Arrow::Tensor`, though). It also can be converted to and from Numo::NArray using [Red Arrow Numo::NArray][]. But, in real-world usage, we need, for example, to convert data with many hops:
+
+    Torch::Tensor -> Numo::NArray -> Red Arrow -> MemoryView -> some process...
+
+It might not be difficult, but a little bit cumbersome. Additionally, Red Arrow doesn't accept MemoryView.
+
+USAGE
+-----
+
+`ndav` gem is just a base library. You need to install bridges as well. Say, assume you want to make conversions between Numo::NArray each other.
+
+    require "numo/narray"
+    require "ndav"
+    require "ndav/numo/narray"
+    
+    numo = Numo::SFloat.new(3, 5).seq # => Numo::SFloat
+    ndav = numo.to_ndav               # => NDAV
+    
+    numo = Numo::SFloat.from_ndav(ndav) # => Numo::SFloat
+    ndav = NDAV.from_numo_narray(numo)  # => NDAV
+    
+    include NDAV::Converter
+    numo = NumoNArray(ndav) # => Numo::SFloat
+    ndav = NDAV(numo)       # => NDAV
+
+For `Torch::Tensor` and `OnnxRuntime::OrtValue`, you can do the same operation, therefore you may convert them to each other, like this:
+
+    numo
+      .to_ndav
+      .to_torch_tensor
+      .then {|torch_tensor| some_process(torch_tensor)}
+      .to_ndav
+      .to_ort_value
+      .then {|ort_value|
+        OnnxRuntime::Session.new("model.onnx")
+          .run(
+            [:output],
+            {input: ort_value},
+            output_type: :ort_value
+          )[0]
+      }
+      .to_ndav
+      .to_torch_tensor
+      .then {|torch_tensor| TorchAudio.save(torch_tensor, sample_rate)}
+
+### Working With MemoryView ###
+
+NDAV can be initialized *directly* from libraries which export [MemoryView][] such as [Red Arrow][], without any bridge library:
+
+    arrow = Arrow::Int16Array.new([1, 2, 3])
+    ndav = NDAV.new(arrow)
+
+On the other hand, it also exports MemoryView. You can pass NDAV arrays *directly* to methods which accept MemoryView such as [whispercpp][] without bridge libraries:
+
+    waveform, sample_rate = TorchAudio.load("path/to/audio.wav")
+    samples = waveform.to_ndav
+    whisper.full(params, samples)
+
+### Notice On Memory Sharing ###
+
+Notice that NDAV is just a memory view and libraries share a memory address. If you change source data destructively, it affects converted data.
+
+Additionally, you potentially encounter odd data corruption or segmentation fault. These might be bugs in bridge libraries such as [ndav-numo-narray][]. As a user, you don't need to worry about this kind of memory management, but it's worth knowing such situations may occur.
+
+INSTALLATION
+------------
+
+    % gem install ndav
+
+or,
+
+    % bundle add ndav
+
+But, you need bridges for real-world use. See each bridge's documentation for individual requirements.
+
+BRIDGES
+-------
+
+There are some bridges using NDAV:
+
+* {file:ndav/ffi ndav/ffi}: [`FFI::MemoryPointer`][FFI Pointer], [`FFI::Pointer`][FFI Pointer] <-> `NDAV`
+* [ndav-numo-narray][]: [`Numo::NArray`][Numo::NArray] <-> `NDAV`
+* [ndav-ort_value][]: [`OnnxRuntime::OrtValue`][ONNX Runtime Ruby] <-> `NDAV`
+* [ndav-torch-tensor][]: [`Torch::Tensor`][Torch.rb] <-> `NDAV`
+* [onnxruntime-torch-tensor][]: [`Torch::Tensor`][Torch.rb] <-> [`OnnxRuntime::OrtValue`][ONNX Runtime Ruby] via `NDAV`
+
+APPLICATIONS AND LIBRARIES WHICH USE NDAV
+-----------------------------------------
+
+* [GTCRN][] - An audio speech enhancement (noise reduction) library. It uses [TorchAudio][] for loading and preprocessing audio before passing it to ONNX Runtime, then performs post-processing on the result and writes it back to a file.
+* [Itak][] - An audio processing tool for podcasters. After reducing noise with the GTCRN mentioned above, it uses the VAD function of [whispercpp][] to remove silent periods. Although whispercpp does not accept existing tensor libraries, it does accept MemoryView, so it can be passed via NDAV.
+
+CREATING BRIDGES
+----------------
+
+Refer to existing bridge implementations listed above to create your bridge.
+
+The points are:
+
+* Implement `FromNDAV#from_ndav`, `ToNDAV#to_ndav`, {NDAV.register register} them, and `NDAV.from_your_data` and `NDAV#to_your_data` are automatically derived
+* When initializing NDAV object from your object, use `lifetime` keyword argument for {NDAV#initialize} effectively to prevent Ruby from GCing your object, which would lead to a dangling pointer
+* When initializing your object from NDAV object, keep NDAV object alive to prevent Ruby from GCing NDAV object, which would lead to a dangling pointer, [ndav-numo-narray][], for instance, embeds the NDAV object in an instance variable
+
+An advantage of NDAV over raw MemoryView is that you can write bridges in pure Ruby in most cases. It helps prototyping and experimentation. As an exception, I had to write C code for ndav-numo-narray because Numo::NArray only provides methods that access data by copying and does not directly expose its raw data pointer to Ruby API. However, in even such case, pure Ruby bridge remains a viable option for prototyping and experiments where a single initial copy is acceptable.
+
+If you are a library author, I want you to consider making your library work with [MemoryView][] instead of creating an NDAV bridge.
+
+FUTURE
+------
+
+If [MemoryView][] gets popular enough in the Ruby ecosystem, this library will end its role and will no longer be needed. I hope such future.
 
 LICENSE
 -------
 
-BSD-2-Clause license. See LICENSE.txt file.
+BSD-2-Clause license. See {file:LICENSE.txt} file.
+
+[MemoryView]: https://docs.ruby-lang.org/en/master/contributing/memory_view_md.html
+[Numo::NArray]: https://ruby-numo.github.io/narray/
+[Torch.rb]: https://github.com/ankane/torch.rb
+[ONNX Runtime Ruby]: https://github.com/ankane/onnxruntime-ruby
+[Red Arrow]: https://github.com/apache/arrow/tree/main/ruby
+[Red Arrow Numo::NArray]: https://github.com/red-data-tools/red-arrow-numo-narray
+[ndav/ffi]: https://gitlab.com/KitaitiMakoto/ndav/-/blob/main/lib/ndav/ffi.rb
+[FFI Pointer]: https://github.com/ffi/ffi/wiki/Pointers
+[whispercpp]: https://github.com/ggml-org/whisper.cpp/tree/master/bindings/ruby
+[ndav-numo-narray]: https://gitlab.com/KitaitiMakoto/ndav-numo-narray
+[ndav-ort_value]: https://gitlab.com/KitaitiMakoto/ndav-ort_value
+[ndav-torch-tensor]: https://gitlab.com/KitaitiMakoto/ndav-torch-tensor
+[onnxruntime-torch-tensor]: https://gitlab.com/KitaitiMakoto/onnxruntime-torch-tensor
+[GTCRN]: https://gitlab.com/KitaitiMakoto/gtcrn
+[TorchAudio]: https://github.com/ankane/torchaudio-ruby
+[Itak]: https://gitlab.com/KitaitiMakoto/itak

data/Rakefile

@@ -1,18 +1,20 @@
 require "rake/clean"
 require "rake/testtask"
 require "rubygems/tasks"
+require "yard"
 
 DL_NAME = "ndav".ext(RbConfig::CONFIG["DLEXT"])
 DL_BUILD_PATH = File.join("ext", DL_NAME)
 DL_PATH = File.join("lib", DL_NAME)
 
 SRC = FileList["ext/**.{h,c,rb}"]
+CLEAN.include SRC.select {|src| src.end_with?(".o", ".so", ".bundle", ".dll") }
 
 task default: :test
 
 Rake::TestTask.new test: DL_PATH
-tasks = Gem::Tasks.new
-gemspec = tasks.build.gem.project.gemspec
+Gem::Tasks.new
+YARD::Rake::YardocTask.new
 
 file DL_PATH => DL_BUILD_PATH do |t|
   copy t.source, t.name

data/ext/ndav.c

@@ -1,22 +1,34 @@
-#include "ndav.h"
+#include <ruby.h>
+#include <ruby/memory_view.h>
+
+static ID id_ndav_validated_descriptor;
+static VALUE sym_data;
+static VALUE sym_shape;
+static VALUE sym_strides;
+static VALUE sym_format;
+static VALUE sym_byte_size;
+static VALUE sym_readonly_p;
+static VALUE sym_sub_offsets;
+static VALUE sym_ndim;
+static VALUE sym_item_size;
+static VALUE sym_row_major_contiguous_p;
+static VALUE sym_column_major_contiguous_p;
+
+typedef struct {
+  VALUE descriptor;
+} private_data_t;
 
-ID id_to_s;
-
-typedef struct fill_size_array_args_t {
-  VALUE src;
-  ssize_t *dest;
-  ssize_t size;
-} fill_size_array_args_t;
-
-static VALUE
-fill_size_array(VALUE rb_args)
+static bool
+fill_size_array(VALUE src, ssize_t *dest, ssize_t size)
 {
-  struct fill_size_array_args_t *args = (fill_size_array_args_t *)rb_args;
-  for (ssize_t i = 0; i < args->size; i++) {
-    VALUE val = rb_ary_entry(args->src, i);
-    args->dest[i] = NUM2SSIZET(val);
+  for (ssize_t i = 0; i < size; i++) {
+    VALUE val = rb_ary_entry(src, i);
+    if (!RB_INTEGER_TYPE_P(val)) {
+      return false;
+    }
+    dest[i] = NUM2SSIZET(val);
   }
-  return Qnil;
+  return true;
 }
 
 static bool
@@ -25,24 +37,38 @@ ndav_get_memory_view(const VALUE ndav, rb_memory_view_t *view, int flags)
   bool row_major_requested = (flags & RUBY_MEMORY_VIEW_ROW_MAJOR) == RUBY_MEMORY_VIEW_ROW_MAJOR;
   bool column_major_requested = (flags & RUBY_MEMORY_VIEW_COLUMN_MAJOR) == RUBY_MEMORY_VIEW_COLUMN_MAJOR;
   bool indirect_requested = (flags & RUBY_MEMORY_VIEW_INDIRECT) == RUBY_MEMORY_VIEW_INDIRECT;
-  if ((column_major_requested && !row_major_requested) || indirect_requested) {
+  VALUE desc = rb_funcall(ndav, id_ndav_validated_descriptor, 1, INT2NUM(flags));
+  if (!RB_TYPE_P(desc, T_HASH)) {
+    rb_warn("descriptor not Hash");
+    return false;
+  }
+  bool row_major_contiguous = rb_hash_aref(desc, sym_row_major_contiguous_p);
+  bool column_major_contiguous = rb_hash_aref(desc, sym_column_major_contiguous_p);
+  if (row_major_requested && column_major_requested) { // row-major OR column-major requested
+    if (!row_major_contiguous && !column_major_contiguous) {
+      return false;
+    }
+  }
+  // Currently, non-contiguous array not supported
+  else if (row_major_requested && !row_major_contiguous) {
+    return false;
+  }
+  // Currently, non-contiguous array not supported
+  else if (column_major_requested && !column_major_contiguous) {
     return false;
   }
   bool writable_requested = (flags & RUBY_MEMORY_VIEW_WRITABLE) == RUBY_MEMORY_VIEW_WRITABLE;
   // TODO: Handle flags
 
-  VALUE readonly = rb_iv_get(ndav, "@readonly");
-  if (NIL_P(readonly)) {
-    if (writable_requested) {
-      rb_warn("not writable");
-      return false;
-    }
-    view->readonly = true;
-  } else {
-    view->readonly = RTEST(readonly);
+  VALUE readonly_v = rb_hash_aref(desc, sym_readonly_p);
+  bool readonly = NIL_P(readonly_v) || RTEST(readonly_v);
+  if (writable_requested && readonly) {
+    rb_warn("not writable");
+    return false;
   }
+  view->readonly = readonly;
   view->obj = ndav;
-  VALUE val = rb_funcall(ndav, id_to_s, 0);
+  VALUE val = rb_hash_aref(desc, sym_data);
   char *data = StringValuePtr(val);
   view->data = data;
   // TODO: Commonalize
@@ -50,59 +76,91 @@ ndav_get_memory_view(const VALUE ndav, rb_memory_view_t *view, int flags)
   // * Use TypedData?
   //   * If so, calling attr readers at Ruby layer has performance disadvantage.
   //   * Freezing instance vars in #initialize and then embed them to TypedData may be a solution
-  VALUE item_size = rb_iv_get(ndav, "@item_size");
+  VALUE item_size = rb_hash_aref(desc, sym_item_size);
   view->item_size = NUM2SSIZET(item_size);
-  VALUE byte_size = rb_iv_get(ndav, "@byte_size");
+  VALUE byte_size = rb_hash_aref(desc, sym_byte_size);
   view->byte_size = NUM2SSIZET(byte_size);
-  VALUE ndim = rb_iv_get(ndav, "@ndim");
+  VALUE ndim = rb_hash_aref(desc, sym_ndim);
   view->ndim = NUM2SSIZET(ndim);
-  VALUE format = rb_iv_get(ndav, "@format");
+  VALUE format = rb_hash_aref(desc, sym_format);
   view->format = StringValueCStr(format);
 
-  int state;
-
   // TODO: Commonalize
-  VALUE shape = rb_iv_get(ndav, "@shape");
+  VALUE shape = rb_hash_aref(desc, sym_shape);
   if (!RB_TYPE_P(shape, T_ARRAY)) {
     rb_warn("@shape is not an array");
     return false;
   }
-  ssize_t *view_shape = ALLOC_N(ssize_t, view->ndim);
-  fill_size_array_args_t shape_args = {
-    shape,
-    view_shape,
-    view->ndim,
-  };
-  rb_protect(fill_size_array, (VALUE)&shape_args, &state);
-  if (state) {
-    xfree(view_shape);
-    rb_jump_tag(state);
+  ssize_t *view_shape = (ssize_t *)malloc(sizeof(ssize_t) * view->ndim);
+  if (!view_shape) {
+    return false;
+  }
+  if (!fill_size_array(shape, view_shape, view->ndim)) {
+    free((void *)view_shape);
     return false;
   }
   view->shape = view_shape;
 
-  VALUE strides = rb_iv_get(ndav, "@strides");
+  VALUE strides = rb_hash_aref(desc, sym_strides);
   if (!RB_TYPE_P(strides, T_ARRAY)) {
     rb_warn("@strides is not an array");
-    xfree((void *)view->shape);
+    free((void *)view->shape);
     return false;
   }
-  ssize_t *view_strides = ALLOC_N(ssize_t, view->ndim);
-  fill_size_array_args_t strides_args = {
-    strides,
-    view_strides,
-    view->ndim
-  };
-  rb_protect(fill_size_array, (VALUE)&strides_args, &state);
-  if (state) {
-    xfree((void *)view_shape);
-    xfree((void *)view_strides);
-    rb_jump_tag(state);
+  ssize_t *view_strides = (ssize_t *)malloc(sizeof(ssize_t) * view->ndim);
+  if (!view_strides) {
+    free((void *)view->shape);
+    return false;
+  }
+  if (!fill_size_array(strides, view_strides, view->ndim)) {
+    free((void *)view->shape);
+    free((void *)view_strides);
     return false;
   }
   view->strides = view_strides;
 
-  view->sub_offsets = NULL;
+  VALUE sub_offsets = rb_hash_aref(desc, sym_sub_offsets);
+  if (NIL_P(sub_offsets)) {
+    if (indirect_requested) {
+      rb_warn("indirect requested but sub_offsets is NULL");
+      free((void *)view->shape);
+      free((void *)view->strides);
+      return false;
+    }
+    view->sub_offsets = NULL;
+  } else if (!RB_TYPE_P(sub_offsets, T_ARRAY)) {
+    rb_warn("sub_offsets is not an array");
+    free((void *)view->shape);
+    free((void *)view->strides);
+    return false;
+  } else {
+    ssize_t *view_sub_offsets = (ssize_t *)malloc(sizeof(ssize_t) * view->ndim);
+    if (!view_sub_offsets) {
+      free((void *)view->shape);
+      free((void *)view->strides);
+      return false;
+    }
+    if (!fill_size_array(sub_offsets, view_sub_offsets, view->ndim)) {
+      free((void *)view->shape);
+      free((void *)view->strides);
+      free((void *)view_sub_offsets);
+      return false;
+    }
+    view->sub_offsets = view_sub_offsets;
+  }
+
+  private_data_t *private_data = malloc(sizeof(private_data_t));
+  if (!private_data) {
+    free((void *)view->shape);
+    free((void *)view->strides);
+    free((void *)view->sub_offsets);
+    rb_warn("failed to alloc private_data");
+    return false;
+  }
+  private_data->descriptor = Qnil;
+  rb_gc_register_address(&private_data->descriptor);
+  private_data->descriptor = desc;
+  view->private_data = private_data;
 
   return true;
 }
@@ -111,13 +169,23 @@ static bool
 ndav_release_memory_view(const VALUE ndav, rb_memory_view_t *view)
 {
   if (view->shape) {
-    xfree((void *)view->shape);
+    free((void *)view->shape);
     view->shape = NULL;
   }
   if (view->strides) {
-    xfree((void *)view->strides);
+    free((void *)view->strides);
     view->strides = NULL;
   }
+  if (view->sub_offsets) {
+    free((void *)view->sub_offsets);
+    view->sub_offsets = NULL;
+  }
+  if (view->private_data) {
+    private_data_t *private_data = (private_data_t *)view->private_data;
+    rb_gc_unregister_address(&private_data->descriptor);
+    free(private_data);
+    view->private_data = NULL;
+  }
 
   return true;
 }
@@ -125,8 +193,8 @@ ndav_release_memory_view(const VALUE ndav, rb_memory_view_t *view)
 static bool
 ndav_memory_view_available_p(const VALUE obj)
 {
-  VALUE fmv = rb_iv_get(obj, "@fmv");
-  return !NIL_P(fmv);
+  VALUE descriptor = rb_funcall(obj, id_ndav_validated_descriptor, 0);
+  return RTEST(descriptor);
 }
 
 const struct rb_memory_view_entry ndav_view_entry = {
@@ -135,11 +203,34 @@ const struct rb_memory_view_entry ndav_view_entry = {
   ndav_memory_view_available_p
 };
 
+static VALUE
+ndav_memory_viewable_s_register(VALUE mod, VALUE klass)
+{
+  if (!rb_memory_view_register(klass, &ndav_view_entry)) {
+    // error message output in rb_memory_view_register()
+    rb_raise(rb_eArgError, "");
+  }
+
+  return Qtrue;
+}
+
 void
 Init_ndav(void)
 {
-  id_to_s = rb_intern("to_s");
+  id_ndav_validated_descriptor = rb_intern("ndav_validated_descriptor");
+  sym_data = ID2SYM(rb_intern("data"));
+  sym_shape = ID2SYM(rb_intern("shape"));
+  sym_strides = ID2SYM(rb_intern("strides"));
+  sym_format = ID2SYM(rb_intern("format"));
+  sym_byte_size = ID2SYM(rb_intern("byte_size"));
+  sym_readonly_p = ID2SYM(rb_intern("readonly?"));
+  sym_sub_offsets = ID2SYM(rb_intern("sub_offsets"));
+  sym_ndim = ID2SYM(rb_intern("ndim"));
+  sym_item_size = ID2SYM(rb_intern("item_size"));
+  sym_row_major_contiguous_p = ID2SYM(rb_intern("row_major_contiguous?"));
+  sym_column_major_contiguous_p = ID2SYM(rb_intern("column_major_contiguous?"));
 
   VALUE cNDAV = rb_define_class("NDAV", rb_cObject);
-  rb_memory_view_register(cNDAV, &ndav_view_entry);
+  VALUE mMemoryViewable = rb_define_module_under(cNDAV, "MemoryViewable");
+  rb_define_singleton_method(mMemoryViewable, "register", ndav_memory_viewable_s_register, 1);
 }

data/lib/ndav/converter.rb

@@ -4,6 +4,10 @@ class NDAV
     alias from_fiddle_memory_view new
     alias from_fiddle_pointer new
 
+    def from_string(str, *, **)
+      new(Fiddle::Pointer[str], *, **)
+    end
+
     def register(cls, mdl, name:)
       if mdl.const_defined?(:FromNDAV)
         cls.extend mdl::FromNDAV
@@ -13,6 +17,19 @@ class NDAV
         end
       end
 
+      if mdl.const_defined?(:MemoryViewable)
+        cls.include ::NDAV::MemoryViewable
+        cls.include mdl::MemoryViewable
+
+        unless mdl.const_defined?(:ToNDAV)
+          mdl.const_set(:ToNDAV, Module.new {
+            def to_ndav(lifetime: self, **)
+              ::NDAV.new(self, lifetime:, **)
+            end
+          })
+        end
+      end
+
       if mdl.const_defined?(:ToNDAV)
         cls.include mdl::ToNDAV
 

data/lib/ndav/ffi.rb

@@ -26,6 +26,25 @@ class NDAV
         end
       end
 
+      module MemoryViewable
+        def ndav_descriptor(**)
+          shape = [size / type_size]
+          format = TYPE_SIZE_TO_FORMAT[type_size]
+          item_size = ITEM_SIZES[format]
+          byte_size = shape.reduce(item_size, :*)
+          ptr = ::Fiddle::Pointer.new(address, byte_size)
+
+          {
+            data: Fiddle::MemoryView.new(ptr).to_s,
+            shape:,
+            strides: ::NDAV.default_strides(shape:, item_size:),
+            format:,
+            byte_size:,
+            readonly?: true
+          }
+        end
+      end
+
       module Converter
         def FFIMemoryPointer(array, *, **)
           if ::FFI::MemoryPointer === array

data/lib/ndav/flags.rb

@@ -0,0 +1,26 @@
+class NDAV
+  module Flags
+    FLAGS = {}
+    FLAGS[:simple]            = 0
+    FLAGS[:writable]          = (1<<0)
+    FLAGS[:format]            = (1<<1)
+    FLAGS[:multi_dimensional] = (1<<2)
+    FLAGS[:strides]           = (1<<3) | FLAGS[:multi_dimensional]
+    FLAGS[:row_major]         = (1<<4) | FLAGS[:strides]
+    FLAGS[:column_major]      = (1<<5) | FLAGS[:strides]
+    FLAGS[:any_contiguous]    = FLAGS[:row_major] | FLAGS[:column_major]
+    FLAGS[:indirect]          = (1<<6) | FLAGS[:strides]
+
+    module_function
+
+    def decode(flags)
+      FLAGS.each_with_object({}) {|(name, value), state|
+        if name == :simple
+          state[name] = flags == 0
+        else
+          state[name] = flags & value == value
+        end
+      }
+    end
+  end
+end

data/lib/ndav/memory_viewable.rb

@@ -0,0 +1,47 @@
+class NDAV
+  module MemoryViewable
+    class << self
+      def included(base)
+        ::NDAV::MemoryViewable.register base
+      end
+    end
+
+    private
+
+    # Overwrite this method or define each ndav_xxx method
+    def ndav_descriptor(simple: true, writable: false, format: false, multi_dimensional: false, strides: false, row_major: false, column_major: false, any_contiguous: false, indirect: false)
+      {
+        data: ndav_data,
+        shape: ndav_shape,
+        strides: ndav_strides,
+        format: ndav_format,
+        byte_size: ndav_byte_size,
+        readonly?: ndav_readonly?,
+        sub_offsets: ndav_sub_offsets
+      }
+    end
+
+    def ndav_validated_descriptor(flags = ::NDAV::Flags::FLAGS[:simple])
+      desc = ndav_descriptor(**NDAV::Flags.decode(flags)).to_h
+      unless desc.kind_of? Hash
+        warn "descriptor not Hash"
+        return false
+      end
+      [:data, :shape, :strides, :format, :byte_size, :readonly?].each do |key|
+        unless desc.key? key
+          warn ":#{key} not in descriptor"
+          return false
+        end
+      end
+      desc.merge(
+        ndim: desc[:shape].length,
+        item_size: NDAV::ITEM_SIZES[desc[:format]],
+        row_major_contiguous?: ::NDAV.row_major_contiguous?(**desc),
+        column_major_contiguous?: ::NDAV.column_major_contiguous?(**desc)
+      )
+    rescue => err
+      warn err
+      return false
+    end
+  end
+end

data/lib/ndav.rb

@@ -1,88 +1,143 @@
 require "fiddle"
 require "ndav.so"
+require "ndav/flags"
 require "ndav/converter"
+require "ndav/memory_viewable"
 
 # TODO: Manage flags of internal data object
 class NDAV
+  include MemoryViewable
+
   # TODO: Want to use return value of rb_memory_view_parse_item_format
   ITEM_SIZES = ["c", "C", "s", "S", "l", "L", "q", "Q", "f", "d", "s!", "S!", "n", "v", "i", "i!", "I", "I!", "l!", "L!", "N", "V", "e", "g", "q!", "Q!", "E", "G"].collect {|format|
     [format, [0].pack(format).bytesize]
   }.to_h
 
-  attr_reader :ndim, :shape, :format, :item_size, :byte_size, :strides, :lifetime
+  attr_reader :ndim, :shape, :format, :item_size, :byte_size, :strides, :to_s, :lifetime
+
+  class << self
+    def default_strides(shape:, item_size:, row_major: true)
+      each_shape = row_major ? shape.reverse_each : shape.each
+      each_shape.reduce([[], item_size]) {|(strides, stride), s|
+        if row_major
+          strides.unshift stride
+        else
+          strides << stride
+        end
+        [strides, stride * s]
+      }[0]
+    end
+
+    # Algorithm stolen from memory_view.c
+    def row_major_contiguous?(format:, shape:, strides:, **)
+      ndim = shape.length
+      n = ITEM_SIZES[format]
+      return strides[0] == n if ndim == 1
+
+      (ndim - 1).downto 0 do |i|
+        return false unless strides[i] == n
+
+        n *= shape[i]
+      end
+
+      true
+    end
+
+    # Algorithm stolen from memory_view.c
+    def column_major_contiguous?(format:, shape:, strides:, **)
+      ndim = shape.length
+      n = ITEM_SIZES[format]
+      return strides[0] == n if ndim == 1
+
+      0.upto(ndim - 1) do |i|
+        return false unless strides[i] == n
+
+        n *= shape[i]
+      end
+
+      true
+    end
+  end
 
   # TODO: More options to trick for the case of MemoryView export is wrong but developer don't fix it
-  def initialize(input, shape: nil, format: nil, lifetime: nil)
-    @fmv = input.kind_of?(::Fiddle::MemoryView) ? input : ::Fiddle::MemoryView.new(input)
-    @lifetime = [@fmv.obj, lifetime].compact
-    @lifetime = nil if @lifetime.empty?
+  def initialize(obj, shape: nil, strides: nil, format: nil, lifetime: nil)
+    fmv = obj.kind_of?(::Fiddle::MemoryView) ? obj : ::Fiddle::MemoryView.new(obj)
+    # Fiddle::MemoryView#to_s is zero-copy
+    # Fiddle::MemoryView#to_s embeds obj into the String, so we can prevent GC from collecting it
+    @to_s = fmv.to_s
+    @lifetime = lifetime
 
-    @format = format || @fmv.format
+    @format = format || fmv.format
     raise ArgumentError, %Q|unsupported format: #{@format.to_s.dump}, currently supported: #{ITEM_SIZES.keys}| unless ITEM_SIZES.key?(@format)
 
     item_size_by_format = ITEM_SIZES[@format]
 
-    @byte_size = @fmv.byte_size
-    @item_size = input.kind_of?(::Fiddle::Pointer) ? item_size_by_format : @fmv.item_size || item_size_by_format
-    @shape = shape || @fmv.shape
+    @byte_size = fmv.byte_size
+    @item_size = obj.kind_of?(::Fiddle::Pointer) ? item_size_by_format : fmv.item_size || item_size_by_format
+    @shape = shape || fmv.shape
     case [@shape, @byte_size]
     in [nil, nil]
-      raise ArgumentError, "either shape or byte_size must be present in input or arguments"
-    in [*, nil]
+      raise ArgumentError, "either shape or byte_size must be present in obj or arguments"
+    in [_, nil]
       @byte_size = @shape.reduce(@item_size, :*)
-    in [nil, *]
+    in [nil, _]
       n, r = @byte_size.divmod(@item_size)
       raise ArgumentError, "byte_size must be n-times of item_size" unless r.zero?
       @shape = [n]
     else
       # noop
     end
-    @strides = @fmv.strides || @shape.reverse_each.reduce([[], @item_size]) {|(strides, stride), s|
-      strides.unshift(stride)
-      [strides, stride * s]
-    }[0]
     @ndim = @shape.size
-    @readonly = @fmv.readonly?
+    @readonly = fmv.readonly?
+    @strides = strides || fmv.strides || self.class.default_strides(shape: @shape, item_size: @item_size)
+    @sub_offsets = fmv.sub_offsets
+    @row_major_contiguous = self.class.row_major_contiguous?(format: @format, shape: @shape, strides: @strides)
+    @column_major_contiguous = self.class.column_major_contiguous?(format: @format, shape: @shape, strides: @strides)
 
-    validate
+    validate fmv
   end
 
   def readonly?
     @readonly
   end
 
-  # Zero-copy
-  def to_s
-    # MemoryView#to_s is zero-copy
-    # Assign to @data to keep reference to the string to prevent GC
-    @data ||= @fmv.to_s
+  def row_major_contiguous?
+    @row_major_contiguous
+  end
+
+  def column_major_contiguous?
+    @column_major_contiguous
   end
 
   def to_ptr
-    ::Fiddle::Pointer[to_s]
+    ::Fiddle::Pointer[@to_s]
   end
   alias to_fiddle_pointer to_ptr
 
   private
 
-  def validate
+  def ndav_descriptor(**)
+    {
+      data: @to_s,
+      shape: @shape,
+      strides: @strides,
+      format: @format,
+      byte_size: @byte_size,
+      readonly?: @readonly,
+      sub_offsets: @sub_offsets
+    }
+  end
+
+  def validate(fmv)
     unless @item_size == ITEM_SIZES[@format]
       raise ArgumentError, "item_size does not match format"
     end
 
-    if @fmv.sub_offsets
-      raise ArgumentError, "suboffsets not supported"
-    end
-
-    stride = @item_size
-    @shape.reverse_each.with_index do |s, i|
-      unless @strides[@ndim - 1 - i] == stride
-        raise ArgumentError, "only row-major contiguous strides is supported"
+    if @row_major_contiguous || @column_major_contiguous
+      byte_size = @shape.reduce(@item_size, :*)
+      unless byte_size == @byte_size
+        raise ArgumentError, "strides not match byte_size"
       end
-      stride *= s
-    end
-    unless stride == @byte_size
-      raise ArgumentError, "strides not match byte_size"
     end
   end
 end

data/ndav.gemspec

@@ -1,10 +1,11 @@
 Gem::Specification.new do |s|
   s.name = "ndav"
-  s.version = "0.0.3"
+  s.version = "0.0.5"
   s.summary = "N-Dimensional Array View"
   s.authors = ["Kitaiti Makoto"]
   s.licenses = ["BSD-2-Clause"]
-  s.homepage = "https://gitlab.com/KitaitiMakoto/ndav"
+  s.homepage = "https://kitaitimakoto.gitlab.io/ndav"
+  s.metadata["source_code_uri"] = "https://gitlab.com/KitaitiMakoto/ndav"
 
   s.files = Dir.chdir(__dir__) {`git ls-files -z`.split("\x0")}
   s.extensions = ["ext/extconf.rb"]
@@ -20,4 +21,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rubygems-requirements-system"
   s.add_development_dependency "red-arrow"
   s.add_development_dependency "ffi"
+  s.add_development_dependency "yard"
+  s.add_development_dependency "commonmarker"
 end

data/test/test_ffi.rb

@@ -48,6 +48,14 @@ class TestFFI < TestBase
     assert_equal ndav.to_ptr.to_i, ptr.address
   end
 
+  def test_memory_view_from_ffi_memory_pointer
+    ptr = ::FFI::MemoryPointer.new(:int16, 3)
+    ptr.write_array_of_int16([1, 2, 3])
+    fmv = Fiddle::MemoryView.new(ptr)
+
+    assert_int_array fmv
+  end
+
   class TestConverter < self
     def test_ndav_from_ffi_memory_pointer
       ptr = ::FFI::MemoryPointer.new(:int16, 3)

data/test/test_ndav.rb

@@ -28,4 +28,19 @@ class TestNDAV < TestBase
     assert_equal ptr, ndav.to_fiddle_pointer
     assert_int_array ndav
   end
+
+  def test_string
+    str = [1, 2, 3].pack("s*")
+    ndav = NDAV.from_string(str, format: "s")
+
+    assert_int_array ndav
+  end
+
+  def test_column_major
+    str = [1, 2, 3,
+           4, 5, 6].pack("s*")
+    ndav = NDAV.from_string(str, format: "s", shape: [2, 3], strides: [2, 4])
+
+    assert_true ndav.column_major_contiguous?
+  end
 end

data/test/test_ractor.rb

@@ -0,0 +1,79 @@
+require_relative "helper"
+
+class TestRactor < TestBase
+  ractor
+
+  def test_ractor
+    ractor = Ractor.new {
+      Ractor.receive
+    }
+    ndav = NDAV.new(Arrow::Int16Array.new([1, 2, 3]))
+    ractor << ndav
+    result = ractor.value
+
+    assert_not_same ndav, result
+    assert_int_array result
+  end
+
+  def test_ractor_from_string
+    ractor = Ractor.new {
+      Ractor.receive
+    }
+    ndav = NDAV.from_string([1, 2, 3].pack("s*"), format: "s")
+    ractor << ndav
+    result = ractor.value
+
+    assert_int_array result
+  end
+
+  def test_ractor_share
+    ractor = Ractor.new {
+      Ractor.receive
+    }
+    ndav = NDAV.new(Arrow::Int16Array.new([1, 2, 3]))
+    Ractor.make_shareable(ndav)
+    ractor << ndav
+    result = ractor.value
+
+    assert_same ndav, result
+    assert_int_array result
+  end
+
+  def test_port
+    ractor = Ractor.new {
+      ndav, port = Ractor.receive
+      port << ndav
+    }
+    ndav = NDAV.new(Arrow::Int16Array.new([1, 2, 3]))
+    port = Ractor::Port.new
+    ractor << [ndav, port]
+    result = port.receive
+
+    assert_not_same ndav, result
+    assert_int_array result
+  end
+
+  def test_port_share
+    ractor = Ractor.new {
+      ndav, port = Ractor.receive
+      port << ndav
+    }
+    ndav = NDAV.new(Arrow::Int16Array.new([1, 2, 3]))
+    Ractor.make_shareable(ndav)
+    port = Ractor::Port.new
+    ractor << [ndav, port]
+    result = port.receive
+
+    assert_same ndav, result
+    assert_int_array result
+  end
+
+  def test_ractor_not_initializable
+    ractor = Ractor.new {
+      NDAV.new(Arrow::Int16Array.new([1, 2, 3]))
+    }
+    assert_raise Ractor::RemoteError do
+      ractor.value
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ndav
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Kitaiti Makoto
@@ -149,6 +149,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: commonmarker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 executables: []
 extensions:
 - ext/extconf.rb
@@ -156,16 +184,18 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitlab-ci.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - ext/extconf.rb
 - ext/ndav.c
-- ext/ndav.h
 - lib/ndav.rb
 - lib/ndav/converter.rb
 - lib/ndav/ffi.rb
+- lib/ndav/flags.rb
+- lib/ndav/memory_viewable.rb
 - ndav.gemspec
 - test/helper.rb
 - test/test_converter.rb
@@ -173,10 +203,12 @@ files:
 - test/test_ffi.rb
 - test/test_ndav.rb
 - test/test_package.rb
-homepage: https://gitlab.com/KitaitiMakoto/ndav
+- test/test_ractor.rb
+homepage: https://kitaitimakoto.gitlab.io/ndav
 licenses:
 - BSD-2-Clause
-metadata: {}
+metadata:
+  source_code_uri: https://gitlab.com/KitaitiMakoto/ndav
 rdoc_options: []
 require_paths:
 - lib
@@ -191,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: N-Dimensional Array View
 test_files: []

data/ext/ndav.h

@@ -1,2 +0,0 @@
-#include <ruby.h>
-#include <ruby/memory_view.h>