ruby-hdf5 0.0.3 → 0.0.4

data/lib/hdf5/string_codec.rb

@@ -21,21 +21,25 @@ module HDF5
       buffer_for_values([normalize(value)])
     end
 
-    def buffer_for_values(values)
-      pointers = values.map { |value| ::FFI::MemoryPointer.from_string(normalize(value)) }
+    def buffer_for_values(values, encoding: Encoding::UTF_8)
+      pointers = values.map { |value| ::FFI::MemoryPointer.from_string(normalize(value, encoding:)) }
       buffer = ::FFI::MemoryPointer.new(:pointer, pointers.length)
       buffer.write_array_of_pointer(pointers)
       [buffer, pointers]
     end
 
-    def read(buffer)
-      pointer = buffer.read_pointer
-      pointer.null? ? nil : pointer.read_string.force_encoding(Encoding::UTF_8)
+    def read(buffer, encoding: Encoding::UTF_8)
+      read_values(buffer, 1, [], encoding:)
     end
 
-    def read_values(buffer, count, shape)
+    def read_values(buffer, count, shape, encoding: Encoding::UTF_8)
       values = buffer.read_array_of_pointer(count).map do |pointer|
-        pointer.null? ? nil : pointer.read_string.force_encoding(Encoding::UTF_8)
+        next nil if pointer.null?
+
+        value = pointer.read_string.force_encoding(encoding)
+        raise HDF5::Error, "String data is not valid #{encoding.name}" unless value.valid_encoding?
+
+        value
       end
       return values.first if shape.empty?
 
@@ -55,35 +59,44 @@ module HDF5
       !values.empty? && values.all? { |item| item.is_a?(String) }
     end
 
-    def normalize_data(value)
-      return [[normalize(value)], []] if value.is_a?(String)
-      return [value.to_a.flatten.map { |item| normalize(item) }, value.shape] if value.is_a?(Numo::RObject)
+    def normalize_data(value, encoding: Encoding::UTF_8)
+      return [[normalize(value, encoding:)], []] if value.is_a?(String)
+      return [value.to_a.flatten.map { |item| normalize(item, encoding:) }, value.shape] if value.is_a?(Numo::RObject)
 
       shape = array_shape(value)
-      [value.flatten.map { |item| normalize(item) }, shape]
+      [value.flatten.map { |item| normalize(item, encoding:) }, shape]
     end
 
     def array_shape(value)
       return [] unless value.is_a?(Array)
       raise HDF5::Error, 'String data must not be empty' if value.empty?
 
-      child_shapes = value.map { |item| array_shape(item) }
-      raise HDF5::ShapeError, 'String data must be rectangular' unless child_shapes.uniq.length == 1
-
-      [value.length, *child_shapes.first]
+      DataHelpers.array_shape(value)
     end
 
     def variable?(type_id)
       HDF5::FFI.H5Tis_variable_str(type_id).positive?
     end
 
-    def normalize(value)
+    def encoding_for(type_id)
+      case HDF5::FFI.H5Tget_cset(type_id)
+      when :H5T_CSET_ASCII then Encoding::US_ASCII
+      when :H5T_CSET_UTF8 then Encoding::UTF_8
+      else raise UnsupportedTypeError, 'Unsupported HDF5 string encoding'
+      end
+    end
+
+    def normalize(value, encoding: Encoding::UTF_8)
       raise HDF5::Error, 'String dataset data must be a String' unless value.is_a?(String)
+      raise HDF5::Error, 'String data has an invalid encoding' unless value.valid_encoding?
       raise HDF5::Error, 'Variable-length strings cannot contain NUL bytes' if value.include?("\0")
 
-      value.encode(Encoding::UTF_8)
+      result = value.encode(encoding)
+      raise HDF5::Error, "String data is not valid #{encoding.name}" unless result.valid_encoding?
+
+      result
     rescue EncodingError
-      raise HDF5::Error, 'String dataset data must be valid UTF-8'
+      raise HDF5::Error, "String data must be representable as #{encoding.name}"
     end
 
     def check(status, operation)

data/lib/hdf5/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HDF5
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-hdf5
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - kojix2