ruby-hdf5 0.0.2 → 0.0.3

data/lib/hdf5/file.rb

@@ -1,8 +1,11 @@
 module HDF5
   class File
+    include Hierarchy
+
     H5F_ACC_RDONLY = 0x0000
     H5F_ACC_RDWR = 0x0001
     H5F_ACC_TRUNC = 0x0002
+    H5F_ACC_EXCL = 0x0004
 
     class << self
       def create(filename, flags = H5F_ACC_TRUNC)
@@ -17,8 +20,9 @@ module HDF5
         end
       end
 
-      def open(filename, mode = H5F_ACC_RDONLY)
-        file = from_id(HDF5::FFI.H5Fopen(filename, mode, HDF5::DEFAULT_PROPERTY_LIST), filename, mode)
+      def open(filename, mode = 'r')
+        file_id, flags = open_file(filename, mode)
+        file = from_id(file_id, filename, flags)
         return file unless block_given?
 
         begin
@@ -30,6 +34,29 @@ module HDF5
 
       private
 
+      def open_file(filename, mode)
+        return [HDF5::FFI.H5Fopen(filename, mode, HDF5::DEFAULT_PROPERTY_LIST), mode] if mode.is_a?(Integer)
+
+        case mode
+        when 'r'
+          [HDF5::FFI.H5Fopen(filename, H5F_ACC_RDONLY, HDF5::DEFAULT_PROPERTY_LIST), H5F_ACC_RDONLY]
+        when 'r+'
+          [HDF5::FFI.H5Fopen(filename, H5F_ACC_RDWR, HDF5::DEFAULT_PROPERTY_LIST), H5F_ACC_RDWR]
+        when 'w'
+          [HDF5::FFI.H5Fcreate(filename, H5F_ACC_TRUNC, HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST),
+           H5F_ACC_TRUNC]
+        when 'x'
+          [HDF5::FFI.H5Fcreate(filename, H5F_ACC_EXCL, HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST),
+           H5F_ACC_EXCL]
+        when 'a'
+          file_id = HDF5::FFI.H5Fcreate(filename, H5F_ACC_EXCL, HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST)
+          file_id = HDF5::FFI.H5Fopen(filename, H5F_ACC_RDWR, HDF5::DEFAULT_PROPERTY_LIST) if file_id < 0
+          [file_id, H5F_ACC_RDWR]
+        else
+          raise ArgumentError, "Unsupported file mode: #{mode.inspect}"
+        end
+      end
+
       def from_id(file_id, filename, mode)
         file = allocate
         file.send(:initialize_from_id, file_id, filename, mode)
@@ -42,21 +69,36 @@ module HDF5
     end
 
     def close
-      return if @file_id.nil?
+      return if @file_id.nil? || @context.closed?
 
-      HDF5::FFI.H5Fclose(@file_id)
+      @context.close_file
       @file_id = nil
     end
 
+    def closed?
+      @file_id.nil? || @context.closed?
+    end
+
+    def flush
+      ensure_open!
+      status = HDF5::FFI.H5Fflush(@file_id, :H5F_SCOPE_GLOBAL)
+      raise HDF5::Error, 'Failed to flush file' if status < 0
+
+      self
+    end
+
     def create_group(name, &block)
-      Group.create(@file_id, name, &block)
+      ensure_open!
+      Group.create(@file_id, name, @context, &block)
     end
 
-    def create_dataset(name, data, &block)
-      Dataset.create(@file_id, name, data, &block)
+    def create_dataset(name, data = nil, **options, &block)
+      ensure_open!
+      Dataset.create(@file_id, name, data, context: @context, **options, &block)
     end
 
     def list_entries
+      ensure_open!
       list = []
       callback = ::FFI::Function.new(:int, %i[int64_t string pointer pointer]) do |_, name, _, _|
         list << name
@@ -74,27 +116,41 @@ module HDF5
     end
 
     def [](name)
+      ensure_open!
       if group?(name)
-        Group.open(@file_id, name)
+        Group.open(@file_id, name, @context)
       elsif dataset?(name)
-        Dataset.open(@file_id, name)
+        Dataset.open(@file_id, name, context: @context)
       else
         raise HDF5::Error, "Group or dataset not found: #{name}"
       end
     end
 
     def attrs
-      @attrs ||= AttributeManager.new(@file_id)
+      ensure_open!
+      @attrs ||= AttributeManager.new(@file_id, @context)
     end
 
     private
 
+    def hdf5_id
+      ensure_open!
+      @file_id
+    end
+
     def initialize_from_id(file_id, filename, mode)
       raise HDF5::Error, "Failed to open file: #{filename}" if file_id < 0
 
       @filename = filename
       @mode = mode
       @file_id = file_id
+      @context = FileContext.new(file_id)
+    end
+
+    def ensure_open!
+      raise ClosedError, 'HDF5 file is closed' if @file_id.nil?
+
+      @context.ensure_open!(@file_id)
     end
 
     def group?(name)
@@ -122,5 +178,7 @@ module HDF5
              end
       info[:type] == :H5O_TYPE_DATASET
     end
+
+    prepend FileContext.guard(:flush, :create_group, :create_dataset, :list_entries, :[], :attrs)
   end
 end

data/lib/hdf5/file_context.rb

@@ -0,0 +1,70 @@
+module HDF5
+  class FileContext
+    CLOSE_FUNCTIONS = {
+      file: :H5Fclose,
+      group: :H5Gclose,
+      dataset: :H5Dclose
+    }.freeze
+
+    def initialize(file_id)
+      @file_id = file_id
+      @handles = { file_id => :file }
+      @closed = false
+    end
+
+    def register(id, type)
+      raise ClosedError, 'HDF5 file is closed' if closed?
+
+      @handles[id] = type
+      id
+    end
+
+    def close(id)
+      synchronize do
+        type = @handles[id]
+        return if type.nil?
+
+        status = HDF5::FFI.public_send(CLOSE_FUNCTIONS.fetch(type), id)
+        raise HDF5::Error, "Failed to close HDF5 #{type}" if status < 0
+
+        @handles.delete(id)
+        @closed = true if type == :file
+        nil
+      end
+    end
+
+    def close_file
+      synchronize do
+        return if closed?
+
+        children = @handles.keys.reject { |id| id == @file_id }
+        children.reverse_each { |id| close(id) }
+        close(@file_id)
+      end
+    end
+
+    def ensure_open!(id = nil)
+      raise ClosedError, 'HDF5 file is closed' if closed?
+      raise ClosedError, 'HDF5 object is closed' if id && !@handles.key?(id)
+    end
+
+    def closed?
+      @closed
+    end
+
+    def synchronize(&block)
+      HDF5::FFI::CALL_LOCK.synchronize(&block)
+    end
+
+    def self.guard(*methods)
+      Module.new do
+        methods.each do |method|
+          define_method(method) do |*args, **kwargs, &block|
+            lock = @context || HDF5::FFI::CALL_LOCK
+            lock.synchronize { super(*args, **kwargs, &block) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hdf5/group.rb

@@ -1,10 +1,12 @@
 module HDF5
   class Group
+    include Hierarchy
+
     class << self
-      def create(parent_id, name)
+      def create(parent_id, name, context = nil)
         group = from_id(
           HDF5::FFI.H5Gcreate2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST,
-                               HDF5::DEFAULT_PROPERTY_LIST), name
+                               HDF5::DEFAULT_PROPERTY_LIST), name, context
         )
         return group unless block_given?
 
@@ -15,8 +17,8 @@ module HDF5
         end
       end
 
-      def open(parent_id, name)
-        group = from_id(HDF5::FFI.H5Gopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name)
+      def open(parent_id, name, context = nil)
+        group = from_id(HDF5::FFI.H5Gopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, context)
         return group unless block_given?
 
         begin
@@ -28,33 +30,40 @@ module HDF5
 
       private
 
-      def from_id(group_id, name)
+      def from_id(group_id, name, context)
         group = allocate
-        group.send(:initialize_from_id, group_id, name)
+        group.send(:initialize_from_id, group_id, name, context)
         group
       end
     end
 
     def initialize(file_id, name)
-      initialize_from_id(HDF5::FFI.H5Gopen2(file_id, name, HDF5::DEFAULT_PROPERTY_LIST), name)
+      initialize_from_id(HDF5::FFI.H5Gopen2(file_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, nil)
     end
 
     def close
       return if @group_id.nil?
 
-      HDF5::FFI.H5Gclose(@group_id)
+      @context ? @context.close(@group_id) : HDF5::FFI.H5Gclose(@group_id)
       @group_id = nil
     end
 
+    def closed?
+      @group_id.nil? || (@context && @context.closed?)
+    end
+
     def create_group(name, &block)
-      self.class.create(@group_id, name, &block)
+      ensure_open!
+      self.class.create(@group_id, name, @context, &block)
     end
 
-    def create_dataset(name, data, &block)
-      Dataset.create(@group_id, name, data, &block)
+    def create_dataset(name, data = nil, **options, &block)
+      ensure_open!
+      Dataset.create(@group_id, name, data, context: @context, **options, &block)
     end
 
     def list_entries
+      ensure_open!
       entries = []
       callback = ::FFI::Function.new(:int, %i[int64_t string pointer pointer]) do |_, name, _, _|
         entries << name
@@ -76,26 +85,41 @@ module HDF5
     end
 
     def [](name)
+      ensure_open!
       if group?(name)
-        self.class.open(@group_id, name)
+        self.class.open(@group_id, name, @context)
       elsif dataset?(name)
-        Dataset.open(@group_id, name)
+        Dataset.open(@group_id, name, context: @context)
       else
         raise HDF5::Error, "Group or dataset not found: #{name}"
       end
     end
 
     def attrs
-      @attrs ||= AttributeManager.new(@group_id)
+      ensure_open!
+      @attrs ||= AttributeManager.new(@group_id, @context)
     end
 
     private
 
-    def initialize_from_id(group_id, name)
+    def hdf5_id
+      ensure_open!
+      @group_id
+    end
+
+    def initialize_from_id(group_id, name, context)
       raise HDF5::Error, "Failed to open group: #{name}" if group_id < 0
 
       @group_id = group_id
       @name = name
+      @context = context
+      @context.register(group_id, :group) if @context
+    end
+
+    def ensure_open!
+      raise ClosedError, 'HDF5 group is closed' if @group_id.nil?
+
+      @context&.ensure_open!(@group_id)
     end
 
     def group?(name)
@@ -123,5 +147,7 @@ module HDF5
              end
       info[:type] == :H5O_TYPE_DATASET
     end
+
+    prepend FileContext.guard(:create_group, :create_dataset, :list_entries, :list_datasets, :[], :attrs)
   end
 end

data/lib/hdf5/hierarchy.rb

@@ -0,0 +1,107 @@
+module HDF5
+  module Hierarchy
+    def keys
+      list_entries
+    end
+
+    def each_key(&block)
+      return enum_for(__method__) unless block
+
+      keys.each(&block)
+      self
+    end
+
+    def require_group(path)
+      path.split('/').reject(&:empty?).inject(self) do |parent, name|
+        if parent.key?(name)
+          child = parent[name]
+          raise HDF5::Error, "Existing object is not a group: #{name}" unless child.is_a?(Group)
+
+          child
+        else
+          parent.create_group(name)
+        end
+      end
+    end
+
+    def open_dataset(name)
+      object = self[name]
+      raise HDF5::Error, "Object is not a dataset: #{name}" unless object.is_a?(Dataset)
+      return object unless block_given?
+
+      begin
+        yield object
+      ensure
+        object.close
+      end
+    end
+
+    def key?(name)
+      exists = HDF5::FFI.H5Lexists(hdf5_id, name, HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to check link existence: #{name}" if exists.negative?
+
+      exists.positive?
+    end
+
+    def delete(name)
+      status = HDF5::FFI.H5Ldelete(hdf5_id, name, HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to delete link: #{name}" if status < 0
+
+      self
+    end
+
+    def create_hard_link(target, name)
+      status = HDF5::FFI.H5Lcreate_hard(hdf5_id, target, hdf5_id, name, HDF5::DEFAULT_PROPERTY_LIST,
+                                        HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to create hard link: #{name}" if status < 0
+
+      self
+    end
+
+    def create_soft_link(target, name)
+      status = HDF5::FFI.H5Lcreate_soft(target, hdf5_id, name, HDF5::DEFAULT_PROPERTY_LIST,
+                                        HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to create soft link: #{name}" if status < 0
+
+      self
+    end
+
+    def link_info(name)
+      info = HDF5::FFI::MiV == 10 ? HDF5::FFI::H5LInfoT.new : HDF5::FFI::H5LInfo2T.new
+      status = if HDF5::FFI::MiV == 10
+                 HDF5::FFI.H5Lget_info(hdf5_id, name, info, HDF5::DEFAULT_PROPERTY_LIST)
+               else
+                 HDF5::FFI.H5Lget_info2(hdf5_id, name, info, HDF5::DEFAULT_PROPERTY_LIST)
+               end
+      raise HDF5::Error, "Failed to get link information: #{name}" if status < 0
+
+      type = info[:type]
+      return { type: :hard } if type == :H5L_TYPE_HARD
+
+      size = info[:u][:val_size]
+      buffer = ::FFI::MemoryPointer.new(:char, size)
+      status = HDF5::FFI.H5Lget_val(hdf5_id, name, buffer, size, HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to get link value: #{name}" if status < 0
+
+      raw = buffer.read_bytes(size)
+      return { type: :soft, target: raw.delete_suffix("\0") } if type == :H5L_TYPE_SOFT
+
+      if type == :H5L_TYPE_EXTERNAL
+        filename, path = raw.byteslice(1..).split("\0", 2)
+        return { type: :external, filename:, path: path&.delete_suffix("\0") }
+      end
+
+      raise UnsupportedTypeError, "Unsupported link type: #{type}"
+    end
+
+    def move(source, destination)
+      status = HDF5::FFI.H5Lmove(hdf5_id, source, hdf5_id, destination, HDF5::DEFAULT_PROPERTY_LIST,
+                                 HDF5::DEFAULT_PROPERTY_LIST)
+      raise HDF5::Error, "Failed to move link: #{source}" if status < 0
+
+      self
+    end
+
+    prepend FileContext.guard(*instance_methods(false))
+  end
+end

data/lib/hdf5/selection.rb

@@ -0,0 +1,81 @@
+module HDF5
+  class Selection
+    attr_reader :start, :stride, :count, :result_shape
+
+    def self.normalize(selection, shape)
+      return new(Array.new(shape.length, true), shape) if selection.nil?
+
+      selectors = selection.is_a?(Array) ? selection.dup : [selection]
+      raise IndexError, 'Too many indices for dataset' if selectors.length > shape.length
+
+      selectors.concat(Array.new(shape.length - selectors.length, true))
+      new(selectors, shape)
+    end
+
+    def initialize(selectors, shape)
+      @start = []
+      @stride = []
+      @count = []
+      @result_shape = []
+
+      selectors.zip(shape).each do |selector, dimension|
+        normalize_axis(selector, dimension)
+      end
+    end
+
+    def scalar?
+      result_shape.empty?
+    end
+
+    def size
+      count.inject(1, :*)
+    end
+
+    private
+
+    def normalize_axis(selector, dimension)
+      case selector
+      when true
+        @start << 0
+        @stride << 1
+        @count << dimension
+        @result_shape << dimension
+      when Integer
+        index = selector.negative? ? dimension + selector : selector
+        raise IndexError, "Index #{selector} is outside dimension of size #{dimension}" unless index.between?(0,
+                                                                                                              dimension - 1)
+
+        @start << index
+        @stride << 1
+        @count << 1
+      when Range
+        first, last = normalize_range(selector, dimension)
+        @start << first
+        @stride << 1
+        @count << last - first
+        @result_shape << last - first
+      when HDF5::Slice
+        first, last = normalize_range(selector.range, dimension)
+        count = (last - first).fdiv(selector.step).ceil
+        @start << first
+        @stride << selector.step
+        @count << count
+        @result_shape << count
+      else
+        raise IndexError, "Unsupported index: #{selector.inspect}"
+      end
+    end
+
+    def normalize_range(range, dimension)
+      first = range.begin.nil? ? 0 : range.begin
+      last = range.end.nil? ? dimension : range.end
+      first += dimension if first.negative?
+      last += dimension if last.negative?
+      last += 1 unless range.exclude_end? || range.end.nil?
+
+      first = [[first, 0].max, dimension].min
+      last = [[last, 0].max, dimension].min
+      [first, [last, first].max]
+    end
+  end
+end

data/lib/hdf5/string_codec.rb

@@ -0,0 +1,93 @@
+module HDF5
+  module StringCodec
+    module_function
+
+    UTF8 = 1
+    VARIABLE_SIZE = (1 << (::FFI.type_size(:size_t) * 8)) - 1
+
+    def datatype_id
+      type_id = HDF5::FFI.H5Tcopy(HDF5::FFI.H5T_C_S1)
+      raise HDF5::Error, 'Failed to create string datatype' if type_id < 0
+
+      check(HDF5::FFI.H5Tset_size(type_id, VARIABLE_SIZE), 'set variable string size')
+      check(HDF5::FFI.H5Tset_cset(type_id, UTF8), 'set UTF-8 string encoding')
+      type_id
+    rescue StandardError
+      HDF5::FFI.H5Tclose(type_id) if type_id && type_id >= 0
+      raise
+    end
+
+    def buffer_for(value)
+      buffer_for_values([normalize(value)])
+    end
+
+    def buffer_for_values(values)
+      pointers = values.map { |value| ::FFI::MemoryPointer.from_string(normalize(value)) }
+      buffer = ::FFI::MemoryPointer.new(:pointer, pointers.length)
+      buffer.write_array_of_pointer(pointers)
+      [buffer, pointers]
+    end
+
+    def read(buffer)
+      pointer = buffer.read_pointer
+      pointer.null? ? nil : pointer.read_string.force_encoding(Encoding::UTF_8)
+    end
+
+    def read_values(buffer, count, shape)
+      values = buffer.read_array_of_pointer(count).map do |pointer|
+        pointer.null? ? nil : pointer.read_string.force_encoding(Encoding::UTF_8)
+      end
+      return values.first if shape.empty?
+
+      Numo::RObject.cast(values).reshape(*shape)
+    end
+
+    def string_data?(value)
+      return true if value.is_a?(String)
+
+      values = if value.is_a?(Numo::RObject)
+                 value.to_a.flatten
+               elsif value.is_a?(Array)
+                 value.flatten
+               else
+                 []
+               end
+      !values.empty? && values.all? { |item| item.is_a?(String) }
+    end
+
+    def normalize_data(value)
+      return [[normalize(value)], []] if value.is_a?(String)
+      return [value.to_a.flatten.map { |item| normalize(item) }, value.shape] if value.is_a?(Numo::RObject)
+
+      shape = array_shape(value)
+      [value.flatten.map { |item| normalize(item) }, shape]
+    end
+
+    def array_shape(value)
+      return [] unless value.is_a?(Array)
+      raise HDF5::Error, 'String data must not be empty' if value.empty?
+
+      child_shapes = value.map { |item| array_shape(item) }
+      raise HDF5::ShapeError, 'String data must be rectangular' unless child_shapes.uniq.length == 1
+
+      [value.length, *child_shapes.first]
+    end
+
+    def variable?(type_id)
+      HDF5::FFI.H5Tis_variable_str(type_id).positive?
+    end
+
+    def normalize(value)
+      raise HDF5::Error, 'String dataset data must be a String' unless value.is_a?(String)
+      raise HDF5::Error, 'Variable-length strings cannot contain NUL bytes' if value.include?("\0")
+
+      value.encode(Encoding::UTF_8)
+    rescue EncodingError
+      raise HDF5::Error, 'String dataset data must be valid UTF-8'
+    end
+
+    def check(status, operation)
+      raise HDF5::Error, "Failed to #{operation}" if status < 0
+    end
+  end
+end

data/lib/hdf5/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HDF5
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/hdf5.rb

@@ -1,15 +1,40 @@
 # frozen_string_literal: true
 
 require 'ffi'
+require 'English'
+require 'numo/narray/alt'
 require_relative 'hdf5/version'
 
 module HDF5
   class Error < StandardError; end
+  class ClosedError < Error; end
+  class UnsupportedTypeError < Error; end
+  class UnsupportedFeatureError < Error; end
+  class ShapeError < Error; end
+  class ConversionError < Error; end
+  class NativeError < Error; end
+
+  class Empty
+    attr_reader :dtype
+
+    def initialize(dtype)
+      @dtype = dtype.is_a?(DType) ? dtype : DType.for_symbol(dtype)
+      freeze
+    end
+  end
+  Slice = Struct.new(:range, :step, keyword_init: true)
   DEFAULT_PROPERTY_LIST = 0
 
   class << self
     attr_accessor :lib_path
 
+    def slice(range, step:)
+      raise ArgumentError, 'slice step must be a positive integer' unless step.is_a?(Integer) && step.positive?
+      raise ArgumentError, 'slice range must be a Range' unless range.is_a?(Range)
+
+      Slice.new(range:, step:)
+    end
+
     def search_hdf5lib
       name = "libhdf5.#{FFI::Platform::LIBSUFFIX}"
       env_path = ENV['HDF5_LIB_PATH']
@@ -37,6 +62,12 @@ end
 
 require_relative 'hdf5/ffi'
 
+require_relative 'hdf5/file_context'
+require_relative 'hdf5/dtype'
+require_relative 'hdf5/data_helpers'
+require_relative 'hdf5/selection'
+require_relative 'hdf5/hierarchy'
+require_relative 'hdf5/string_codec'
 require_relative 'hdf5/file'
 require_relative 'hdf5/group'
 require_relative 'hdf5/dataset'