convolver 2.0.0 → 3.0.1

data/ext/convolver/correlate_raw.h

@@ -0,0 +1,28 @@
+// ext/convolver/correlate_raw.h
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Declarations of narray helper functions
+//
+
+#ifndef CORRELATE_RAW_H
+#define CORRELATE_RAW_H
+
+#ifndef CONVOLVER_RAW_CONFIG
+#define CONVOLVER_RAW_CONFIG
+#include <ruby.h>
+#if defined(__SSE__)
+#include <xmmintrin.h>
+#define CONVOLVER_USE_SSE 1
+#else
+#define CONVOLVER_USE_SSE 0
+#endif
+#define LARGEST_RANK 16
+#endif
+
+void correlate_raw(
+    int in_rank, const size_t *in_shape, const float *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
+    int out_rank, const size_t *out_shape, float *out_ptr );
+
+#endif

data/lib/convolver/buffer_limits.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Checks individual allocations against native size and tagged-stride limits.
+  # @private
+  class BufferLimits
+    SIZE_MAX = (1 << ([0].pack('J').bytesize * 8)) - 1
+    # Numo tags signed strides with a left shift: floor(SSIZE_MAX / 2).
+    BYTE_MAX = SIZE_MAX / 4
+    INT_MAX = (1 << (([0].pack('i').bytesize * 8) - 1)) - 1
+    INDEX_BYTES = [0].pack('J').bytesize
+
+    attr_reader :size_max, :byte_max, :int_max
+
+    def initialize(size_max: SIZE_MAX, byte_max: BYTE_MAX, int_max: INT_MAX)
+      @size_max = size_max
+      @byte_max = byte_max
+      @int_max = int_max
+    end
+
+    def buffer!(count, width, description, error: RangeError)
+      return if count <= byte_max / width
+
+      raise error, "#{description} exceeds native byte/stride limit"
+    end
+
+    def product!(shape, description, error: RangeError)
+      shape.reduce(1) do |product, size|
+        raise error, "#{description} exceeds native element limit" if size > size_max / product
+
+        product * size
+      end
+    end
+
+    NATIVE = new.freeze
+  end
+
+  private_constant :BufferLimits
+end

data/lib/convolver/buffer_preparation.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'convolver/fft_limits'
+
+module Convolver
+  # Preflights casts, extension, indexing and direct-path cache allocations.
+  # @private
+  class BufferPreparation
+    def initialize(plan, signal_shape, kernel_shape, limits: BufferLimits::NATIVE)
+      @plan = plan
+      @signal_size = limits.product!(signal_shape, 'signal size')
+      @kernel_size = limits.product!(kernel_shape, 'kernel size')
+      @limits = limits
+      @scalar = signal_shape.empty?
+    end
+
+    def basic!
+      common!
+      extension!
+      limits.buffer!(kernel_size, BufferLimits::INDEX_BYTES, 'direct kernel offset cache')
+    end
+
+    def fft!
+      common!
+      return if @scalar
+      return extension! unless plan.wrap?
+
+      limits.buffer!(signal_size, 8, 'circular signal preparation', error: FftUnavailable)
+      limits.buffer!(kernel_size, 8, 'circular kernel preparation', error: FftUnavailable)
+      limits.buffer!(kernel_size, BufferLimits::INDEX_BYTES, 'circular kernel index buffer', error: FftUnavailable)
+    end
+
+    private
+
+    attr_reader :plan, :signal_size, :kernel_size, :limits
+
+    def common!
+      limits.buffer!(plan.result_size, 4, 'result buffer')
+      limits.buffer!(signal_size, 4, 'signal conversion buffer')
+      limits.buffer!(kernel_size, 4, 'kernel conversion buffer')
+    end
+
+    def extension!
+      return if plan.valid?
+
+      limits.buffer!(plan.extended_size, 4, 'extended signal buffer')
+      return if plan.boundary == :constant
+
+      plan.extended_shape.each do |length|
+        limits.buffer!(length, BufferLimits::INDEX_BYTES, 'boundary index buffer')
+      end
+    end
+  end
+
+  private_constant :BufferPreparation
+end

data/lib/convolver/circular_fft_operation.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Calculates same-sized periodic operations through PocketFFT transforms.
+  # @private
+  class CircularFftOperation
+    def initialize(operation, signal, kernel, plan)
+      @real_axis = plan.fft_buffers.real_axis
+      @operation = operation
+      @signal = Numo::DFloat.cast(signal)
+      @kernel = fold_kernel(Numo::DFloat.cast(kernel), signal.shape, plan.anchors)
+    end
+
+    def call
+      return complex_product unless real_axis
+
+      real_product(real_axis)
+    end
+
+    def self.real_axis(shape)
+      return nil if shape.empty?
+
+      final_axis = shape.length - 1
+      return final_axis if shape[final_axis].even?
+
+      shape.each_index.select { |axis| shape[axis].even? }.max_by { |axis| shape[axis] }
+    end
+
+    private
+
+    attr_reader :operation, :signal, :kernel, :real_axis
+
+    def real_product(real_axis)
+      final_axis = signal.ndim - 1
+      prepared_signal = prepare_real_operand(signal, real_axis, final_axis)
+      prepared_kernel = prepare_real_operand(kernel, real_axis, final_axis)
+      result = real_spectrum_product(prepared_signal, prepared_kernel)
+      result = result.swapaxes(real_axis, final_axis) unless real_axis == final_axis
+      Numo::SFloat.cast(result)
+    end
+
+    def prepare_real_operand(value, real_axis, final_axis)
+      real_axis == final_axis ? value : value.swapaxes(real_axis, final_axis)
+    end
+
+    def real_spectrum_product(signal_value, kernel_value)
+      signal_spectrum = Numo::Pocketfft.rfftn(signal_value)
+      kernel_spectrum = Numo::Pocketfft.rfftn(kernel_value)
+      kernel_spectrum.inplace.conj if operation == :correlation
+      Numo::Pocketfft.irfftn(signal_spectrum.inplace * kernel_spectrum)
+    end
+
+    def complex_product
+      signal_spectrum = Numo::Pocketfft.fftn(signal)
+      kernel_spectrum = Numo::Pocketfft.fftn(kernel)
+      kernel_spectrum.inplace.conj if operation == :correlation
+      result = Numo::Pocketfft.ifftn(signal_spectrum.inplace * kernel_spectrum).real
+      Numo::SFloat.cast(result)
+    end
+
+    def fold_kernel(source, signal_shape, anchors)
+      folded = Numo::DFloat.zeros(*signal_shape)
+      source.flatten.each_with_index do |value, flat_index|
+        target = folded_target(flat_index, source.shape, signal_shape, anchors)
+        folded[*target] = folded[*target] + value
+      end
+      folded
+    end
+
+    def folded_target(flat_index, kernel_shape, signal_shape, anchors)
+      remainder = flat_index
+      Array.new(kernel_shape.length).tap do |target|
+        (kernel_shape.length - 1).downto(0) do |axis|
+          coordinate = remainder % kernel_shape[axis]
+          remainder /= kernel_shape[axis]
+          target[axis] = (coordinate - anchors[axis]) % signal_shape[axis]
+        end
+      end
+    end
+  end
+
+  private_constant :CircularFftOperation
+end

data/lib/convolver/fft_buffer_plan.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'convolver/fft_limits'
+require 'convolver/buffer_preparation'
+
+module Convolver
+  # Shares allocation-free FFT dimensions between execution and estimation.
+  # @private
+  class FftBufferPlan
+    attr_reader :transform_shape, :real_axis, :transform_size, :spectrum_size
+
+    def initialize(plan, signal_shape, kernel_shape, limits: BufferLimits::NATIVE)
+      BufferPreparation.new(plan, signal_shape, kernel_shape, limits:).fft!
+      @real_axis = plan.wrap? ? CircularFftOperation.real_axis(signal_shape) : signal_shape.length - 1
+      @transform_shape = select_shape(plan, signal_shape, kernel_shape, FftLimits.new(limits))
+      @transform_size = transform_shape.reduce(1, :*)
+      @spectrum_size = spectrum_elements
+    end
+
+    private
+
+    def select_shape(plan, signal_shape, kernel_shape, limits)
+      return [].freeze if signal_shape.empty?
+      return plan.linear_fft_shape(kernel_shape, limits:) unless plan.wrap?
+
+      limits.validate!(signal_shape, real_axis:)
+      signal_shape.dup.freeze
+    end
+
+    def spectrum_elements
+      return transform_size if transform_shape.empty? || real_axis.nil?
+
+      length = transform_shape[real_axis]
+      transform_size / length * ((length / 2) + 1)
+    end
+  end
+
+  private_constant :FftBufferPlan
+end

data/lib/convolver/fft_estimator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Estimates the selected real or complex PocketFFT implementation cost.
+  # @private
+  class FftEstimator
+    REAL_FFT_COST = { 1 => 1.0e-9, 2 => 9.0e-10, 3 => 1.45e-9 }.freeze
+    COMPLEX_FFT_COST = { 1 => 2.4e-9, 2 => 2.4e-9, 3 => 2.8e-9 }.freeze
+    FIXED_COST = 1.5e-4
+    PREPARATION_COST = 1.0e-9
+    AXIS_MOVE_COST = 1.6e-9
+    CORRELATION_COST = 2.0e-10
+
+    def initialize(operation, signal, kernel, plan)
+      @operation = operation
+      @signal = signal
+      @kernel = kernel
+      @plan = plan
+      @buffers = plan.fft_buffers
+    end
+
+    def call
+      transform_size, spectrum_size, coefficient, moved_size = dimensions
+      transform_cost = coefficient * transform_size * Math.log([transform_size, 2].max)
+      preparation_cost = PREPARATION_COST * preparation_size(transform_size)
+      axis_move_cost = AXIS_MOVE_COST * moved_size
+      correlation_cost = operation == :correlation ? CORRELATION_COST * spectrum_size : 0.0
+      FIXED_COST + transform_cost + preparation_cost + axis_move_cost + correlation_cost
+    end
+
+    private
+
+    attr_reader :operation, :signal, :kernel, :plan, :buffers
+
+    def dimensions
+      return linear_dimensions unless plan.wrap?
+
+      circular_dimensions
+    end
+
+    def circular_dimensions
+      real_axis = buffers.real_axis
+      [buffers.transform_size, buffers.spectrum_size, transform_coefficient(real_axis), moved_size(real_axis)]
+    end
+
+    def linear_dimensions
+      [buffers.transform_size, buffers.spectrum_size,
+       cost_for(REAL_FFT_COST), 0]
+    end
+
+    def preparation_size(transform_size)
+      extension_size = plan.valid? || plan.wrap? ? 0 : plan.extended_size
+      working_size = plan.wrap? ? signal.size + kernel.size : 2 * transform_size
+      extension_size + working_size
+    end
+
+    def cost_for(costs)
+      costs.fetch(signal.ndim, costs.values.last)
+    end
+
+    def transform_coefficient(real_axis)
+      cost_for(real_axis ? REAL_FFT_COST : COMPLEX_FFT_COST)
+    end
+
+    def moved_size(real_axis)
+      real_axis && real_axis != signal.ndim - 1 ? 3 * signal.size : 0
+    end
+  end
+
+  private_constant :FftEstimator
+end

data/lib/convolver/fft_limits.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'convolver/buffer_limits'
+
+# Internal allocation and arithmetic checks for the PocketFFT paths.
+module Convolver
+  # Only FFT representability failures permit automatic direct fallback.
+  # @private
+  class FftUnavailable < RangeError; end
+
+  # Bounds PocketFFT buffers and signed arithmetic without constructing plans.
+  # @private
+  class FftLimits
+    def initialize(buffers = BufferLimits::NATIVE)
+      @buffers = buffers
+    end
+
+    def validate!(shape, real_axis:)
+      total = buffers.product!(shape, 'FFT size', error: FftUnavailable)
+      validate_buffers!(shape, total, real_axis)
+      # The wrapper casts total to int before dividing it by the axis length.
+      integer!(total, buffers.int_max, 'FFT total size')
+      # Real forward computes int z_step = (z / 2 + 1) * 2.
+      integer!(shape[real_axis], buffers.int_max - 2, 'FFT real stride') if real_axis
+      validate_axes!(shape)
+    end
+
+    private
+
+    attr_reader :buffers
+
+    def validate_axes!(shape)
+      # FFTPACK twiddle counts are <= 2n complex / 3n real. Bluestein m < 4n,
+      # so each variable allocation is <= 128n bytes, including inner twiddles.
+      # The 16n int bound also protects signed intermediates at the inner m.
+      shape.each do |length|
+        integer!(length, buffers.int_max / 16, 'FFT axis arithmetic')
+        buffer!(length, 128, 'FFT axis work buffer')
+      end
+    end
+
+    def validate_buffers!(shape, total, real_axis)
+      buffer!(total, 8, 'FFT spatial buffer')
+      return buffer!(total, 16, 'FFT complex buffer') unless real_axis
+
+      spectrum = total / shape[real_axis] * ((shape[real_axis] / 2) + 1)
+      buffer!(spectrum, 16, 'FFT spectrum buffer')
+      buffer!(total, 16, 'FFT inverse staging buffer')
+      buffer!(total, 8, 'FFT inverse output buffer')
+    end
+
+    def buffer!(count, width, description)
+      buffers.buffer!(count, width, description, error: FftUnavailable)
+    end
+
+    def integer!(value, maximum, description)
+      return if value <= maximum
+
+      raise FftUnavailable, "#{description} exceeds native integer limit"
+    end
+  end
+
+  private_constant :FftUnavailable, :FftLimits
+end

data/lib/convolver/linear_fft_operation.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Calculates a valid linear operation through real PocketFFT transforms.
+  # @private
+  class LinearFftOperation
+    def initialize(operation, signal, kernel, plan)
+      @operation = operation
+      @signal = signal
+      @kernel = kernel
+      @transform_shape = plan.fft_buffers.transform_shape
+    end
+
+    def call
+      signal_padded = zero_padded(signal)
+      kernel_padded = zero_padded(kernel)
+      full_result = real_spectrum_product(signal_padded, kernel_padded)
+      Numo::SFloat.cast(full_result[*valid_ranges])
+    end
+
+    private
+
+    attr_reader :operation, :signal, :kernel, :transform_shape
+
+    def zero_padded(value)
+      Numo::DFloat.zeros(*transform_shape).tap do |padded|
+        padded[*value.shape.map { |size| 0...size }] = value
+      end
+    end
+
+    def real_spectrum_product(signal_value, kernel_value)
+      signal_spectrum = Numo::Pocketfft.rfftn(signal_value)
+      kernel_spectrum = Numo::Pocketfft.rfftn(kernel_value)
+      kernel_spectrum.inplace.conj if operation == :correlation
+      Numo::Pocketfft.irfftn(signal_spectrum.inplace * kernel_spectrum)
+    end
+
+    def valid_ranges
+      signal.shape.zip(kernel.shape).map do |signal_size, kernel_size|
+        if operation == :correlation
+          0...(signal_size - kernel_size + 1)
+        else
+          (kernel_size - 1)...signal_size
+        end
+      end
+    end
+  end
+
+  private_constant :LinearFftOperation
+end

data/lib/convolver/operation_execution.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Coordinates validation, implementation selection, and one operation family.
+  # @private
+  class OperationExecution
+    DIRECT_OPERATION_COST = { 1 => 6.8e-10, 2 => 5.9e-10, 3 => 4.5e-10 }.freeze
+    EXTENSION_COST = 1.0e-9
+    FFT_SPEEDUP_MARGIN = 0.8
+
+    METHODS = {
+      correlation: {
+        basic: :correlate_basic,
+        fft: :correlate_fft,
+        predict_basic: :predict_correlate_basic_time,
+        predict_fft: :predict_correlate_fft_time
+      },
+      convolution: {
+        basic: :convolve_basic,
+        fft: :convolve_fft,
+        predict_basic: :predict_convolve_basic_time,
+        predict_fft: :predict_convolve_fft_time
+      }
+    }.freeze
+
+    def initialize(operation, signal, kernel, mode:, boundary:, fill_value:, origin:)
+      @operation = operation
+      @signal = signal
+      @kernel = kernel
+      @mode = mode
+      @boundary = boundary
+      @fill_value = fill_value
+      @origin = origin
+    end
+
+    def automatic(receiver)
+      return invoke(receiver, :basic) if plan.extended_size < 1000
+
+      direct_time = invoke(receiver, :predict_basic)
+      return invoke(receiver, :fft) if automatic_fft_time(receiver) < FFT_SPEEDUP_MARGIN * direct_time
+
+      invoke(receiver, :basic)
+    end
+
+    def basic
+      plan.validate_basic!
+      extended_signal = plan.extend_signal(signal)
+      return Convolver.send(:correlate_basic_valid, extended_signal, kernel) if operation == :correlation
+
+      Convolver.send(:convolve_basic_valid, extended_signal, kernel)
+    end
+
+    def fft
+      plan.fft_buffers
+      return Numo::SFloat.cast(signal) * Numo::SFloat.cast(kernel) if signal.ndim.zero?
+
+      fft_operation.call
+    end
+
+    def fft_time
+      FftEstimator.new(operation, signal, kernel, plan).call
+    end
+
+    def basic_time
+      plan.validate_basic!
+      calculation_cost = direct_operation_cost * plan.result_size * kernel.size
+      extension_cost = plan.valid? ? 0.0 : EXTENSION_COST * plan.extended_size
+      calculation_cost + extension_cost
+    end
+
+    private
+
+    attr_reader :operation, :signal, :kernel, :mode, :boundary, :fill_value, :origin
+
+    def plan
+      @plan ||= OperationPlan.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+    end
+
+    def invoke(receiver, method)
+      receiver.public_send(METHODS.fetch(operation).fetch(method), signal, kernel, **public_options)
+    end
+
+    def public_options
+      options = { mode:, boundary:, origin: }
+      options[:fill_value] = fill_value unless fill_value.equal?(UNSPECIFIED_FILL)
+      options
+    end
+
+    def fft_operation
+      return CircularFftOperation.new(operation, signal, kernel, plan) if plan.wrap?
+
+      LinearFftOperation.new(operation, plan.extend_signal(signal), kernel, plan)
+    end
+
+    def automatic_fft_time(receiver)
+      invoke(receiver, :predict_fft)
+    rescue FftUnavailable
+      Float::INFINITY
+    end
+
+    def direct_operation_cost
+      DIRECT_OPERATION_COST.fetch(signal.ndim, DIRECT_OPERATION_COST.values.last)
+    end
+  end
+
+  private_constant :OperationExecution
+end

data/lib/convolver/operation_options.rb

@@ -1,17 +1,19 @@
 # frozen_string_literal: true
 
 module Convolver
-  # Validates and normalizes the options for a correlation operation.
+  # Validates and normalizes one convolution or correlation operation.
+  # @private
   class OperationOptions
+    OPERATIONS = %i[convolution correlation].freeze
     MODES = %i[valid same full].freeze
     BOUNDARIES = %i[constant nearest reflect mirror wrap].freeze
 
-    attr_reader :mode, :boundary, :fill_value, :origins, :anchors
+    attr_reader :operation, :mode, :boundary, :fill_value, :origins, :anchors
 
-    def initialize(signal, kernel, mode:, boundary:, fill_value:, origin:)
+    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
       validate_inputs!(signal, kernel)
-      validate_vocabulary!(mode, boundary)
-      assign_options(signal, kernel, mode:, boundary:, fill_value:, origin:)
+      validate_vocabulary!(operation, mode, boundary)
+      assign_options(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
       validate_combinations!(signal.shape, kernel.shape)
     end
 
@@ -26,14 +28,16 @@ module Convolver
       raise ArgumentError, "maximum supported rank is #{MAX_RANK}" if signal.ndim > MAX_RANK
     end
 
-    def validate_vocabulary!(mode, boundary)
+    def validate_vocabulary!(operation, mode, boundary)
+      raise ArgumentError, "operation must be one of #{OPERATIONS.inspect}" unless OPERATIONS.include?(operation)
       raise ArgumentError, "mode must be one of #{MODES.inspect}" unless MODES.include?(mode)
       return if BOUNDARIES.include?(boundary)
 
       raise ArgumentError, "boundary must be one of #{BOUNDARIES.inspect}"
     end
 
-    def assign_options(signal, kernel, mode:, boundary:, fill_value:, origin:)
+    def assign_options(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+      @operation = operation
       @mode = mode
       @boundary = boundary
       @fill_value_given = !fill_value.equal?(UNSPECIFIED_FILL)

data/lib/convolver/operation_plan.rb

@@ -4,25 +4,32 @@ require 'forwardable'
 require 'convolver/operation_options'
 require 'convolver/operation_shapes'
 require 'convolver/signal_extension'
+require 'convolver/fft_buffer_plan'
 
 # Internal planning and extension support for Convolver's public operations.
 module Convolver
   # Distinguishes an omitted fill_value keyword from an explicitly supplied
   # value. This lets non-constant boundaries reject even an explicit zero.
+  # @private
   UNSPECIFIED_FILL = Object.new.freeze
 
-  # Validated dimensions and boundary-extension details for one correlation.
+  # Validated dimensions and boundary-extension details for one operation.
+  # @private
   class OperationPlan
     extend Forwardable
 
-    def_delegators :options, :mode, :boundary, :fill_value, :origins, :anchors
+    def_delegators :options, :operation, :mode, :boundary, :fill_value, :origins, :anchors
     def_delegators :shapes, :padding_before, :padding_after, :result_shape,
                    :extended_shape, :result_size, :extended_size, :linear_fft_shape,
-                   :linear_fft_size
-
-    def initialize(signal, kernel, mode:, boundary:, fill_value:, origin:)
-      @options = OperationOptions.new(signal, kernel, mode:, boundary:, fill_value:, origin:)
-      @shapes = OperationShapes.new(signal.shape, kernel.shape, mode:, anchors: options.anchors)
+                   :linear_fft_size, :linear_spectrum_size
+
+    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+      @options = OperationOptions.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+      @signal_shape = signal.shape.freeze
+      @kernel_shape = kernel.shape.freeze
+      @shapes = OperationShapes.new(
+        signal.shape, kernel.shape, operation:, mode:, anchors: options.anchors
+      )
     end
 
     def valid?
@@ -39,6 +46,14 @@ module Convolver
       SignalExtension.new(shapes, boundary:, fill_value:).call(signal)
     end
 
+    def validate_basic!
+      BufferPreparation.new(self, @signal_shape, @kernel_shape).basic!
+    end
+
+    def fft_buffers
+      @fft_buffers ||= FftBufferPlan.new(self, @signal_shape, @kernel_shape)
+    end
+
     private
 
     attr_reader :options, :shapes