convolver 2.0.0 → 4.0.0

data/ext/convolver/convolver.c

@@ -2,10 +2,33 @@
 #include <numo/narray.h>
 #include <numo/intern.h>
 
+#include "correlate_raw.h"
 #include "convolve_raw.h"
+#include "fft_shape.h"
 
 static VALUE mConvolver;
 
+/* Contiguous views share their backing storage but start at a byte offset. */
+static const void *logical_pointer_for_read(VALUE value) {
+  return (const void *)(na_get_pointer_for_read(value) + na_get_offset(value));
+}
+
+static VALUE contiguous_float(VALUE value) {
+  narray_t *narray;
+
+  GetNArray(value, narray);
+  /* A scalar has no strides; Numo's contiguous check requires an axis. */
+  if (narray->ndim > 0 && !RTEST(na_check_contiguous(value))) {
+    value = na_copy(value);
+  }
+  return value;
+}
+
+enum convolver_operation {
+  CONVOLVER_CORRELATION,
+  CONVOLVER_CONVOLUTION
+};
+
 static void copy_shape(int rank, const size_t *source, size_t *target) {
   int i;
 
@@ -14,15 +37,7 @@ static void copy_shape(int rank, const size_t *source, size_t *target) {
   }
 }
 
-/*
- * Calculates a valid cross-correlation using the direct native implementation.
- *
- * @overload convolve_basic_valid(signal, kernel)
- *   @param signal [Numo::NArray] input values
- *   @param kernel [Numo::NArray] correlation kernel
- *   @return [Numo::SFloat] valid cross-correlation result
- */
-static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
+static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_operation operation) {
   volatile VALUE signal_value;
   volatile VALUE kernel_value;
   volatile VALUE result_value;
@@ -35,20 +50,17 @@ static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kern
   size_t result_shape[LARGEST_RANK];
   size_t numo_result_shape[LARGEST_RANK];
 
-  (void)self;
-
   if (!rb_obj_is_kind_of(signal, numo_cNArray) || !rb_obj_is_kind_of(kernel, numo_cNArray)) {
     rb_raise(rb_eArgError, "signal and kernel must be Numo::NArray values");
   }
 
-  signal_value = rb_funcall(numo_cSFloat, rb_intern("cast"), 1, signal);
-  kernel_value = rb_funcall(numo_cSFloat, rb_intern("cast"), 1, kernel);
-  if (!RTEST(na_check_contiguous(signal_value))) {
-    signal_value = na_copy(signal_value);
-  }
-  if (!RTEST(na_check_contiguous(kernel_value))) {
-    kernel_value = na_copy(kernel_value);
+  VALUE dtype = CLASS_OF(signal);
+  if ((dtype != numo_cSFloat && dtype != numo_cDFloat) || CLASS_OF(kernel) != dtype) {
+    rb_raise(rb_eArgError, "native inputs must have matching SFloat or DFloat dtypes");
   }
+
+  signal_value = contiguous_float(signal);
+  kernel_value = contiguous_float(kernel);
   GetNArray(signal_value, signal_narray);
   GetNArray(kernel_value, kernel_narray);
 
@@ -75,17 +87,71 @@ static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kern
     numo_result_shape[rank - i - 1] = (size_t)result_shape[i];
   }
 
-  result_value = nary_new(numo_cSFloat, rank, numo_result_shape);
-  convolve_raw(
-    rank, signal_shape, (float *)na_get_pointer_for_read(signal_value),
-    rank, kernel_shape, (float *)na_get_pointer_for_read(kernel_value),
-    rank, result_shape, (float *)na_get_pointer_for_write(result_value)
-  );
+  result_value = nary_new(dtype, rank, numo_result_shape);
+  if (dtype == numo_cSFloat) {
+    if (operation == CONVOLVER_CORRELATION) {
+      correlate_raw(
+        rank, signal_shape, logical_pointer_for_read(signal_value),
+        rank, kernel_shape, logical_pointer_for_read(kernel_value),
+        rank, result_shape, (float *)na_get_pointer_for_write(result_value)
+      );
+    } else {
+      convolve_raw(
+        rank, signal_shape, logical_pointer_for_read(signal_value),
+        rank, kernel_shape, logical_pointer_for_read(kernel_value),
+        rank, result_shape, (float *)na_get_pointer_for_write(result_value)
+      );
+    }
+  } else {
+    if (operation == CONVOLVER_CORRELATION) {
+      correlate_raw_double(
+        rank, signal_shape, logical_pointer_for_read(signal_value),
+        rank, kernel_shape, logical_pointer_for_read(kernel_value),
+        rank, result_shape, (double *)na_get_pointer_for_write(result_value)
+      );
+    } else {
+      convolve_raw_double(
+        rank, signal_shape, logical_pointer_for_read(signal_value),
+        rank, kernel_shape, logical_pointer_for_read(kernel_value),
+        rank, result_shape, (double *)na_get_pointer_for_write(result_value)
+      );
+    }
+  }
 
   return result_value;
 }
 
+/*
+ * Calculates a valid cross-correlation using the direct native implementation.
+ *
+ * @private
+ * @overload correlate_basic_valid(signal, kernel)
+ *   @param signal [Numo::NArray] input values
+ *   @param kernel [Numo::NArray] correlation kernel
+ *   @return [Numo::SFloat, Numo::DFloat] valid cross-correlation result
+ */
+static VALUE convolver_correlate_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
+  (void)self;
+  return convolver_basic_valid(signal, kernel, CONVOLVER_CORRELATION);
+}
+
+/*
+ * Calculates a valid mathematical convolution using the direct native implementation.
+ *
+ * @private
+ * @overload convolve_basic_valid(signal, kernel)
+ *   @param signal [Numo::NArray] input values
+ *   @param kernel [Numo::NArray] convolution kernel
+ *   @return [Numo::SFloat, Numo::DFloat] valid mathematical convolution result
+ */
+static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
+  (void)self;
+  return convolver_basic_valid(signal, kernel, CONVOLVER_CONVOLUTION);
+}
+
 void Init_convolver(void) {
   mConvolver = rb_define_module("Convolver");
+  convolver_init_fft_shape(mConvolver);
+  rb_define_singleton_method(mConvolver, "correlate_basic_valid", convolver_correlate_basic_valid, 2);
   rb_define_singleton_method(mConvolver, "convolve_basic_valid", convolver_convolve_basic_valid, 2);
 }

data/ext/convolver/correlate_raw.c

@@ -0,0 +1,57 @@
+#include "correlate_raw.h"
+#include "raw_helpers.h"
+
+#define RAW_INDEX(j) (j)
+#if defined(__SSE__)
+#define RAW_SIMD 1
+#else
+#define RAW_SIMD 0
+#endif
+#define RAW_NAME correlate_raw
+#define RAW_TYPE float
+#define RAW_LANES 4
+#define RAW_VECTOR __m128
+#define RAW_ZERO _mm_setzero_ps
+#define RAW_LOAD _mm_loadu_ps
+#define RAW_STORE _mm_storeu_ps
+#define RAW_ADD _mm_add_ps
+#define RAW_MUL _mm_mul_ps
+#include "raw_loop.h"
+#undef RAW_NAME
+#undef RAW_TYPE
+#undef RAW_LANES
+#undef RAW_VECTOR
+#undef RAW_ZERO
+#undef RAW_LOAD
+#undef RAW_STORE
+#undef RAW_ADD
+#undef RAW_MUL
+#undef RAW_SIMD
+
+#if defined(__SSE2__)
+#define RAW_SIMD 1
+#else
+#define RAW_SIMD 0
+#endif
+#define RAW_NAME correlate_raw_double
+#define RAW_TYPE double
+#define RAW_LANES 2
+#define RAW_VECTOR __m128d
+#define RAW_ZERO _mm_setzero_pd
+#define RAW_LOAD _mm_loadu_pd
+#define RAW_STORE _mm_storeu_pd
+#define RAW_ADD _mm_add_pd
+#define RAW_MUL _mm_mul_pd
+#include "raw_loop.h"
+#undef RAW_NAME
+#undef RAW_TYPE
+#undef RAW_LANES
+#undef RAW_VECTOR
+#undef RAW_ZERO
+#undef RAW_LOAD
+#undef RAW_STORE
+#undef RAW_ADD
+#undef RAW_MUL
+#undef RAW_SIMD
+
+#undef RAW_INDEX

data/ext/convolver/correlate_raw.h

@@ -0,0 +1,14 @@
+#ifndef CORRELATE_RAW_H
+#define CORRELATE_RAW_H
+#include "raw_config.h"
+
+void correlate_raw(
+    int in_rank, const size_t *in_shape, const float *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
+    int out_rank, const size_t *out_shape, float *out_ptr );
+
+void correlate_raw_double(
+    int in_rank, const size_t *in_shape, const double *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
+    int out_rank, const size_t *out_shape, double *out_ptr );
+#endif

data/ext/convolver/extconf.rb

@@ -11,6 +11,7 @@ $LOAD_PATH.each do |load_path|
 end
 
 abort 'numo/narray.h not found' unless have_header('numo/narray.h')
+abort 'log2 not found' unless have_func('log2', 'math.h') || have_library('m', 'log2', 'math.h')
 
 if RUBY_PLATFORM.match?(/mswin|cygwin|mingw/)
   $LOAD_PATH.each do |load_path|

data/ext/convolver/fft_shape.c

@@ -0,0 +1,116 @@
+#include "fft_shape.h"
+#include "raw_config.h"
+#include <limits.h>
+#include <math.h>
+
+static size_t positive_size(VALUE value) {
+  if (!RB_INTEGER_TYPE_P(value) || RTEST(rb_funcall(value, rb_intern("<="), 1, INT2FIX(0)))) {
+    rb_raise(rb_eArgError, "FFT dimensions and size limit must be positive integers");
+  }
+  return NUM2SIZET(value);
+}
+
+static long read_shape(VALUE shape, size_t *dimensions) {
+  Check_Type(shape, T_ARRAY);
+  long rank = RARRAY_LEN(shape);
+  if (rank < 1 || rank > LARGEST_RANK) {
+    rb_raise(rb_eArgError, "FFT shape rank must be between 1 and %d", LARGEST_RANK);
+  }
+  for (long axis = 0; axis < rank; axis++) {
+    dimensions[axis] = positive_size(rb_ary_entry(shape, axis));
+  }
+  return rank;
+}
+
+static size_t doubled_candidate(size_t initial, size_t target, int even, size_t maximum) {
+  size_t candidate = initial;
+  while (candidate < target || (even && candidate % 2 != 0)) {
+    if (candidate > maximum / 2) return 0;
+    candidate *= 2;
+  }
+  return candidate;
+}
+
+/* Enumerate 3/5 powers only while they can improve the best 2/3/5 candidate. */
+static size_t next_fast_size(size_t target, int even, size_t maximum) {
+  size_t best = 0, five = 1;
+  for (;;) {
+    size_t three = five;
+    for (;;) {
+      size_t candidate = doubled_candidate(three, target, even, maximum);
+      if (candidate != 0 && (best == 0 || candidate < best)) best = candidate;
+      size_t bound = best != 0 ? best : maximum;
+      if (three > bound / 3) break;
+      three *= 3;
+    }
+    size_t bound = best != 0 ? best : maximum;
+    if (five > bound / 5) break;
+    five *= 5;
+  }
+  return best;
+}
+
+/*
+ * Generates optional smooth dimensions without allocating numeric buffers.
+ * @private
+ */
+static VALUE convolver_fft_fast_shape(VALUE self, VALUE shape, VALUE maximum_value) {
+  (void)self;
+  size_t dimensions[LARGEST_RANK];
+  long rank = read_shape(shape, dimensions);
+  size_t maximum = positive_size(maximum_value);
+  for (long axis = 0; axis < rank; axis++) {
+    if (dimensions[axis] > maximum) return Qnil;
+    dimensions[axis] = next_fast_size(dimensions[axis], axis == rank - 1, maximum);
+    if (dimensions[axis] == 0) return Qnil;
+  }
+  VALUE result = rb_ary_new_capa(rank);
+  for (long axis = 0; axis < rank; axis++) {
+    rb_ary_push(result, SIZET2NUM(dimensions[axis]));
+  }
+  return result;
+}
+
+static double large_factor_penalty(size_t value) {
+  double penalty = 0.0;
+  size_t divisor = 2;
+  while (divisor <= value / divisor) {
+    while (value % divisor == 0) {
+      if (divisor > 7) penalty += 1.5 * (log2((double)divisor) - 3.0);
+      value /= divisor;
+    }
+    divisor += divisor == 2 ? 1 : 2;
+  }
+  if (value > 7) penalty += 1.5 * (log2((double)value) - 3.0);
+  return penalty;
+}
+
+/*
+ * Scores an admissible candidate; Ruby owns the complete FFT buffer policy.
+ * @private
+ */
+static VALUE convolver_fft_shape_cost(VALUE self, VALUE shape) {
+  (void)self;
+  size_t dimensions[LARGEST_RANK], product = 1;
+  long rank = read_shape(shape, dimensions);
+  for (long axis = 0; axis < rank; axis++) {
+    /* Bound trial factorization even if this private primitive is called directly. */
+    if (dimensions[axis] > (size_t)INT_MAX / 16) {
+      rb_raise(rb_eRangeError, "FFT scoring axis exceeds native integer limit");
+    }
+    if (dimensions[axis] > SIZE_MAX / product) {
+      rb_raise(rb_eRangeError, "FFT scoring size exceeds native element limit");
+    }
+    product *= dimensions[axis];
+  }
+  double cost = 0.0;
+  for (long axis = 0; axis < rank; axis++) {
+    cost += log2((double)dimensions[axis]) + large_factor_penalty(dimensions[axis]);
+  }
+  return DBL2NUM((double)product * cost);
+}
+
+void convolver_init_fft_shape(VALUE convolver) {
+  rb_define_singleton_method(convolver, "fft_fast_shape", convolver_fft_fast_shape, 2);
+  rb_define_singleton_method(convolver, "fft_shape_cost", convolver_fft_shape_cost, 1);
+}

data/ext/convolver/fft_shape.h

@@ -0,0 +1,8 @@
+#ifndef CONVOLVER_FFT_SHAPE_H
+#define CONVOLVER_FFT_SHAPE_H
+
+#include <ruby.h>
+
+void convolver_init_fft_shape(VALUE convolver);
+
+#endif

data/ext/convolver/raw_config.h

@@ -0,0 +1,11 @@
+#ifndef CONVOLVER_RAW_CONFIG_H
+#define CONVOLVER_RAW_CONFIG_H
+#include <ruby.h>
+#if defined(__SSE__)
+#include <xmmintrin.h>
+#endif
+#if defined(__SSE2__)
+#include <emmintrin.h>
+#endif
+#define LARGEST_RANK 16
+#endif

data/ext/convolver/raw_helpers.h

@@ -0,0 +1,71 @@
+#ifndef CONVOLVER_RAW_HELPERS_H
+#define CONVOLVER_RAW_HELPERS_H
+
+static inline size_t checked_add( size_t left, size_t right, const char *description ) {
+  if ( right > SIZE_MAX - left ) {
+    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
+  }
+  return left + right;
+}
+
+static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
+  if ( left != 0 && right > SIZE_MAX / left ) {
+    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
+  }
+  return left * right;
+}
+
+static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
+  size_t size = 1;
+  int i;
+  for ( i = 0; i < rank; i++ ) {
+    size = checked_multiply( size, shape[i], description );
+  }
+  return size;
+}
+
+// Sets reverse indices
+static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
+  int i;
+  for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
+}
+
+// Counts indices down, returns number of ranks that reset
+static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
+  int i = 0;
+  (void) rank;
+  while ( ! rev_indices[i]-- ) {
+    rev_indices[i] = shape[i] - 1;
+    i++;
+  }
+  return i;
+}
+
+// Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
+static inline void calc_co_increment(
+    int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
+  size_t factor = 1;
+  int i;
+  co_increment[0] = 1; // co-increment is always 1 in lowest rank
+  for ( i = 0; i < rank; i++ ) {
+    size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
+    co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
+    factor = checked_multiply( factor, outer_shape[i], "array size" );
+  }
+}
+
+static inline size_t maximum_offset(
+    int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
+  size_t factor = 1;
+  size_t offset = 0;
+  int i;
+  for ( i = 0; i < rank; i++ ) {
+    size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
+    offset = checked_add( offset, dimension_offset, description );
+    factor = checked_multiply( factor, outer_shape[i], "array size" );
+  }
+  return offset;
+}
+
+
+#endif

data/ext/convolver/raw_loop.h

@@ -0,0 +1,93 @@
+/* Intentionally included once per dtype; operation/dtype dispatch is outside hot loops. */
+void RAW_NAME(
+    int in_rank, const size_t *in_shape, const RAW_TYPE *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const RAW_TYPE *kernel_ptr,
+    int out_rank, const size_t *out_shape, RAW_TYPE *out_ptr ) {
+  size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
+  size_t maximum_input_offset;
+  size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
+  size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
+  size_t *kernel_co_incr_cache;
+  VALUE cache_storage = 0;
+
+  kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
+  kernel_aligned = kernel_size - kernel_size % RAW_LANES;
+  out_size = size_from_shape( out_rank, out_shape, "output size" );
+  input_size = size_from_shape( in_rank, in_shape, "input size" );
+
+  calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
+  calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
+  maximum_input_offset = checked_add(
+    maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
+    maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
+    "input offset"
+  );
+  if ( maximum_input_offset >= input_size ) {
+    rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
+  }
+
+  kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
+  kernel_co_incr_cache[0] = 0;
+
+  corner_reset( kernel_rank, kernel_shape, ker_q );
+  for ( i = 1; i < kernel_size; i++ ) {
+    kernel_co_incr_cache[i] = checked_add(
+      kernel_co_incr_cache[i-1],
+      kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
+      "kernel offset"
+    );
+  }
+
+  offset = 0;
+  corner_reset( out_rank, out_shape, out_q );
+
+  for ( i = 0; i < out_size; i++ ) {
+    RAW_TYPE t = 0;
+#if RAW_SIMD
+    RAW_VECTOR simd_t = RAW_ZERO();
+    RAW_TYPE v[RAW_LANES];
+    for ( j = 0; j < kernel_aligned; j += RAW_LANES ) {
+      RAW_VECTOR simd_x = RAW_LOAD(kernel_ptr + j);
+#if RAW_LANES == 4
+      RAW_VECTOR simd_y = _mm_set_ps(
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+3)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+2)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+1)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]]);
+#else
+      RAW_VECTOR simd_y = _mm_set_pd(
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+1)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]]);
+#endif
+      simd_t = RAW_ADD(RAW_MUL(simd_x, simd_y), simd_t);
+    }
+    RAW_STORE(v, simd_t);
+#else
+    for ( j = 0; j < kernel_aligned; j++ ) {
+      t += in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]] * kernel_ptr[j];
+    }
+#endif
+    for ( j = kernel_aligned; j < kernel_size; j++ ) {
+      t += in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]] * kernel_ptr[j];
+    }
+#if RAW_SIMD
+#if RAW_LANES == 4
+    out_ptr[i] = v[0] + v[1] + v[2] + v[3] + t;
+#else
+    out_ptr[i] = v[0] + v[1] + t;
+#endif
+#else
+    out_ptr[i] = t;
+#endif
+
+    if ( i + 1 < out_size ) {
+      offset = checked_add(
+        offset,
+        out_co_incr[ corner_dec( out_rank, out_shape, out_q ) ],
+        "input offset"
+      );
+    }
+  }
+
+  RB_ALLOCV_END( cache_storage );
+}

data/lib/convolver/buffer_limits.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Checks individual allocations against native size and tagged-stride limits.
+  # @private
+  class BufferLimits
+    SIZE_MAX = (1 << ([0].pack('J').bytesize * 8)) - 1
+    # Numo tags signed strides with a left shift: floor(SSIZE_MAX / 2).
+    BYTE_MAX = SIZE_MAX / 4
+    INT_MAX = (1 << (([0].pack('i').bytesize * 8) - 1)) - 1
+    INDEX_BYTES = [0].pack('J').bytesize
+
+    attr_reader :size_max, :byte_max, :int_max
+
+    def initialize(size_max: SIZE_MAX, byte_max: BYTE_MAX, int_max: INT_MAX)
+      @size_max = size_max
+      @byte_max = byte_max
+      @int_max = int_max
+    end
+
+    def buffer!(count, width, description, error: RangeError)
+      return if count <= byte_max / width
+
+      raise error, "#{description} exceeds native byte/stride limit"
+    end
+
+    def product!(shape, description, error: RangeError)
+      shape.reduce(1) do |product, size|
+        raise error, "#{description} exceeds native element limit" if size > size_max / product
+
+        product * size
+      end
+    end
+
+    NATIVE = new.freeze
+  end
+
+  private_constant :BufferLimits
+end

data/lib/convolver/buffer_preparation.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'convolver/fft_limits'
+
+module Convolver
+  # Preflights casts, extension, indexing and direct-path cache allocations.
+  # @private
+  class BufferPreparation
+    def initialize(plan, signal_shape, kernel_shape, limits: BufferLimits::NATIVE)
+      @plan = plan
+      @signal_size = limits.product!(signal_shape, 'signal size')
+      @kernel_size = limits.product!(kernel_shape, 'kernel size')
+      @limits = limits
+      @scalar = signal_shape.empty?
+    end
+
+    def basic!
+      common!
+      extension!
+      limits.buffer!(kernel_size, BufferLimits::INDEX_BYTES, 'direct kernel offset cache')
+    end
+
+    def fft!
+      common!
+      return if @scalar
+      return extension! unless plan.wrap?
+
+      limits.buffer!(signal_size, 8, 'circular signal preparation', error: FftUnavailable)
+      limits.buffer!(kernel_size, 8, 'circular kernel preparation', error: FftUnavailable)
+      limits.buffer!(kernel_size, BufferLimits::INDEX_BYTES, 'circular kernel index buffer', error: FftUnavailable)
+    end
+
+    private
+
+    attr_reader :plan, :signal_size, :kernel_size, :limits
+
+    def common!
+      limits.buffer!(plan.result_size, plan.dtype::ELEMENT_BYTE_SIZE, 'result buffer')
+      limits.buffer!(signal_size, plan.dtype::ELEMENT_BYTE_SIZE, 'signal conversion buffer')
+      limits.buffer!(kernel_size, plan.dtype::ELEMENT_BYTE_SIZE, 'kernel conversion buffer')
+    end
+
+    def extension!
+      return if plan.valid?
+
+      limits.buffer!(plan.extended_size, plan.dtype::ELEMENT_BYTE_SIZE, 'extended signal buffer')
+      return if plan.boundary == :constant
+
+      plan.extended_shape.each do |length|
+        limits.buffer!(length, BufferLimits::INDEX_BYTES, 'boundary index buffer')
+      end
+    end
+  end
+
+  private_constant :BufferPreparation
+end