convolver 3.0.1 → 4.0.0

data/ext/convolver/correlate_raw.c

@@ -1,167 +1,57 @@
-// ext/convolver/correlate_raw.c
-
 #include "correlate_raw.h"
+#include "raw_helpers.h"
 
-static inline size_t checked_add( size_t left, size_t right, const char *description ) {
-  if ( right > SIZE_MAX - left ) {
-    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
-  }
-  return left + right;
-}
-
-static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
-  if ( left != 0 && right > SIZE_MAX / left ) {
-    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
-  }
-  return left * right;
-}
-
-static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
-  size_t size = 1;
-  int i;
-  for ( i = 0; i < rank; i++ ) {
-    size = checked_multiply( size, shape[i], description );
-  }
-  return size;
-}
-
-// Sets reverse indices
-static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
-  int i;
-  for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
-}
-
-// Counts indices down, returns number of ranks that reset
-static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
-  int i = 0;
-  (void) rank;
-  while ( ! rev_indices[i]-- ) {
-    rev_indices[i] = shape[i] - 1;
-    i++;
-  }
-  return i;
-}
-
-// Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
-static inline void calc_co_increment(
-    int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
-  size_t factor = 1;
-  int i;
-  co_increment[0] = 1; // co-increment is always 1 in lowest rank
-  for ( i = 0; i < rank; i++ ) {
-    size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
-    co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
-    factor = checked_multiply( factor, outer_shape[i], "array size" );
-  }
-}
-
-static inline size_t maximum_offset(
-    int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
-  size_t factor = 1;
-  size_t offset = 0;
-  int i;
-  for ( i = 0; i < rank; i++ ) {
-    size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
-    offset = checked_add( offset, dimension_offset, description );
-    factor = checked_multiply( factor, outer_shape[i], "array size" );
-  }
-  return offset;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-//
-//  Correlate
-//
-
-void correlate_raw(
-    int in_rank, const size_t *in_shape, const float *in_ptr,
-    int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
-    int out_rank, const size_t *out_shape, float *out_ptr ) {
-  size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
-  size_t maximum_input_offset;
-  size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
-  size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
-  size_t *kernel_co_incr_cache;
-  VALUE cache_storage = 0;
-
-  kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
-  kernel_aligned = kernel_size - kernel_size % 4;
-  out_size = size_from_shape( out_rank, out_shape, "output size" );
-  input_size = size_from_shape( in_rank, in_shape, "input size" );
-
-  calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
-  calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
-  maximum_input_offset = checked_add(
-    maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
-    maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
-    "input offset"
-  );
-  if ( maximum_input_offset >= input_size ) {
-    rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
-  }
-
-  kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
-  kernel_co_incr_cache[0] = 0;
-
-  corner_reset( kernel_rank, kernel_shape, ker_q );
-  for ( i = 1; i < kernel_size; i++ ) {
-    kernel_co_incr_cache[i] = checked_add(
-      kernel_co_incr_cache[i-1],
-      kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
-      "kernel offset"
-    );
-  }
-
-  offset = 0;
-  corner_reset( out_rank, out_shape, out_q );
-
-  // Main correlation loop
-  for ( i = 0; i < out_size; i++ ) {
-#if CONVOLVER_USE_SSE
-    __m128 simd_x, simd_y, simd_t;
-    float v[4];
-    simd_t = _mm_setzero_ps();
-#endif
-    float t = 0.0;
-
-#if CONVOLVER_USE_SSE
-    // Contiguous kernel views need not start on a 16-byte boundary.
-    for ( j = 0; j < kernel_aligned; j +=4 ) {
-      simd_x = _mm_loadu_ps( kernel_ptr + j );
-      // Yes the backwards alignment is correct
-      simd_y = _mm_set_ps( in_ptr[ offset + kernel_co_incr_cache[j+3] ], in_ptr[ offset + kernel_co_incr_cache[j+2] ],
-                           in_ptr[ offset + kernel_co_incr_cache[j+1] ], in_ptr[ offset + kernel_co_incr_cache[j] ] );
-      simd_x = _mm_mul_ps( simd_x, simd_y );
-      simd_t = _mm_add_ps( simd_x, simd_t );
-    }
-    _mm_store_ps( v, simd_t );
+#define RAW_INDEX(j) (j)
+#if defined(__SSE__)
+#define RAW_SIMD 1
 #else
-    // SSE is unavailable on architectures such as Apple Silicon.
-    // Use the same calculation without x86-specific intrinsics.
-    for ( j = 0; j < kernel_aligned; j++ ) {
-      t += in_ptr[ offset + kernel_co_incr_cache[j] ] * kernel_ptr[j];
-    }
+#define RAW_SIMD 0
 #endif
-
-    // Complete any remaining 1,2 or 3 items one at a time
-    for ( j = kernel_aligned; j < kernel_size; j++ ) {
-      t += in_ptr[ offset + kernel_co_incr_cache[j] ] * kernel_ptr[ j ];
-    }
-
-#if CONVOLVER_USE_SSE
-    out_ptr[i] = v[0] + v[1] + v[2] + v[3] + t;
+#define RAW_NAME correlate_raw
+#define RAW_TYPE float
+#define RAW_LANES 4
+#define RAW_VECTOR __m128
+#define RAW_ZERO _mm_setzero_ps
+#define RAW_LOAD _mm_loadu_ps
+#define RAW_STORE _mm_storeu_ps
+#define RAW_ADD _mm_add_ps
+#define RAW_MUL _mm_mul_ps
+#include "raw_loop.h"
+#undef RAW_NAME
+#undef RAW_TYPE
+#undef RAW_LANES
+#undef RAW_VECTOR
+#undef RAW_ZERO
+#undef RAW_LOAD
+#undef RAW_STORE
+#undef RAW_ADD
+#undef RAW_MUL
+#undef RAW_SIMD
+
+#if defined(__SSE2__)
+#define RAW_SIMD 1
 #else
-    out_ptr[i] = t;
+#define RAW_SIMD 0
 #endif
-
-    if ( i + 1 < out_size ) {
-      offset = checked_add(
-        offset,
-        out_co_incr[ corner_dec( out_rank, out_shape, out_q ) ],
-        "input offset"
-      );
-    }
-  }
-
-  RB_ALLOCV_END( cache_storage );
-}
+#define RAW_NAME correlate_raw_double
+#define RAW_TYPE double
+#define RAW_LANES 2
+#define RAW_VECTOR __m128d
+#define RAW_ZERO _mm_setzero_pd
+#define RAW_LOAD _mm_loadu_pd
+#define RAW_STORE _mm_storeu_pd
+#define RAW_ADD _mm_add_pd
+#define RAW_MUL _mm_mul_pd
+#include "raw_loop.h"
+#undef RAW_NAME
+#undef RAW_TYPE
+#undef RAW_LANES
+#undef RAW_VECTOR
+#undef RAW_ZERO
+#undef RAW_LOAD
+#undef RAW_STORE
+#undef RAW_ADD
+#undef RAW_MUL
+#undef RAW_SIMD
+
+#undef RAW_INDEX

data/ext/convolver/correlate_raw.h

@@ -1,28 +1,14 @@
-// ext/convolver/correlate_raw.h
-
-////////////////////////////////////////////////////////////////////////////////////////////////
-//
-// Declarations of narray helper functions
-//
-
 #ifndef CORRELATE_RAW_H
 #define CORRELATE_RAW_H
-
-#ifndef CONVOLVER_RAW_CONFIG
-#define CONVOLVER_RAW_CONFIG
-#include <ruby.h>
-#if defined(__SSE__)
-#include <xmmintrin.h>
-#define CONVOLVER_USE_SSE 1
-#else
-#define CONVOLVER_USE_SSE 0
-#endif
-#define LARGEST_RANK 16
-#endif
+#include "raw_config.h"
 
 void correlate_raw(
     int in_rank, const size_t *in_shape, const float *in_ptr,
     int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
     int out_rank, const size_t *out_shape, float *out_ptr );
 
+void correlate_raw_double(
+    int in_rank, const size_t *in_shape, const double *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
+    int out_rank, const size_t *out_shape, double *out_ptr );
 #endif

data/ext/convolver/extconf.rb

@@ -11,6 +11,7 @@ $LOAD_PATH.each do |load_path|
 end
 
 abort 'numo/narray.h not found' unless have_header('numo/narray.h')
+abort 'log2 not found' unless have_func('log2', 'math.h') || have_library('m', 'log2', 'math.h')
 
 if RUBY_PLATFORM.match?(/mswin|cygwin|mingw/)
   $LOAD_PATH.each do |load_path|

data/ext/convolver/fft_shape.c

@@ -0,0 +1,116 @@
+#include "fft_shape.h"
+#include "raw_config.h"
+#include <limits.h>
+#include <math.h>
+
+static size_t positive_size(VALUE value) {
+  if (!RB_INTEGER_TYPE_P(value) || RTEST(rb_funcall(value, rb_intern("<="), 1, INT2FIX(0)))) {
+    rb_raise(rb_eArgError, "FFT dimensions and size limit must be positive integers");
+  }
+  return NUM2SIZET(value);
+}
+
+static long read_shape(VALUE shape, size_t *dimensions) {
+  Check_Type(shape, T_ARRAY);
+  long rank = RARRAY_LEN(shape);
+  if (rank < 1 || rank > LARGEST_RANK) {
+    rb_raise(rb_eArgError, "FFT shape rank must be between 1 and %d", LARGEST_RANK);
+  }
+  for (long axis = 0; axis < rank; axis++) {
+    dimensions[axis] = positive_size(rb_ary_entry(shape, axis));
+  }
+  return rank;
+}
+
+static size_t doubled_candidate(size_t initial, size_t target, int even, size_t maximum) {
+  size_t candidate = initial;
+  while (candidate < target || (even && candidate % 2 != 0)) {
+    if (candidate > maximum / 2) return 0;
+    candidate *= 2;
+  }
+  return candidate;
+}
+
+/* Enumerate 3/5 powers only while they can improve the best 2/3/5 candidate. */
+static size_t next_fast_size(size_t target, int even, size_t maximum) {
+  size_t best = 0, five = 1;
+  for (;;) {
+    size_t three = five;
+    for (;;) {
+      size_t candidate = doubled_candidate(three, target, even, maximum);
+      if (candidate != 0 && (best == 0 || candidate < best)) best = candidate;
+      size_t bound = best != 0 ? best : maximum;
+      if (three > bound / 3) break;
+      three *= 3;
+    }
+    size_t bound = best != 0 ? best : maximum;
+    if (five > bound / 5) break;
+    five *= 5;
+  }
+  return best;
+}
+
+/*
+ * Generates optional smooth dimensions without allocating numeric buffers.
+ * @private
+ */
+static VALUE convolver_fft_fast_shape(VALUE self, VALUE shape, VALUE maximum_value) {
+  (void)self;
+  size_t dimensions[LARGEST_RANK];
+  long rank = read_shape(shape, dimensions);
+  size_t maximum = positive_size(maximum_value);
+  for (long axis = 0; axis < rank; axis++) {
+    if (dimensions[axis] > maximum) return Qnil;
+    dimensions[axis] = next_fast_size(dimensions[axis], axis == rank - 1, maximum);
+    if (dimensions[axis] == 0) return Qnil;
+  }
+  VALUE result = rb_ary_new_capa(rank);
+  for (long axis = 0; axis < rank; axis++) {
+    rb_ary_push(result, SIZET2NUM(dimensions[axis]));
+  }
+  return result;
+}
+
+static double large_factor_penalty(size_t value) {
+  double penalty = 0.0;
+  size_t divisor = 2;
+  while (divisor <= value / divisor) {
+    while (value % divisor == 0) {
+      if (divisor > 7) penalty += 1.5 * (log2((double)divisor) - 3.0);
+      value /= divisor;
+    }
+    divisor += divisor == 2 ? 1 : 2;
+  }
+  if (value > 7) penalty += 1.5 * (log2((double)value) - 3.0);
+  return penalty;
+}
+
+/*
+ * Scores an admissible candidate; Ruby owns the complete FFT buffer policy.
+ * @private
+ */
+static VALUE convolver_fft_shape_cost(VALUE self, VALUE shape) {
+  (void)self;
+  size_t dimensions[LARGEST_RANK], product = 1;
+  long rank = read_shape(shape, dimensions);
+  for (long axis = 0; axis < rank; axis++) {
+    /* Bound trial factorization even if this private primitive is called directly. */
+    if (dimensions[axis] > (size_t)INT_MAX / 16) {
+      rb_raise(rb_eRangeError, "FFT scoring axis exceeds native integer limit");
+    }
+    if (dimensions[axis] > SIZE_MAX / product) {
+      rb_raise(rb_eRangeError, "FFT scoring size exceeds native element limit");
+    }
+    product *= dimensions[axis];
+  }
+  double cost = 0.0;
+  for (long axis = 0; axis < rank; axis++) {
+    cost += log2((double)dimensions[axis]) + large_factor_penalty(dimensions[axis]);
+  }
+  return DBL2NUM((double)product * cost);
+}
+
+void convolver_init_fft_shape(VALUE convolver) {
+  rb_define_singleton_method(convolver, "fft_fast_shape", convolver_fft_fast_shape, 2);
+  rb_define_singleton_method(convolver, "fft_shape_cost", convolver_fft_shape_cost, 1);
+}

data/ext/convolver/fft_shape.h

@@ -0,0 +1,8 @@
+#ifndef CONVOLVER_FFT_SHAPE_H
+#define CONVOLVER_FFT_SHAPE_H
+
+#include <ruby.h>
+
+void convolver_init_fft_shape(VALUE convolver);
+
+#endif

data/ext/convolver/raw_config.h

@@ -0,0 +1,11 @@
+#ifndef CONVOLVER_RAW_CONFIG_H
+#define CONVOLVER_RAW_CONFIG_H
+#include <ruby.h>
+#if defined(__SSE__)
+#include <xmmintrin.h>
+#endif
+#if defined(__SSE2__)
+#include <emmintrin.h>
+#endif
+#define LARGEST_RANK 16
+#endif

data/ext/convolver/raw_helpers.h

@@ -0,0 +1,71 @@
+#ifndef CONVOLVER_RAW_HELPERS_H
+#define CONVOLVER_RAW_HELPERS_H
+
+static inline size_t checked_add( size_t left, size_t right, const char *description ) {
+  if ( right > SIZE_MAX - left ) {
+    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
+  }
+  return left + right;
+}
+
+static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
+  if ( left != 0 && right > SIZE_MAX / left ) {
+    rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
+  }
+  return left * right;
+}
+
+static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
+  size_t size = 1;
+  int i;
+  for ( i = 0; i < rank; i++ ) {
+    size = checked_multiply( size, shape[i], description );
+  }
+  return size;
+}
+
+// Sets reverse indices
+static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
+  int i;
+  for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
+}
+
+// Counts indices down, returns number of ranks that reset
+static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
+  int i = 0;
+  (void) rank;
+  while ( ! rev_indices[i]-- ) {
+    rev_indices[i] = shape[i] - 1;
+    i++;
+  }
+  return i;
+}
+
+// Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
+static inline void calc_co_increment(
+    int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
+  size_t factor = 1;
+  int i;
+  co_increment[0] = 1; // co-increment is always 1 in lowest rank
+  for ( i = 0; i < rank; i++ ) {
+    size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
+    co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
+    factor = checked_multiply( factor, outer_shape[i], "array size" );
+  }
+}
+
+static inline size_t maximum_offset(
+    int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
+  size_t factor = 1;
+  size_t offset = 0;
+  int i;
+  for ( i = 0; i < rank; i++ ) {
+    size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
+    offset = checked_add( offset, dimension_offset, description );
+    factor = checked_multiply( factor, outer_shape[i], "array size" );
+  }
+  return offset;
+}
+
+
+#endif

data/ext/convolver/raw_loop.h

@@ -0,0 +1,93 @@
+/* Intentionally included once per dtype; operation/dtype dispatch is outside hot loops. */
+void RAW_NAME(
+    int in_rank, const size_t *in_shape, const RAW_TYPE *in_ptr,
+    int kernel_rank, const size_t *kernel_shape, const RAW_TYPE *kernel_ptr,
+    int out_rank, const size_t *out_shape, RAW_TYPE *out_ptr ) {
+  size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
+  size_t maximum_input_offset;
+  size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
+  size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
+  size_t *kernel_co_incr_cache;
+  VALUE cache_storage = 0;
+
+  kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
+  kernel_aligned = kernel_size - kernel_size % RAW_LANES;
+  out_size = size_from_shape( out_rank, out_shape, "output size" );
+  input_size = size_from_shape( in_rank, in_shape, "input size" );
+
+  calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
+  calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
+  maximum_input_offset = checked_add(
+    maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
+    maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
+    "input offset"
+  );
+  if ( maximum_input_offset >= input_size ) {
+    rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
+  }
+
+  kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
+  kernel_co_incr_cache[0] = 0;
+
+  corner_reset( kernel_rank, kernel_shape, ker_q );
+  for ( i = 1; i < kernel_size; i++ ) {
+    kernel_co_incr_cache[i] = checked_add(
+      kernel_co_incr_cache[i-1],
+      kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
+      "kernel offset"
+    );
+  }
+
+  offset = 0;
+  corner_reset( out_rank, out_shape, out_q );
+
+  for ( i = 0; i < out_size; i++ ) {
+    RAW_TYPE t = 0;
+#if RAW_SIMD
+    RAW_VECTOR simd_t = RAW_ZERO();
+    RAW_TYPE v[RAW_LANES];
+    for ( j = 0; j < kernel_aligned; j += RAW_LANES ) {
+      RAW_VECTOR simd_x = RAW_LOAD(kernel_ptr + j);
+#if RAW_LANES == 4
+      RAW_VECTOR simd_y = _mm_set_ps(
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+3)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+2)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+1)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]]);
+#else
+      RAW_VECTOR simd_y = _mm_set_pd(
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j+1)]],
+        in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]]);
+#endif
+      simd_t = RAW_ADD(RAW_MUL(simd_x, simd_y), simd_t);
+    }
+    RAW_STORE(v, simd_t);
+#else
+    for ( j = 0; j < kernel_aligned; j++ ) {
+      t += in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]] * kernel_ptr[j];
+    }
+#endif
+    for ( j = kernel_aligned; j < kernel_size; j++ ) {
+      t += in_ptr[offset + kernel_co_incr_cache[RAW_INDEX(j)]] * kernel_ptr[j];
+    }
+#if RAW_SIMD
+#if RAW_LANES == 4
+    out_ptr[i] = v[0] + v[1] + v[2] + v[3] + t;
+#else
+    out_ptr[i] = v[0] + v[1] + t;
+#endif
+#else
+    out_ptr[i] = t;
+#endif
+
+    if ( i + 1 < out_size ) {
+      offset = checked_add(
+        offset,
+        out_co_incr[ corner_dec( out_rank, out_shape, out_q ) ],
+        "input offset"
+      );
+    }
+  }
+
+  RB_ALLOCV_END( cache_storage );
+}

data/lib/convolver/buffer_preparation.rb

@@ -35,15 +35,15 @@ module Convolver
     attr_reader :plan, :signal_size, :kernel_size, :limits
 
     def common!
-      limits.buffer!(plan.result_size, 4, 'result buffer')
-      limits.buffer!(signal_size, 4, 'signal conversion buffer')
-      limits.buffer!(kernel_size, 4, 'kernel conversion buffer')
+      limits.buffer!(plan.result_size, plan.dtype::ELEMENT_BYTE_SIZE, 'result buffer')
+      limits.buffer!(signal_size, plan.dtype::ELEMENT_BYTE_SIZE, 'signal conversion buffer')
+      limits.buffer!(kernel_size, plan.dtype::ELEMENT_BYTE_SIZE, 'kernel conversion buffer')
     end
 
     def extension!
       return if plan.valid?
 
-      limits.buffer!(plan.extended_size, 4, 'extended signal buffer')
+      limits.buffer!(plan.extended_size, plan.dtype::ELEMENT_BYTE_SIZE, 'extended signal buffer')
       return if plan.boundary == :constant
 
       plan.extended_shape.each do |length|

data/lib/convolver/circular_fft_operation.rb

@@ -5,6 +5,7 @@ module Convolver
   # @private
   class CircularFftOperation
     def initialize(operation, signal, kernel, plan)
+      @dtype = plan.dtype
       @real_axis = plan.fft_buffers.real_axis
       @operation = operation
       @signal = Numo::DFloat.cast(signal)
@@ -28,7 +29,7 @@ module Convolver
 
     private
 
-    attr_reader :operation, :signal, :kernel, :real_axis
+    attr_reader :operation, :signal, :kernel, :real_axis, :dtype
 
     def real_product(real_axis)
       final_axis = signal.ndim - 1
@@ -36,7 +37,7 @@ module Convolver
       prepared_kernel = prepare_real_operand(kernel, real_axis, final_axis)
       result = real_spectrum_product(prepared_signal, prepared_kernel)
       result = result.swapaxes(real_axis, final_axis) unless real_axis == final_axis
-      Numo::SFloat.cast(result)
+      dtype.cast(result)
     end
 
     def prepare_real_operand(value, real_axis, final_axis)
@@ -55,7 +56,7 @@ module Convolver
       kernel_spectrum = Numo::Pocketfft.fftn(kernel)
       kernel_spectrum.inplace.conj if operation == :correlation
       result = Numo::Pocketfft.ifftn(signal_spectrum.inplace * kernel_spectrum).real
-      Numo::SFloat.cast(result)
+      dtype.cast(result)
     end
 
     def fold_kernel(source, signal_shape, anchors)

data/lib/convolver/cost_profiles.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Immutable measured-environment coefficients in seconds; ranks one through three.
+  # @private
+  module CostProfiles
+    MAC = {
+      direct: [[6.55e-6, 3.24e-10, 7.08e-10].freeze, [6.55e-6, 3.24e-10, 7.08e-10].freeze].freeze,
+      fixed: [1.64e-5, 5.12e-5, 1.13e-4].freeze,
+      transform: [8.02e-10, 7.17e-10, 1.54e-9].freeze,
+      indices: { reflect: 1.33e-7, nearest: 1.46e-7, mirror: 1.57e-7, wrap: 3.23e-8 }.freeze,
+      folding: [3.78e-7, 3.99e-7, 5.09e-7].freeze
+    }.freeze
+    X86 = {
+      direct: [[1.59e-5, 3.08e-10, 2.52e-10].freeze, [1.63e-5, 3.22e-10, 3.96e-10].freeze].freeze,
+      fixed: [3.77e-5, 1.20e-4, 1.98e-4].freeze,
+      transform: [2.00e-9, 2.13e-9, 3.13e-9].freeze,
+      indices: { reflect: 2.32e-7, nearest: 2.88e-7, mirror: 2.62e-7, wrap: 1.11e-7 }.freeze,
+      folding: [8.20e-7, 7.51e-7, 9.79e-7].freeze
+    }.freeze
+    ARM = {
+      direct: [[1.88e-5, 5.12e-10, 5.96e-10].freeze, [1.88e-5, 5.12e-10, 5.96e-10].freeze].freeze,
+      fixed: [4.20e-5, 1.20e-4, 2.70e-4].freeze,
+      transform: [3.29e-9, 2.89e-9, 3.74e-9].freeze,
+      indices: { reflect: 2.62e-7, nearest: 3.67e-7, mirror: 2.87e-7, wrap: 1.50e-7 }.freeze,
+      folding: [9.97e-7, 8.80e-7, 1.24e-6].freeze
+    }.freeze
+    def self.for_platform(platform)
+      case platform
+      when /\Aarm64-darwin/ then MAC
+      when /\Ax86_64-linux/ then X86
+      when /\Aaarch64-linux/ then ARM
+      end
+    end
+
+    CURRENT = for_platform(RUBY_PLATFORM)
+
+    def self.for_rank(rank)
+      CURRENT if (1..3).cover?(rank)
+    end
+  end
+
+  private_constant :CostProfiles
+end