convolver 3.0.1 → 4.0.0

data/lib/convolver/fft_cost.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'convolver/preparation_cost'
+
+module Convolver
+  # Shared transform costs and a lower bound that never constructs an FFT plan.
+  # @private
+  class FftCost
+    REAL = { 1 => 1.0e-9, 2 => 1.0e-9, 3 => 1.5e-9 }.freeze
+    COMPLEX = { 1 => 2.4e-9, 2 => 2.4e-9, 3 => 2.8e-9 }.freeze
+    FIXED = { 0 => 5.0e-6, 1 => 1.5e-5, 2 => 3.0e-5, 3 => 1.0e-4 }.freeze
+    PREPARATION = 1.0e-9
+
+    def self.fixed(rank, profile: nil)
+      return profile.fetch(:fixed).fetch(rank - 1) if profile
+
+      FIXED.fetch(rank, FIXED.values.last)
+    end
+
+    def self.transform(size, rank, real: true, profile: nil)
+      coefficient(rank, real:, profile:) * size * Math.log([size, 2].max)
+    end
+
+    def self.lower_bound(plan, kernel_shape, threshold:)
+      rank = kernel_shape.length
+      profile = plan.cost_profile
+      setup = fixed(rank, profile:)
+      return setup if setup >= threshold || plan.wrap? || rank.zero?
+
+      size = minimum_size(plan.extended_shape, kernel_shape)
+      setup + transform(size, rank, profile:) + (2 * PREPARATION * size) + PreparationCost.new(plan).indices
+    end
+
+    def self.coefficient(rank, real:, profile:)
+      return profile.fetch(:transform).fetch(rank - 1) * (real ? 1 : 2.2) if profile
+
+      costs = real ? REAL : COMPLEX
+      costs.fetch(rank, costs.values.last)
+    end
+    private_class_method :coefficient
+
+    def self.minimum_size(signal_shape, kernel_shape)
+      # Ruby integers avoid overflow: this is only a cost bound, never an allocation.
+      shape = signal_shape.zip(kernel_shape).map { |signal, kernel| signal + kernel - 1 }
+      shape[-1] += 1 if shape[-1].odd?
+      shape.reduce(1, :*)
+    end
+    private_class_method :minimum_size
+  end
+
+  private_constant :FftCost
+end

data/lib/convolver/fft_estimator.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
+require 'convolver/fft_cost'
+
 module Convolver
   # Estimates the selected real or complex PocketFFT implementation cost.
   # @private
   class FftEstimator
-    REAL_FFT_COST = { 1 => 1.0e-9, 2 => 9.0e-10, 3 => 1.45e-9 }.freeze
-    COMPLEX_FFT_COST = { 1 => 2.4e-9, 2 => 2.4e-9, 3 => 2.8e-9 }.freeze
-    FIXED_COST = 1.5e-4
-    PREPARATION_COST = 1.0e-9
     AXIS_MOVE_COST = 1.6e-9
+    RESULT_COST = 5.0e-10
     CORRELATION_COST = 2.0e-10
 
     def initialize(operation, signal, kernel, plan)
@@ -16,54 +15,48 @@ module Convolver
       @signal = signal
       @kernel = kernel
       @plan = plan
+      @profile = plan.cost_profile
       @buffers = plan.fft_buffers
+      @preparation = PreparationCost.new(plan)
     end
 
     def call
-      transform_size, spectrum_size, coefficient, moved_size = dimensions
-      transform_cost = coefficient * transform_size * Math.log([transform_size, 2].max)
-      preparation_cost = PREPARATION_COST * preparation_size(transform_size)
-      axis_move_cost = AXIS_MOVE_COST * moved_size
-      correlation_cost = operation == :correlation ? CORRELATION_COST * spectrum_size : 0.0
-      FIXED_COST + transform_cost + preparation_cost + axis_move_cost + correlation_cost
+      return FftCost.fixed(0) + preparation.conversion + result_cost if signal.ndim.zero?
+
+      transform_cost + preparation_cost + extra_cost
     end
 
     private
 
-    attr_reader :operation, :signal, :kernel, :plan, :buffers
+    attr_reader :operation, :signal, :kernel, :plan, :buffers, :preparation
 
-    def dimensions
-      return linear_dimensions unless plan.wrap?
-
-      circular_dimensions
+    def transform_cost
+      transform = FftCost.transform(buffers.transform_size, signal.ndim,
+                                    real: !buffers.real_axis.nil?, profile: @profile)
+      FftCost.fixed(signal.ndim, profile: @profile) + transform + (AXIS_MOVE_COST * moved_size)
     end
 
-    def circular_dimensions
-      real_axis = buffers.real_axis
-      [buffers.transform_size, buffers.spectrum_size, transform_coefficient(real_axis), moved_size(real_axis)]
+    def extra_cost
+      correlation_cost = operation == :correlation ? CORRELATION_COST * buffers.spectrum_size : 0.0
+      correlation_cost + preparation.conversion + result_cost
     end
 
-    def linear_dimensions
-      [buffers.transform_size, buffers.spectrum_size,
-       cost_for(REAL_FFT_COST), 0]
+    def result_cost
+      RESULT_COST * plan.result_size * (plan.dtype::ELEMENT_BYTE_SIZE / 4)
     end
 
-    def preparation_size(transform_size)
-      extension_size = plan.valid? || plan.wrap? ? 0 : plan.extended_size
-      working_size = plan.wrap? ? signal.size + kernel.size : 2 * transform_size
-      extension_size + working_size
-    end
+    def preparation_cost
+      return circular_preparation_cost if plan.wrap?
 
-    def cost_for(costs)
-      costs.fetch(signal.ndim, costs.values.last)
+      (2 * FftCost::PREPARATION * buffers.transform_size) + preparation.extension
     end
 
-    def transform_coefficient(real_axis)
-      cost_for(real_axis ? REAL_FFT_COST : COMPLEX_FFT_COST)
+    def circular_preparation_cost
+      (FftCost::PREPARATION * (signal.size + kernel.size)) + preparation.folding(kernel.size)
     end
 
-    def moved_size(real_axis)
-      real_axis && real_axis != signal.ndim - 1 ? 3 * signal.size : 0
+    def moved_size
+      buffers.real_axis && buffers.real_axis != signal.ndim - 1 ? 3 * signal.size : 0
     end
   end
 

data/lib/convolver/linear_fft_operation.rb

@@ -5,6 +5,7 @@ module Convolver
   # @private
   class LinearFftOperation
     def initialize(operation, signal, kernel, plan)
+      @dtype = plan.dtype
       @operation = operation
       @signal = signal
       @kernel = kernel
@@ -15,12 +16,13 @@ module Convolver
       signal_padded = zero_padded(signal)
       kernel_padded = zero_padded(kernel)
       full_result = real_spectrum_product(signal_padded, kernel_padded)
-      Numo::SFloat.cast(full_result[*valid_ranges])
+      cropped = full_result[*valid_ranges]
+      dtype == Numo::DFloat ? cropped.dup : dtype.cast(cropped)
     end
 
     private
 
-    attr_reader :operation, :signal, :kernel, :transform_shape
+    attr_reader :operation, :signal, :kernel, :transform_shape, :dtype
 
     def zero_padded(value)
       Numo::DFloat.zeros(*transform_shape).tap do |padded|

data/lib/convolver/operation_dtype.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Resolves public precision without allocating operand copies.
+  # @private
+  class OperationDtype
+    SINGLE_INPUTS = [Numo::SFloat, Numo::Int8, Numo::UInt8, Numo::Int16, Numo::UInt16].freeze
+    DOUBLE_INPUTS = [Numo::DFloat, Numo::Int32, Numo::UInt32, Numo::Int64, Numo::UInt64].freeze
+    INPUTS = (SINGLE_INPUTS + DOUBLE_INPUTS).freeze
+    OUTPUTS = [Numo::SFloat, Numo::DFloat].freeze
+
+    def self.resolve(signal, kernel, dtype)
+      validate_inputs!(signal, kernel)
+      validate_dtype!(dtype)
+      types = [signal.class, kernel.class]
+      dtype || (types.any? { |type| DOUBLE_INPUTS.include?(type) } ? Numo::DFloat : Numo::SFloat)
+    end
+
+    def self.validate_inputs!(signal, kernel)
+      unless signal.is_a?(Numo::NArray) && kernel.is_a?(Numo::NArray)
+        raise ArgumentError, 'signal and kernel must be Numo::NArray values'
+      end
+
+      return if [signal.class, kernel.class].all? { |type| INPUTS.include?(type) }
+
+      raise ArgumentError, 'unsupported signal or kernel dtype'
+    end
+
+    def self.validate_dtype!(dtype)
+      return if dtype.nil? || OUTPUTS.include?(dtype)
+
+      raise ArgumentError, 'dtype must be Numo::SFloat, Numo::DFloat, or nil'
+    end
+
+    def self.cast_fill(value, dtype)
+      # A scalar IEEE float conversion, without allocating a Numo buffer before preflight.
+      dtype == Numo::SFloat ? [value].pack('f').unpack1('f') : value
+    end
+  end
+
+  private_constant :OperationDtype
+end

data/lib/convolver/operation_execution.rb

@@ -4,26 +4,13 @@ module Convolver
   # Coordinates validation, implementation selection, and one operation family.
   # @private
   class OperationExecution
-    DIRECT_OPERATION_COST = { 1 => 6.8e-10, 2 => 5.9e-10, 3 => 4.5e-10 }.freeze
-    EXTENSION_COST = 1.0e-9
+    DIRECT_OPERATION_COST = { 1 => 3.8e-10, 2 => 5.9e-10, 3 => 4.5e-10 }.freeze
+    DOUBLE_OPERATION_COST = { 1 => 3.9e-10, 2 => 6.0e-10, 3 => 4.6e-10 }.freeze
+    LARGE_KERNEL_COST = 6.5e-10
     FFT_SPEEDUP_MARGIN = 0.8
+    CALIBRATED_FFT_SPEEDUP_MARGIN = 0.9
 
-    METHODS = {
-      correlation: {
-        basic: :correlate_basic,
-        fft: :correlate_fft,
-        predict_basic: :predict_correlate_basic_time,
-        predict_fft: :predict_correlate_fft_time
-      },
-      convolution: {
-        basic: :convolve_basic,
-        fft: :convolve_fft,
-        predict_basic: :predict_convolve_basic_time,
-        predict_fft: :predict_convolve_fft_time
-      }
-    }.freeze
-
-    def initialize(operation, signal, kernel, mode:, boundary:, fill_value:, origin:)
+    def initialize(operation, signal, kernel, mode:, boundary:, fill_value:, origin:, dtype:)
       @operation = operation
       @signal = signal
       @kernel = kernel
@@ -31,28 +18,31 @@ module Convolver
       @boundary = boundary
       @fill_value = fill_value
       @origin = origin
+      @dtype = dtype
     end
 
-    def automatic(receiver)
-      return invoke(receiver, :basic) if plan.extended_size < 1000
+    def automatic
+      return basic if plan.result_shape.empty?
 
-      direct_time = invoke(receiver, :predict_basic)
-      return invoke(receiver, :fft) if automatic_fft_time(receiver) < FFT_SPEEDUP_MARGIN * direct_time
+      threshold = fft_margin * basic_time
+      return basic if FftCost.lower_bound(plan, kernel.shape, threshold:) >= threshold
+      return fft if automatic_fft_time < threshold
 
-      invoke(receiver, :basic)
+      basic
     end
 
     def basic
       plan.validate_basic!
-      extended_signal = plan.extend_signal(signal)
-      return Convolver.send(:correlate_basic_valid, extended_signal, kernel) if operation == :correlation
+      prepared_signal, prepared_kernel = prepared_inputs
+      extended_signal = plan.extend_signal(prepared_signal)
+      return Convolver.send(:correlate_basic_valid, extended_signal, prepared_kernel) if operation == :correlation
 
-      Convolver.send(:convolve_basic_valid, extended_signal, kernel)
+      Convolver.send(:convolve_basic_valid, extended_signal, prepared_kernel)
     end
 
     def fft
       plan.fft_buffers
-      return Numo::SFloat.cast(signal) * Numo::SFloat.cast(kernel) if signal.ndim.zero?
+      return prepared_inputs.reduce(:*) if signal.ndim.zero?
 
       fft_operation.call
     end
@@ -63,43 +53,64 @@ module Convolver
 
     def basic_time
       plan.validate_basic!
-      calculation_cost = direct_operation_cost * plan.result_size * kernel.size
-      extension_cost = plan.valid? ? 0.0 : EXTENSION_COST * plan.extended_size
-      calculation_cost + extension_cost
+      calculation_cost + preparation_cost
     end
 
     private
 
-    attr_reader :operation, :signal, :kernel, :mode, :boundary, :fill_value, :origin
+    attr_reader :operation, :signal, :kernel, :mode, :boundary, :fill_value, :origin, :dtype
 
     def plan
-      @plan ||= OperationPlan.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+      @plan ||= OperationPlan.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
     end
 
-    def invoke(receiver, method)
-      receiver.public_send(METHODS.fetch(operation).fetch(method), signal, kernel, **public_options)
-    end
-
-    def public_options
-      options = { mode:, boundary:, origin: }
-      options[:fill_value] = fill_value unless fill_value.equal?(UNSPECIFIED_FILL)
-      options
+    def prepared_inputs
+      @prepared_inputs ||= [plan.dtype.cast(signal), plan.dtype.cast(kernel)]
     end
 
     def fft_operation
-      return CircularFftOperation.new(operation, signal, kernel, plan) if plan.wrap?
+      prepared_signal, prepared_kernel = prepared_inputs
+      return CircularFftOperation.new(operation, prepared_signal, prepared_kernel, plan) if plan.wrap?
 
-      LinearFftOperation.new(operation, plan.extend_signal(signal), kernel, plan)
+      LinearFftOperation.new(operation, plan.extend_signal(prepared_signal), prepared_kernel, plan)
     end
 
-    def automatic_fft_time(receiver)
-      invoke(receiver, :predict_fft)
+    def automatic_fft_time
+      fft_time
     rescue FftUnavailable
       Float::INFINITY
     end
 
+    def preparation_cost
+      costs = PreparationCost.new(plan)
+      costs.conversion + costs.extension
+    end
+
     def direct_operation_cost
-      DIRECT_OPERATION_COST.fetch(signal.ndim, DIRECT_OPERATION_COST.values.last)
+      return LARGE_KERNEL_COST if kernel.size > 64 && signal.ndim != 2
+
+      costs = plan.dtype == Numo::DFloat ? DOUBLE_OPERATION_COST : DIRECT_OPERATION_COST
+      costs.fetch(signal.ndim, costs.values.last)
+    end
+
+    def calculation_cost
+      return calibrated_calculation_cost if plan.cost_profile
+
+      direct_operation_cost * plan.result_size * kernel.size
+    end
+
+    def calibrated_calculation_cost
+      fixed, short, long = plan.cost_profile.fetch(:direct).fetch(plan.dtype == Numo::DFloat ? 1 : 0)
+      fixed + (kernel_work(short, long) * plan.result_size)
+    end
+
+    def kernel_work(short, long)
+      size = kernel.size
+      (short * [size, 64].min) + (long * [size - 64, 0].max)
+    end
+
+    def fft_margin
+      plan.cost_profile ? CALIBRATED_FFT_SPEEDUP_MARGIN : FFT_SPEEDUP_MARGIN
     end
   end
 

data/lib/convolver/operation_options.rb

@@ -8,21 +8,20 @@ module Convolver
     MODES = %i[valid same full].freeze
     BOUNDARIES = %i[constant nearest reflect mirror wrap].freeze
 
-    attr_reader :operation, :mode, :boundary, :fill_value, :origins, :anchors
+    attr_reader :operation, :mode, :boundary, :fill_value, :origins, :anchors, :dtype
 
-    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
+      @dtype = OperationDtype.resolve(signal, kernel, dtype)
       validate_inputs!(signal, kernel)
       validate_vocabulary!(operation, mode, boundary)
       assign_options(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
       validate_combinations!(signal.shape, kernel.shape)
+      @fill_value = OperationDtype.cast_fill(@fill_value, @dtype)
     end
 
     private
 
     def validate_inputs!(signal, kernel)
-      unless signal.is_a?(Numo::NArray) && kernel.is_a?(Numo::NArray)
-        raise ArgumentError, 'signal and kernel must be Numo::NArray values'
-      end
       raise ArgumentError, 'signal and kernel must not be empty' if signal.empty? || kernel.empty?
       raise ArgumentError, 'signal and kernel must have equal rank' unless signal.ndim == kernel.ndim
       raise ArgumentError, "maximum supported rank is #{MAX_RANK}" if signal.ndim > MAX_RANK

data/lib/convolver/operation_plan.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'convolver/cost_profiles'
+require 'convolver/operation_dtype'
 require 'convolver/operation_options'
 require 'convolver/operation_shapes'
 require 'convolver/signal_extension'
@@ -18,15 +20,19 @@ module Convolver
   class OperationPlan
     extend Forwardable
 
-    def_delegators :options, :operation, :mode, :boundary, :fill_value, :origins, :anchors
+    def_delegators :options, :operation, :mode, :boundary, :fill_value, :origins, :anchors, :dtype
     def_delegators :shapes, :padding_before, :padding_after, :result_shape,
                    :extended_shape, :result_size, :extended_size, :linear_fft_shape,
                    :linear_fft_size, :linear_spectrum_size
 
-    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
-      @options = OperationOptions.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
+    attr_reader :conversion_size, :cost_profile
+
+    def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype: nil)
+      @options = OperationOptions.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
+      @conversion_size = input_conversion_size(signal, kernel)
       @signal_shape = signal.shape.freeze
       @kernel_shape = kernel.shape.freeze
+      @cost_profile = CostProfiles.for_rank(@signal_shape.length)
       @shapes = OperationShapes.new(
         signal.shape, kernel.shape, operation:, mode:, anchors: options.anchors
       )
@@ -47,7 +53,10 @@ module Convolver
     end
 
     def validate_basic!
+      return if @basic_validated
+
       BufferPreparation.new(self, @signal_shape, @kernel_shape).basic!
+      @basic_validated = true
     end
 
     def fft_buffers
@@ -57,6 +66,10 @@ module Convolver
     private
 
     attr_reader :options, :shapes
+
+    def input_conversion_size(signal, kernel)
+      [signal, kernel].sum { |input| input.instance_of?(dtype) ? 0 : input.size }
+    end
   end
 
   private_constant :OperationPlan, :UNSPECIFIED_FILL

data/lib/convolver/preparation_cost.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Convolver
+  # Shared estimates for input copies and Ruby boundary/folding work.
+  # @private
+  class PreparationCost
+    CONVERSION_COST = 7.5e-10
+    EXTENSION_COST = 1.0e-9
+    INDEX_COST = { nearest: 1.45e-7, reflect: 1.33e-7, mirror: 1.54e-7, wrap: 1.05e-7 }.freeze
+    FOLD_COST = { 1 => 3.3e-7, 2 => 3.7e-7, 3 => 5.5e-7 }.freeze
+
+    def initialize(plan)
+      @plan = plan
+    end
+
+    def conversion
+      CONVERSION_COST * plan.conversion_size
+    end
+
+    def extension
+      return 0.0 if plan.valid?
+
+      (EXTENSION_COST * plan.extended_size * (plan.dtype::ELEMENT_BYTE_SIZE / 4)) + indices
+    end
+
+    def indices
+      return 0.0 if plan.valid? || plan.boundary == :constant
+
+      costs = plan.cost_profile ? plan.cost_profile.fetch(:indices) : INDEX_COST
+      costs.fetch(plan.boundary) * plan.extended_shape.sum
+    end
+
+    def folding(kernel_size)
+      rank = plan.result_shape.length
+      return plan.cost_profile.fetch(:folding).fetch(rank - 1) * kernel_size if plan.cost_profile
+
+      FOLD_COST.fetch(rank, FOLD_COST.values.last) * kernel_size
+    end
+
+    private
+
+    attr_reader :plan
+  end
+
+  private_constant :PreparationCost
+end

data/lib/convolver/real_fft_shape.rb

@@ -39,10 +39,7 @@ module Convolver
     end
 
     def fast_shape(exact_shape)
-      fast_shape = exact_shape.map.with_index do |size, axis|
-        next_fast_size(size, even: axis == exact_shape.length - 1)
-      end
-      fast_shape if fast_shape.all?
+      Convolver.send(:fft_fast_shape, exact_shape, size_max)
     end
 
     def even_final_axis(shape)
@@ -51,64 +48,8 @@ module Convolver
       result.freeze
     end
 
-    def next_fast_size(target, even:)
-      best = nil
-      each_power(5) do |power_of_five|
-        each_power(3, power_of_five) do |power_of_three|
-          candidate = doubled_candidate(power_of_three, target, even)
-          best = candidate if candidate && (!best || candidate < best)
-        end
-      end
-      best
-    end
-
-    def each_power(factor, initial = 1)
-      value = initial
-      loop do
-        yield value
-        break if value > size_max / factor
-
-        value *= factor
-      end
-    end
-
-    def doubled_candidate(initial, target, even)
-      candidate = initial
-      while candidate < target || (even && candidate.odd?)
-        return nil if candidate > size_max / 2
-
-        candidate *= 2
-      end
-      candidate
-    end
-
     def transform_cost(shape)
-      shape.reduce(:*) * shape.sum { |size| Math.log2(size) + large_factor_penalty(size) }
-    end
-
-    def large_factor_penalty(size)
-      prime_factors(size).sum do |factor|
-        factor > 7 ? 1.5 * (Math.log2(factor) - 3) : 0.0
-      end
-    end
-
-    def prime_factors(value)
-      factors = []
-      divisor = 2
-      while divisor * divisor <= value
-        value = extract_factor(value, divisor, factors)
-        divisor += divisor == 2 ? 1 : 2
-      end
-      factors << value if value > 1
-      factors
-    end
-
-    def extract_factor(value, divisor, factors)
-      while (value % divisor).zero?
-        factors << divisor
-        value /= divisor
-      end
-      value
+      Convolver.send(:fft_shape_cost, shape)
     end
 
     def representable?(shape)

data/lib/convolver/signal_extension.rb

@@ -11,7 +11,7 @@ module Convolver
     end
 
     def call(signal)
-      source = Numo::SFloat.cast(signal)
+      source = signal
       return constant_extension(source) if boundary == :constant
 
       source[*extension_indices]
@@ -22,7 +22,7 @@ module Convolver
     attr_reader :shapes, :boundary, :fill_value
 
     def constant_extension(source)
-      extended = Numo::SFloat.new(*shapes.extended_shape).fill(fill_value)
+      extended = source.class.new(*shapes.extended_shape).fill(fill_value)
       ranges = source.shape.zip(shapes.padding_before).map do |signal_size, before|
         before...(before + signal_size)
       end

data/lib/convolver/version.rb

@@ -2,5 +2,5 @@
 
 module Convolver
   # Current gem version.
-  VERSION = '3.0.1'
+  VERSION = '4.0.0'
 end