convolver 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +84 -2
- data/README.md +150 -51
- data/ext/convolver/convolve_raw.c +51 -163
- data/ext/convolver/convolve_raw.h +5 -16
- data/ext/convolver/convolver.c +90 -24
- data/ext/convolver/correlate_raw.c +57 -0
- data/ext/convolver/correlate_raw.h +14 -0
- data/ext/convolver/extconf.rb +1 -0
- data/ext/convolver/fft_shape.c +116 -0
- data/ext/convolver/fft_shape.h +8 -0
- data/ext/convolver/raw_config.h +11 -0
- data/ext/convolver/raw_helpers.h +71 -0
- data/ext/convolver/raw_loop.h +93 -0
- data/lib/convolver/buffer_limits.rb +39 -0
- data/lib/convolver/buffer_preparation.rb +56 -0
- data/lib/convolver/circular_fft_operation.rb +84 -0
- data/lib/convolver/cost_profiles.rb +44 -0
- data/lib/convolver/fft_buffer_plan.rb +39 -0
- data/lib/convolver/fft_cost.rb +52 -0
- data/lib/convolver/fft_estimator.rb +64 -0
- data/lib/convolver/fft_limits.rb +64 -0
- data/lib/convolver/linear_fft_operation.rb +52 -0
- data/lib/convolver/operation_dtype.rb +42 -0
- data/lib/convolver/operation_execution.rb +118 -0
- data/lib/convolver/operation_options.rb +13 -10
- data/lib/convolver/operation_plan.rb +34 -6
- data/lib/convolver/operation_shapes.rb +42 -9
- data/lib/convolver/preparation_cost.rb +46 -0
- data/lib/convolver/real_fft_shape.rb +76 -0
- data/lib/convolver/signal_extension.rb +3 -2
- data/lib/convolver/version.rb +1 -1
- data/lib/convolver.rb +101 -114
- metadata +28 -8
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Convolver
|
|
4
|
+
# Calculates same-sized periodic operations through PocketFFT transforms.
|
|
5
|
+
# @private
|
|
6
|
+
class CircularFftOperation
|
|
7
|
+
def initialize(operation, signal, kernel, plan)
|
|
8
|
+
@dtype = plan.dtype
|
|
9
|
+
@real_axis = plan.fft_buffers.real_axis
|
|
10
|
+
@operation = operation
|
|
11
|
+
@signal = Numo::DFloat.cast(signal)
|
|
12
|
+
@kernel = fold_kernel(Numo::DFloat.cast(kernel), signal.shape, plan.anchors)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call
|
|
16
|
+
return complex_product unless real_axis
|
|
17
|
+
|
|
18
|
+
real_product(real_axis)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.real_axis(shape)
|
|
22
|
+
return nil if shape.empty?
|
|
23
|
+
|
|
24
|
+
final_axis = shape.length - 1
|
|
25
|
+
return final_axis if shape[final_axis].even?
|
|
26
|
+
|
|
27
|
+
shape.each_index.select { |axis| shape[axis].even? }.max_by { |axis| shape[axis] }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
attr_reader :operation, :signal, :kernel, :real_axis, :dtype
|
|
33
|
+
|
|
34
|
+
def real_product(real_axis)
|
|
35
|
+
final_axis = signal.ndim - 1
|
|
36
|
+
prepared_signal = prepare_real_operand(signal, real_axis, final_axis)
|
|
37
|
+
prepared_kernel = prepare_real_operand(kernel, real_axis, final_axis)
|
|
38
|
+
result = real_spectrum_product(prepared_signal, prepared_kernel)
|
|
39
|
+
result = result.swapaxes(real_axis, final_axis) unless real_axis == final_axis
|
|
40
|
+
dtype.cast(result)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def prepare_real_operand(value, real_axis, final_axis)
|
|
44
|
+
real_axis == final_axis ? value : value.swapaxes(real_axis, final_axis)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def real_spectrum_product(signal_value, kernel_value)
|
|
48
|
+
signal_spectrum = Numo::Pocketfft.rfftn(signal_value)
|
|
49
|
+
kernel_spectrum = Numo::Pocketfft.rfftn(kernel_value)
|
|
50
|
+
kernel_spectrum.inplace.conj if operation == :correlation
|
|
51
|
+
Numo::Pocketfft.irfftn(signal_spectrum.inplace * kernel_spectrum)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def complex_product
|
|
55
|
+
signal_spectrum = Numo::Pocketfft.fftn(signal)
|
|
56
|
+
kernel_spectrum = Numo::Pocketfft.fftn(kernel)
|
|
57
|
+
kernel_spectrum.inplace.conj if operation == :correlation
|
|
58
|
+
result = Numo::Pocketfft.ifftn(signal_spectrum.inplace * kernel_spectrum).real
|
|
59
|
+
dtype.cast(result)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def fold_kernel(source, signal_shape, anchors)
|
|
63
|
+
folded = Numo::DFloat.zeros(*signal_shape)
|
|
64
|
+
source.flatten.each_with_index do |value, flat_index|
|
|
65
|
+
target = folded_target(flat_index, source.shape, signal_shape, anchors)
|
|
66
|
+
folded[*target] = folded[*target] + value
|
|
67
|
+
end
|
|
68
|
+
folded
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def folded_target(flat_index, kernel_shape, signal_shape, anchors)
|
|
72
|
+
remainder = flat_index
|
|
73
|
+
Array.new(kernel_shape.length).tap do |target|
|
|
74
|
+
(kernel_shape.length - 1).downto(0) do |axis|
|
|
75
|
+
coordinate = remainder % kernel_shape[axis]
|
|
76
|
+
remainder /= kernel_shape[axis]
|
|
77
|
+
target[axis] = (coordinate - anchors[axis]) % signal_shape[axis]
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private_constant :CircularFftOperation
|
|
84
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Convolver
|
|
4
|
+
# Immutable measured-environment coefficients in seconds; ranks one through three.
|
|
5
|
+
# @private
|
|
6
|
+
module CostProfiles
|
|
7
|
+
MAC = {
|
|
8
|
+
direct: [[6.55e-6, 3.24e-10, 7.08e-10].freeze, [6.55e-6, 3.24e-10, 7.08e-10].freeze].freeze,
|
|
9
|
+
fixed: [1.64e-5, 5.12e-5, 1.13e-4].freeze,
|
|
10
|
+
transform: [8.02e-10, 7.17e-10, 1.54e-9].freeze,
|
|
11
|
+
indices: { reflect: 1.33e-7, nearest: 1.46e-7, mirror: 1.57e-7, wrap: 3.23e-8 }.freeze,
|
|
12
|
+
folding: [3.78e-7, 3.99e-7, 5.09e-7].freeze
|
|
13
|
+
}.freeze
|
|
14
|
+
X86 = {
|
|
15
|
+
direct: [[1.59e-5, 3.08e-10, 2.52e-10].freeze, [1.63e-5, 3.22e-10, 3.96e-10].freeze].freeze,
|
|
16
|
+
fixed: [3.77e-5, 1.20e-4, 1.98e-4].freeze,
|
|
17
|
+
transform: [2.00e-9, 2.13e-9, 3.13e-9].freeze,
|
|
18
|
+
indices: { reflect: 2.32e-7, nearest: 2.88e-7, mirror: 2.62e-7, wrap: 1.11e-7 }.freeze,
|
|
19
|
+
folding: [8.20e-7, 7.51e-7, 9.79e-7].freeze
|
|
20
|
+
}.freeze
|
|
21
|
+
ARM = {
|
|
22
|
+
direct: [[1.88e-5, 5.12e-10, 5.96e-10].freeze, [1.88e-5, 5.12e-10, 5.96e-10].freeze].freeze,
|
|
23
|
+
fixed: [4.20e-5, 1.20e-4, 2.70e-4].freeze,
|
|
24
|
+
transform: [3.29e-9, 2.89e-9, 3.74e-9].freeze,
|
|
25
|
+
indices: { reflect: 2.62e-7, nearest: 3.67e-7, mirror: 2.87e-7, wrap: 1.50e-7 }.freeze,
|
|
26
|
+
folding: [9.97e-7, 8.80e-7, 1.24e-6].freeze
|
|
27
|
+
}.freeze
|
|
28
|
+
def self.for_platform(platform)
|
|
29
|
+
case platform
|
|
30
|
+
when /\Aarm64-darwin/ then MAC
|
|
31
|
+
when /\Ax86_64-linux/ then X86
|
|
32
|
+
when /\Aaarch64-linux/ then ARM
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
CURRENT = for_platform(RUBY_PLATFORM)
|
|
37
|
+
|
|
38
|
+
def self.for_rank(rank)
|
|
39
|
+
CURRENT if (1..3).cover?(rank)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private_constant :CostProfiles
|
|
44
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'convolver/fft_limits'
|
|
4
|
+
require 'convolver/buffer_preparation'
|
|
5
|
+
|
|
6
|
+
module Convolver
|
|
7
|
+
# Shares allocation-free FFT dimensions between execution and estimation.
|
|
8
|
+
# @private
|
|
9
|
+
class FftBufferPlan
|
|
10
|
+
attr_reader :transform_shape, :real_axis, :transform_size, :spectrum_size
|
|
11
|
+
|
|
12
|
+
def initialize(plan, signal_shape, kernel_shape, limits: BufferLimits::NATIVE)
|
|
13
|
+
BufferPreparation.new(plan, signal_shape, kernel_shape, limits:).fft!
|
|
14
|
+
@real_axis = plan.wrap? ? CircularFftOperation.real_axis(signal_shape) : signal_shape.length - 1
|
|
15
|
+
@transform_shape = select_shape(plan, signal_shape, kernel_shape, FftLimits.new(limits))
|
|
16
|
+
@transform_size = transform_shape.reduce(1, :*)
|
|
17
|
+
@spectrum_size = spectrum_elements
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def select_shape(plan, signal_shape, kernel_shape, limits)
|
|
23
|
+
return [].freeze if signal_shape.empty?
|
|
24
|
+
return plan.linear_fft_shape(kernel_shape, limits:) unless plan.wrap?
|
|
25
|
+
|
|
26
|
+
limits.validate!(signal_shape, real_axis:)
|
|
27
|
+
signal_shape.dup.freeze
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def spectrum_elements
|
|
31
|
+
return transform_size if transform_shape.empty? || real_axis.nil?
|
|
32
|
+
|
|
33
|
+
length = transform_shape[real_axis]
|
|
34
|
+
transform_size / length * ((length / 2) + 1)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private_constant :FftBufferPlan
|
|
39
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'convolver/preparation_cost'
|
|
4
|
+
|
|
5
|
+
module Convolver
|
|
6
|
+
# Shared transform costs and a lower bound that never constructs an FFT plan.
|
|
7
|
+
# @private
|
|
8
|
+
class FftCost
|
|
9
|
+
REAL = { 1 => 1.0e-9, 2 => 1.0e-9, 3 => 1.5e-9 }.freeze
|
|
10
|
+
COMPLEX = { 1 => 2.4e-9, 2 => 2.4e-9, 3 => 2.8e-9 }.freeze
|
|
11
|
+
FIXED = { 0 => 5.0e-6, 1 => 1.5e-5, 2 => 3.0e-5, 3 => 1.0e-4 }.freeze
|
|
12
|
+
PREPARATION = 1.0e-9
|
|
13
|
+
|
|
14
|
+
def self.fixed(rank, profile: nil)
|
|
15
|
+
return profile.fetch(:fixed).fetch(rank - 1) if profile
|
|
16
|
+
|
|
17
|
+
FIXED.fetch(rank, FIXED.values.last)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.transform(size, rank, real: true, profile: nil)
|
|
21
|
+
coefficient(rank, real:, profile:) * size * Math.log([size, 2].max)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.lower_bound(plan, kernel_shape, threshold:)
|
|
25
|
+
rank = kernel_shape.length
|
|
26
|
+
profile = plan.cost_profile
|
|
27
|
+
setup = fixed(rank, profile:)
|
|
28
|
+
return setup if setup >= threshold || plan.wrap? || rank.zero?
|
|
29
|
+
|
|
30
|
+
size = minimum_size(plan.extended_shape, kernel_shape)
|
|
31
|
+
setup + transform(size, rank, profile:) + (2 * PREPARATION * size) + PreparationCost.new(plan).indices
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.coefficient(rank, real:, profile:)
|
|
35
|
+
return profile.fetch(:transform).fetch(rank - 1) * (real ? 1 : 2.2) if profile
|
|
36
|
+
|
|
37
|
+
costs = real ? REAL : COMPLEX
|
|
38
|
+
costs.fetch(rank, costs.values.last)
|
|
39
|
+
end
|
|
40
|
+
private_class_method :coefficient
|
|
41
|
+
|
|
42
|
+
def self.minimum_size(signal_shape, kernel_shape)
|
|
43
|
+
# Ruby integers avoid overflow: this is only a cost bound, never an allocation.
|
|
44
|
+
shape = signal_shape.zip(kernel_shape).map { |signal, kernel| signal + kernel - 1 }
|
|
45
|
+
shape[-1] += 1 if shape[-1].odd?
|
|
46
|
+
shape.reduce(1, :*)
|
|
47
|
+
end
|
|
48
|
+
private_class_method :minimum_size
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private_constant :FftCost
|
|
52
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'convolver/fft_cost'
|
|
4
|
+
|
|
5
|
+
module Convolver
|
|
6
|
+
# Estimates the selected real or complex PocketFFT implementation cost.
|
|
7
|
+
# @private
|
|
8
|
+
class FftEstimator
|
|
9
|
+
AXIS_MOVE_COST = 1.6e-9
|
|
10
|
+
RESULT_COST = 5.0e-10
|
|
11
|
+
CORRELATION_COST = 2.0e-10
|
|
12
|
+
|
|
13
|
+
def initialize(operation, signal, kernel, plan)
|
|
14
|
+
@operation = operation
|
|
15
|
+
@signal = signal
|
|
16
|
+
@kernel = kernel
|
|
17
|
+
@plan = plan
|
|
18
|
+
@profile = plan.cost_profile
|
|
19
|
+
@buffers = plan.fft_buffers
|
|
20
|
+
@preparation = PreparationCost.new(plan)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call
|
|
24
|
+
return FftCost.fixed(0) + preparation.conversion + result_cost if signal.ndim.zero?
|
|
25
|
+
|
|
26
|
+
transform_cost + preparation_cost + extra_cost
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
attr_reader :operation, :signal, :kernel, :plan, :buffers, :preparation
|
|
32
|
+
|
|
33
|
+
def transform_cost
|
|
34
|
+
transform = FftCost.transform(buffers.transform_size, signal.ndim,
|
|
35
|
+
real: !buffers.real_axis.nil?, profile: @profile)
|
|
36
|
+
FftCost.fixed(signal.ndim, profile: @profile) + transform + (AXIS_MOVE_COST * moved_size)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def extra_cost
|
|
40
|
+
correlation_cost = operation == :correlation ? CORRELATION_COST * buffers.spectrum_size : 0.0
|
|
41
|
+
correlation_cost + preparation.conversion + result_cost
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def result_cost
|
|
45
|
+
RESULT_COST * plan.result_size * (plan.dtype::ELEMENT_BYTE_SIZE / 4)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def preparation_cost
|
|
49
|
+
return circular_preparation_cost if plan.wrap?
|
|
50
|
+
|
|
51
|
+
(2 * FftCost::PREPARATION * buffers.transform_size) + preparation.extension
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def circular_preparation_cost
|
|
55
|
+
(FftCost::PREPARATION * (signal.size + kernel.size)) + preparation.folding(kernel.size)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def moved_size
|
|
59
|
+
buffers.real_axis && buffers.real_axis != signal.ndim - 1 ? 3 * signal.size : 0
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private_constant :FftEstimator
|
|
64
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'convolver/buffer_limits'
|
|
4
|
+
|
|
5
|
+
# Internal allocation and arithmetic checks for the PocketFFT paths.
|
|
6
|
+
module Convolver
|
|
7
|
+
# Only FFT representability failures permit automatic direct fallback.
|
|
8
|
+
# @private
|
|
9
|
+
class FftUnavailable < RangeError; end
|
|
10
|
+
|
|
11
|
+
# Bounds PocketFFT buffers and signed arithmetic without constructing plans.
|
|
12
|
+
# @private
|
|
13
|
+
class FftLimits
|
|
14
|
+
def initialize(buffers = BufferLimits::NATIVE)
|
|
15
|
+
@buffers = buffers
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def validate!(shape, real_axis:)
|
|
19
|
+
total = buffers.product!(shape, 'FFT size', error: FftUnavailable)
|
|
20
|
+
validate_buffers!(shape, total, real_axis)
|
|
21
|
+
# The wrapper casts total to int before dividing it by the axis length.
|
|
22
|
+
integer!(total, buffers.int_max, 'FFT total size')
|
|
23
|
+
# Real forward computes int z_step = (z / 2 + 1) * 2.
|
|
24
|
+
integer!(shape[real_axis], buffers.int_max - 2, 'FFT real stride') if real_axis
|
|
25
|
+
validate_axes!(shape)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
attr_reader :buffers
|
|
31
|
+
|
|
32
|
+
def validate_axes!(shape)
|
|
33
|
+
# FFTPACK twiddle counts are <= 2n complex / 3n real. Bluestein m < 4n,
|
|
34
|
+
# so each variable allocation is <= 128n bytes, including inner twiddles.
|
|
35
|
+
# The 16n int bound also protects signed intermediates at the inner m.
|
|
36
|
+
shape.each do |length|
|
|
37
|
+
integer!(length, buffers.int_max / 16, 'FFT axis arithmetic')
|
|
38
|
+
buffer!(length, 128, 'FFT axis work buffer')
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def validate_buffers!(shape, total, real_axis)
|
|
43
|
+
buffer!(total, 8, 'FFT spatial buffer')
|
|
44
|
+
return buffer!(total, 16, 'FFT complex buffer') unless real_axis
|
|
45
|
+
|
|
46
|
+
spectrum = total / shape[real_axis] * ((shape[real_axis] / 2) + 1)
|
|
47
|
+
buffer!(spectrum, 16, 'FFT spectrum buffer')
|
|
48
|
+
buffer!(total, 16, 'FFT inverse staging buffer')
|
|
49
|
+
buffer!(total, 8, 'FFT inverse output buffer')
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def buffer!(count, width, description)
|
|
53
|
+
buffers.buffer!(count, width, description, error: FftUnavailable)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def integer!(value, maximum, description)
|
|
57
|
+
return if value <= maximum
|
|
58
|
+
|
|
59
|
+
raise FftUnavailable, "#{description} exceeds native integer limit"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private_constant :FftUnavailable, :FftLimits
|
|
64
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Convolver
|
|
4
|
+
# Calculates a valid linear operation through real PocketFFT transforms.
|
|
5
|
+
# @private
|
|
6
|
+
class LinearFftOperation
|
|
7
|
+
def initialize(operation, signal, kernel, plan)
|
|
8
|
+
@dtype = plan.dtype
|
|
9
|
+
@operation = operation
|
|
10
|
+
@signal = signal
|
|
11
|
+
@kernel = kernel
|
|
12
|
+
@transform_shape = plan.fft_buffers.transform_shape
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call
|
|
16
|
+
signal_padded = zero_padded(signal)
|
|
17
|
+
kernel_padded = zero_padded(kernel)
|
|
18
|
+
full_result = real_spectrum_product(signal_padded, kernel_padded)
|
|
19
|
+
cropped = full_result[*valid_ranges]
|
|
20
|
+
dtype == Numo::DFloat ? cropped.dup : dtype.cast(cropped)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
attr_reader :operation, :signal, :kernel, :transform_shape, :dtype
|
|
26
|
+
|
|
27
|
+
def zero_padded(value)
|
|
28
|
+
Numo::DFloat.zeros(*transform_shape).tap do |padded|
|
|
29
|
+
padded[*value.shape.map { |size| 0...size }] = value
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def real_spectrum_product(signal_value, kernel_value)
|
|
34
|
+
signal_spectrum = Numo::Pocketfft.rfftn(signal_value)
|
|
35
|
+
kernel_spectrum = Numo::Pocketfft.rfftn(kernel_value)
|
|
36
|
+
kernel_spectrum.inplace.conj if operation == :correlation
|
|
37
|
+
Numo::Pocketfft.irfftn(signal_spectrum.inplace * kernel_spectrum)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def valid_ranges
|
|
41
|
+
signal.shape.zip(kernel.shape).map do |signal_size, kernel_size|
|
|
42
|
+
if operation == :correlation
|
|
43
|
+
0...(signal_size - kernel_size + 1)
|
|
44
|
+
else
|
|
45
|
+
(kernel_size - 1)...signal_size
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private_constant :LinearFftOperation
|
|
52
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Convolver
|
|
4
|
+
# Resolves public precision without allocating operand copies.
|
|
5
|
+
# @private
|
|
6
|
+
class OperationDtype
|
|
7
|
+
SINGLE_INPUTS = [Numo::SFloat, Numo::Int8, Numo::UInt8, Numo::Int16, Numo::UInt16].freeze
|
|
8
|
+
DOUBLE_INPUTS = [Numo::DFloat, Numo::Int32, Numo::UInt32, Numo::Int64, Numo::UInt64].freeze
|
|
9
|
+
INPUTS = (SINGLE_INPUTS + DOUBLE_INPUTS).freeze
|
|
10
|
+
OUTPUTS = [Numo::SFloat, Numo::DFloat].freeze
|
|
11
|
+
|
|
12
|
+
def self.resolve(signal, kernel, dtype)
|
|
13
|
+
validate_inputs!(signal, kernel)
|
|
14
|
+
validate_dtype!(dtype)
|
|
15
|
+
types = [signal.class, kernel.class]
|
|
16
|
+
dtype || (types.any? { |type| DOUBLE_INPUTS.include?(type) } ? Numo::DFloat : Numo::SFloat)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.validate_inputs!(signal, kernel)
|
|
20
|
+
unless signal.is_a?(Numo::NArray) && kernel.is_a?(Numo::NArray)
|
|
21
|
+
raise ArgumentError, 'signal and kernel must be Numo::NArray values'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
return if [signal.class, kernel.class].all? { |type| INPUTS.include?(type) }
|
|
25
|
+
|
|
26
|
+
raise ArgumentError, 'unsupported signal or kernel dtype'
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.validate_dtype!(dtype)
|
|
30
|
+
return if dtype.nil? || OUTPUTS.include?(dtype)
|
|
31
|
+
|
|
32
|
+
raise ArgumentError, 'dtype must be Numo::SFloat, Numo::DFloat, or nil'
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.cast_fill(value, dtype)
|
|
36
|
+
# A scalar IEEE float conversion, without allocating a Numo buffer before preflight.
|
|
37
|
+
dtype == Numo::SFloat ? [value].pack('f').unpack1('f') : value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private_constant :OperationDtype
|
|
42
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Convolver
|
|
4
|
+
# Coordinates validation, implementation selection, and one operation family.
|
|
5
|
+
# @private
|
|
6
|
+
class OperationExecution
|
|
7
|
+
DIRECT_OPERATION_COST = { 1 => 3.8e-10, 2 => 5.9e-10, 3 => 4.5e-10 }.freeze
|
|
8
|
+
DOUBLE_OPERATION_COST = { 1 => 3.9e-10, 2 => 6.0e-10, 3 => 4.6e-10 }.freeze
|
|
9
|
+
LARGE_KERNEL_COST = 6.5e-10
|
|
10
|
+
FFT_SPEEDUP_MARGIN = 0.8
|
|
11
|
+
CALIBRATED_FFT_SPEEDUP_MARGIN = 0.9
|
|
12
|
+
|
|
13
|
+
def initialize(operation, signal, kernel, mode:, boundary:, fill_value:, origin:, dtype:)
|
|
14
|
+
@operation = operation
|
|
15
|
+
@signal = signal
|
|
16
|
+
@kernel = kernel
|
|
17
|
+
@mode = mode
|
|
18
|
+
@boundary = boundary
|
|
19
|
+
@fill_value = fill_value
|
|
20
|
+
@origin = origin
|
|
21
|
+
@dtype = dtype
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def automatic
|
|
25
|
+
return basic if plan.result_shape.empty?
|
|
26
|
+
|
|
27
|
+
threshold = fft_margin * basic_time
|
|
28
|
+
return basic if FftCost.lower_bound(plan, kernel.shape, threshold:) >= threshold
|
|
29
|
+
return fft if automatic_fft_time < threshold
|
|
30
|
+
|
|
31
|
+
basic
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def basic
|
|
35
|
+
plan.validate_basic!
|
|
36
|
+
prepared_signal, prepared_kernel = prepared_inputs
|
|
37
|
+
extended_signal = plan.extend_signal(prepared_signal)
|
|
38
|
+
return Convolver.send(:correlate_basic_valid, extended_signal, prepared_kernel) if operation == :correlation
|
|
39
|
+
|
|
40
|
+
Convolver.send(:convolve_basic_valid, extended_signal, prepared_kernel)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def fft
|
|
44
|
+
plan.fft_buffers
|
|
45
|
+
return prepared_inputs.reduce(:*) if signal.ndim.zero?
|
|
46
|
+
|
|
47
|
+
fft_operation.call
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def fft_time
|
|
51
|
+
FftEstimator.new(operation, signal, kernel, plan).call
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def basic_time
|
|
55
|
+
plan.validate_basic!
|
|
56
|
+
calculation_cost + preparation_cost
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
attr_reader :operation, :signal, :kernel, :mode, :boundary, :fill_value, :origin, :dtype
|
|
62
|
+
|
|
63
|
+
def plan
|
|
64
|
+
@plan ||= OperationPlan.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def prepared_inputs
|
|
68
|
+
@prepared_inputs ||= [plan.dtype.cast(signal), plan.dtype.cast(kernel)]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def fft_operation
|
|
72
|
+
prepared_signal, prepared_kernel = prepared_inputs
|
|
73
|
+
return CircularFftOperation.new(operation, prepared_signal, prepared_kernel, plan) if plan.wrap?
|
|
74
|
+
|
|
75
|
+
LinearFftOperation.new(operation, plan.extend_signal(prepared_signal), prepared_kernel, plan)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def automatic_fft_time
|
|
79
|
+
fft_time
|
|
80
|
+
rescue FftUnavailable
|
|
81
|
+
Float::INFINITY
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def preparation_cost
|
|
85
|
+
costs = PreparationCost.new(plan)
|
|
86
|
+
costs.conversion + costs.extension
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def direct_operation_cost
|
|
90
|
+
return LARGE_KERNEL_COST if kernel.size > 64 && signal.ndim != 2
|
|
91
|
+
|
|
92
|
+
costs = plan.dtype == Numo::DFloat ? DOUBLE_OPERATION_COST : DIRECT_OPERATION_COST
|
|
93
|
+
costs.fetch(signal.ndim, costs.values.last)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def calculation_cost
|
|
97
|
+
return calibrated_calculation_cost if plan.cost_profile
|
|
98
|
+
|
|
99
|
+
direct_operation_cost * plan.result_size * kernel.size
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def calibrated_calculation_cost
|
|
103
|
+
fixed, short, long = plan.cost_profile.fetch(:direct).fetch(plan.dtype == Numo::DFloat ? 1 : 0)
|
|
104
|
+
fixed + (kernel_work(short, long) * plan.result_size)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def kernel_work(short, long)
|
|
108
|
+
size = kernel.size
|
|
109
|
+
(short * [size, 64].min) + (long * [size - 64, 0].max)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def fft_margin
|
|
113
|
+
plan.cost_profile ? CALIBRATED_FFT_SPEEDUP_MARGIN : FFT_SPEEDUP_MARGIN
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private_constant :OperationExecution
|
|
118
|
+
end
|
|
@@ -1,39 +1,42 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Convolver
|
|
4
|
-
# Validates and normalizes
|
|
4
|
+
# Validates and normalizes one convolution or correlation operation.
|
|
5
|
+
# @private
|
|
5
6
|
class OperationOptions
|
|
7
|
+
OPERATIONS = %i[convolution correlation].freeze
|
|
6
8
|
MODES = %i[valid same full].freeze
|
|
7
9
|
BOUNDARIES = %i[constant nearest reflect mirror wrap].freeze
|
|
8
10
|
|
|
9
|
-
attr_reader :mode, :boundary, :fill_value, :origins, :anchors
|
|
11
|
+
attr_reader :operation, :mode, :boundary, :fill_value, :origins, :anchors, :dtype
|
|
10
12
|
|
|
11
|
-
def initialize(signal, kernel, mode:, boundary:, fill_value:, origin:)
|
|
13
|
+
def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
|
|
14
|
+
@dtype = OperationDtype.resolve(signal, kernel, dtype)
|
|
12
15
|
validate_inputs!(signal, kernel)
|
|
13
|
-
validate_vocabulary!(mode, boundary)
|
|
14
|
-
assign_options(signal, kernel, mode:, boundary:, fill_value:, origin:)
|
|
16
|
+
validate_vocabulary!(operation, mode, boundary)
|
|
17
|
+
assign_options(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
|
|
15
18
|
validate_combinations!(signal.shape, kernel.shape)
|
|
19
|
+
@fill_value = OperationDtype.cast_fill(@fill_value, @dtype)
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
private
|
|
19
23
|
|
|
20
24
|
def validate_inputs!(signal, kernel)
|
|
21
|
-
unless signal.is_a?(Numo::NArray) && kernel.is_a?(Numo::NArray)
|
|
22
|
-
raise ArgumentError, 'signal and kernel must be Numo::NArray values'
|
|
23
|
-
end
|
|
24
25
|
raise ArgumentError, 'signal and kernel must not be empty' if signal.empty? || kernel.empty?
|
|
25
26
|
raise ArgumentError, 'signal and kernel must have equal rank' unless signal.ndim == kernel.ndim
|
|
26
27
|
raise ArgumentError, "maximum supported rank is #{MAX_RANK}" if signal.ndim > MAX_RANK
|
|
27
28
|
end
|
|
28
29
|
|
|
29
|
-
def validate_vocabulary!(mode, boundary)
|
|
30
|
+
def validate_vocabulary!(operation, mode, boundary)
|
|
31
|
+
raise ArgumentError, "operation must be one of #{OPERATIONS.inspect}" unless OPERATIONS.include?(operation)
|
|
30
32
|
raise ArgumentError, "mode must be one of #{MODES.inspect}" unless MODES.include?(mode)
|
|
31
33
|
return if BOUNDARIES.include?(boundary)
|
|
32
34
|
|
|
33
35
|
raise ArgumentError, "boundary must be one of #{BOUNDARIES.inspect}"
|
|
34
36
|
end
|
|
35
37
|
|
|
36
|
-
def assign_options(signal, kernel, mode:, boundary:, fill_value:, origin:)
|
|
38
|
+
def assign_options(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:)
|
|
39
|
+
@operation = operation
|
|
37
40
|
@mode = mode
|
|
38
41
|
@boundary = boundary
|
|
39
42
|
@fill_value_given = !fill_value.equal?(UNSPECIFIED_FILL)
|
|
@@ -1,28 +1,41 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'forwardable'
|
|
4
|
+
require 'convolver/cost_profiles'
|
|
5
|
+
require 'convolver/operation_dtype'
|
|
4
6
|
require 'convolver/operation_options'
|
|
5
7
|
require 'convolver/operation_shapes'
|
|
6
8
|
require 'convolver/signal_extension'
|
|
9
|
+
require 'convolver/fft_buffer_plan'
|
|
7
10
|
|
|
8
11
|
# Internal planning and extension support for Convolver's public operations.
|
|
9
12
|
module Convolver
|
|
10
13
|
# Distinguishes an omitted fill_value keyword from an explicitly supplied
|
|
11
14
|
# value. This lets non-constant boundaries reject even an explicit zero.
|
|
15
|
+
# @private
|
|
12
16
|
UNSPECIFIED_FILL = Object.new.freeze
|
|
13
17
|
|
|
14
|
-
# Validated dimensions and boundary-extension details for one
|
|
18
|
+
# Validated dimensions and boundary-extension details for one operation.
|
|
19
|
+
# @private
|
|
15
20
|
class OperationPlan
|
|
16
21
|
extend Forwardable
|
|
17
22
|
|
|
18
|
-
def_delegators :options, :mode, :boundary, :fill_value, :origins, :anchors
|
|
23
|
+
def_delegators :options, :operation, :mode, :boundary, :fill_value, :origins, :anchors, :dtype
|
|
19
24
|
def_delegators :shapes, :padding_before, :padding_after, :result_shape,
|
|
20
25
|
:extended_shape, :result_size, :extended_size, :linear_fft_shape,
|
|
21
|
-
:linear_fft_size
|
|
26
|
+
:linear_fft_size, :linear_spectrum_size
|
|
22
27
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
28
|
+
attr_reader :conversion_size, :cost_profile
|
|
29
|
+
|
|
30
|
+
def initialize(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype: nil)
|
|
31
|
+
@options = OperationOptions.new(signal, kernel, operation:, mode:, boundary:, fill_value:, origin:, dtype:)
|
|
32
|
+
@conversion_size = input_conversion_size(signal, kernel)
|
|
33
|
+
@signal_shape = signal.shape.freeze
|
|
34
|
+
@kernel_shape = kernel.shape.freeze
|
|
35
|
+
@cost_profile = CostProfiles.for_rank(@signal_shape.length)
|
|
36
|
+
@shapes = OperationShapes.new(
|
|
37
|
+
signal.shape, kernel.shape, operation:, mode:, anchors: options.anchors
|
|
38
|
+
)
|
|
26
39
|
end
|
|
27
40
|
|
|
28
41
|
def valid?
|
|
@@ -39,9 +52,24 @@ module Convolver
|
|
|
39
52
|
SignalExtension.new(shapes, boundary:, fill_value:).call(signal)
|
|
40
53
|
end
|
|
41
54
|
|
|
55
|
+
def validate_basic!
|
|
56
|
+
return if @basic_validated
|
|
57
|
+
|
|
58
|
+
BufferPreparation.new(self, @signal_shape, @kernel_shape).basic!
|
|
59
|
+
@basic_validated = true
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def fft_buffers
|
|
63
|
+
@fft_buffers ||= FftBufferPlan.new(self, @signal_shape, @kernel_shape)
|
|
64
|
+
end
|
|
65
|
+
|
|
42
66
|
private
|
|
43
67
|
|
|
44
68
|
attr_reader :options, :shapes
|
|
69
|
+
|
|
70
|
+
def input_conversion_size(signal, kernel)
|
|
71
|
+
[signal, kernel].sum { |input| input.instance_of?(dtype) ? 0 : input.size }
|
|
72
|
+
end
|
|
45
73
|
end
|
|
46
74
|
|
|
47
75
|
private_constant :OperationPlan, :UNSPECIFIED_FILL
|