convolver 3.0.1 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +56 -3
- data/README.md +79 -119
- data/docs/rules.md +206 -0
- data/docs/terminology.md +30 -0
- data/ext/convolver/convolve_raw.c +51 -165
- data/ext/convolver/convolve_raw.h +5 -19
- data/ext/convolver/convolver.c +42 -20
- data/ext/convolver/correlate_raw.c +51 -161
- data/ext/convolver/correlate_raw.h +5 -19
- data/ext/convolver/extconf.rb +1 -0
- data/ext/convolver/fft_shape.c +116 -0
- data/ext/convolver/fft_shape.h +8 -0
- data/ext/convolver/raw_config.h +11 -0
- data/ext/convolver/raw_helpers.h +71 -0
- data/ext/convolver/raw_loop.h +93 -0
- data/lib/convolver/buffer_preparation.rb +4 -4
- data/lib/convolver/circular_fft_operation.rb +4 -3
- data/lib/convolver/cost_profiles.rb +44 -0
- data/lib/convolver/fft_cost.rb +52 -0
- data/lib/convolver/fft_estimator.rb +25 -32
- data/lib/convolver/fft_limits.rb +1 -1
- data/lib/convolver/linear_fft_operation.rb +4 -2
- data/lib/convolver/operation_dtype.rb +42 -0
- data/lib/convolver/operation_execution.rb +56 -45
- data/lib/convolver/operation_options.rb +4 -5
- data/lib/convolver/operation_plan.rb +17 -4
- data/lib/convolver/preparation_cost.rb +46 -0
- data/lib/convolver/real_fft_shape.rb +2 -61
- data/lib/convolver/signal_extension.rb +2 -2
- data/lib/convolver/version.rb +1 -1
- data/lib/convolver.rb +183 -68
- metadata +12 -1
|
@@ -1,171 +1,57 @@
|
|
|
1
|
-
// ext/convolver/convolve_raw.c
|
|
2
|
-
|
|
3
1
|
#include "convolve_raw.h"
|
|
2
|
+
#include "raw_helpers.h"
|
|
4
3
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}
|
|
9
|
-
return left + right;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
|
|
13
|
-
if ( left != 0 && right > SIZE_MAX / left ) {
|
|
14
|
-
rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
|
|
15
|
-
}
|
|
16
|
-
return left * right;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
|
|
20
|
-
size_t size = 1;
|
|
21
|
-
int i;
|
|
22
|
-
for ( i = 0; i < rank; i++ ) {
|
|
23
|
-
size = checked_multiply( size, shape[i], description );
|
|
24
|
-
}
|
|
25
|
-
return size;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// Sets reverse indices
|
|
29
|
-
static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
|
|
30
|
-
int i;
|
|
31
|
-
for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// Counts indices down, returns number of ranks that reset
|
|
35
|
-
static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
|
|
36
|
-
int i = 0;
|
|
37
|
-
(void) rank;
|
|
38
|
-
while ( ! rev_indices[i]-- ) {
|
|
39
|
-
rev_indices[i] = shape[i] - 1;
|
|
40
|
-
i++;
|
|
41
|
-
}
|
|
42
|
-
return i;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
|
|
46
|
-
static inline void calc_co_increment(
|
|
47
|
-
int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
|
|
48
|
-
size_t factor = 1;
|
|
49
|
-
int i;
|
|
50
|
-
co_increment[0] = 1; // co-increment is always 1 in lowest rank
|
|
51
|
-
for ( i = 0; i < rank; i++ ) {
|
|
52
|
-
size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
|
|
53
|
-
co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
|
|
54
|
-
factor = checked_multiply( factor, outer_shape[i], "array size" );
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
static inline size_t maximum_offset(
|
|
59
|
-
int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
|
|
60
|
-
size_t factor = 1;
|
|
61
|
-
size_t offset = 0;
|
|
62
|
-
int i;
|
|
63
|
-
for ( i = 0; i < rank; i++ ) {
|
|
64
|
-
size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
|
|
65
|
-
offset = checked_add( offset, dimension_offset, description );
|
|
66
|
-
factor = checked_multiply( factor, outer_shape[i], "array size" );
|
|
67
|
-
}
|
|
68
|
-
return offset;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
72
|
-
//
|
|
73
|
-
// Convolve
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
void convolve_raw(
|
|
77
|
-
int in_rank, const size_t *in_shape, const float *in_ptr,
|
|
78
|
-
int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
|
|
79
|
-
int out_rank, const size_t *out_shape, float *out_ptr ) {
|
|
80
|
-
size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
|
|
81
|
-
size_t maximum_input_offset;
|
|
82
|
-
size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
|
|
83
|
-
size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
|
|
84
|
-
size_t *kernel_co_incr_cache;
|
|
85
|
-
VALUE cache_storage = 0;
|
|
86
|
-
|
|
87
|
-
kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
|
|
88
|
-
kernel_aligned = kernel_size - kernel_size % 4;
|
|
89
|
-
out_size = size_from_shape( out_rank, out_shape, "output size" );
|
|
90
|
-
input_size = size_from_shape( in_rank, in_shape, "input size" );
|
|
91
|
-
|
|
92
|
-
calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
|
|
93
|
-
calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
|
|
94
|
-
maximum_input_offset = checked_add(
|
|
95
|
-
maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
|
|
96
|
-
maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
|
|
97
|
-
"input offset"
|
|
98
|
-
);
|
|
99
|
-
if ( maximum_input_offset >= input_size ) {
|
|
100
|
-
rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
|
|
104
|
-
kernel_co_incr_cache[0] = 0;
|
|
105
|
-
|
|
106
|
-
corner_reset( kernel_rank, kernel_shape, ker_q );
|
|
107
|
-
for ( i = 1; i < kernel_size; i++ ) {
|
|
108
|
-
kernel_co_incr_cache[i] = checked_add(
|
|
109
|
-
kernel_co_incr_cache[i-1],
|
|
110
|
-
kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
|
|
111
|
-
"kernel offset"
|
|
112
|
-
);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
offset = 0;
|
|
116
|
-
corner_reset( out_rank, out_shape, out_q );
|
|
117
|
-
|
|
118
|
-
// Main convolution loop
|
|
119
|
-
for ( i = 0; i < out_size; i++ ) {
|
|
120
|
-
#if CONVOLVER_USE_SSE
|
|
121
|
-
__m128 simd_x, simd_y, simd_t;
|
|
122
|
-
float v[4];
|
|
123
|
-
simd_t = _mm_setzero_ps();
|
|
124
|
-
#endif
|
|
125
|
-
float t = 0.0;
|
|
126
|
-
|
|
127
|
-
#if CONVOLVER_USE_SSE
|
|
128
|
-
// Contiguous kernel views need not start on a 16-byte boundary.
|
|
129
|
-
for ( j = 0; j < kernel_aligned; j +=4 ) {
|
|
130
|
-
simd_x = _mm_loadu_ps( kernel_ptr + j );
|
|
131
|
-
// Yes the backwards alignment is correct
|
|
132
|
-
simd_y = _mm_set_ps(
|
|
133
|
-
in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+3)] ],
|
|
134
|
-
in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+2)] ],
|
|
135
|
-
in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+1)] ],
|
|
136
|
-
in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - j] ]
|
|
137
|
-
);
|
|
138
|
-
simd_x = _mm_mul_ps( simd_x, simd_y );
|
|
139
|
-
simd_t = _mm_add_ps( simd_x, simd_t );
|
|
140
|
-
}
|
|
141
|
-
_mm_store_ps( v, simd_t );
|
|
4
|
+
#define RAW_INDEX(j) (kernel_size - 1 - (j))
|
|
5
|
+
#if defined(__SSE__)
|
|
6
|
+
#define RAW_SIMD 1
|
|
142
7
|
#else
|
|
143
|
-
|
|
144
|
-
// Use the same calculation without x86-specific intrinsics.
|
|
145
|
-
for ( j = 0; j < kernel_aligned; j++ ) {
|
|
146
|
-
t += in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - j] ] * kernel_ptr[j];
|
|
147
|
-
}
|
|
8
|
+
#define RAW_SIMD 0
|
|
148
9
|
#endif
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
#
|
|
156
|
-
|
|
10
|
+
#define RAW_NAME convolve_raw
|
|
11
|
+
#define RAW_TYPE float
|
|
12
|
+
#define RAW_LANES 4
|
|
13
|
+
#define RAW_VECTOR __m128
|
|
14
|
+
#define RAW_ZERO _mm_setzero_ps
|
|
15
|
+
#define RAW_LOAD _mm_loadu_ps
|
|
16
|
+
#define RAW_STORE _mm_storeu_ps
|
|
17
|
+
#define RAW_ADD _mm_add_ps
|
|
18
|
+
#define RAW_MUL _mm_mul_ps
|
|
19
|
+
#include "raw_loop.h"
|
|
20
|
+
#undef RAW_NAME
|
|
21
|
+
#undef RAW_TYPE
|
|
22
|
+
#undef RAW_LANES
|
|
23
|
+
#undef RAW_VECTOR
|
|
24
|
+
#undef RAW_ZERO
|
|
25
|
+
#undef RAW_LOAD
|
|
26
|
+
#undef RAW_STORE
|
|
27
|
+
#undef RAW_ADD
|
|
28
|
+
#undef RAW_MUL
|
|
29
|
+
#undef RAW_SIMD
|
|
30
|
+
|
|
31
|
+
#if defined(__SSE2__)
|
|
32
|
+
#define RAW_SIMD 1
|
|
157
33
|
#else
|
|
158
|
-
|
|
34
|
+
#define RAW_SIMD 0
|
|
159
35
|
#endif
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
36
|
+
#define RAW_NAME convolve_raw_double
|
|
37
|
+
#define RAW_TYPE double
|
|
38
|
+
#define RAW_LANES 2
|
|
39
|
+
#define RAW_VECTOR __m128d
|
|
40
|
+
#define RAW_ZERO _mm_setzero_pd
|
|
41
|
+
#define RAW_LOAD _mm_loadu_pd
|
|
42
|
+
#define RAW_STORE _mm_storeu_pd
|
|
43
|
+
#define RAW_ADD _mm_add_pd
|
|
44
|
+
#define RAW_MUL _mm_mul_pd
|
|
45
|
+
#include "raw_loop.h"
|
|
46
|
+
#undef RAW_NAME
|
|
47
|
+
#undef RAW_TYPE
|
|
48
|
+
#undef RAW_LANES
|
|
49
|
+
#undef RAW_VECTOR
|
|
50
|
+
#undef RAW_ZERO
|
|
51
|
+
#undef RAW_LOAD
|
|
52
|
+
#undef RAW_STORE
|
|
53
|
+
#undef RAW_ADD
|
|
54
|
+
#undef RAW_MUL
|
|
55
|
+
#undef RAW_SIMD
|
|
56
|
+
|
|
57
|
+
#undef RAW_INDEX
|
|
@@ -1,28 +1,14 @@
|
|
|
1
|
-
// ext/convolver/convolve_raw.h
|
|
2
|
-
|
|
3
|
-
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
4
|
-
//
|
|
5
|
-
// Declarations of narray helper functions
|
|
6
|
-
//
|
|
7
|
-
|
|
8
1
|
#ifndef CONVOLVE_RAW_H
|
|
9
2
|
#define CONVOLVE_RAW_H
|
|
10
|
-
|
|
11
|
-
#ifndef CONVOLVER_RAW_CONFIG
|
|
12
|
-
#define CONVOLVER_RAW_CONFIG
|
|
13
|
-
#include <ruby.h>
|
|
14
|
-
#if defined(__SSE__)
|
|
15
|
-
#include <xmmintrin.h>
|
|
16
|
-
#define CONVOLVER_USE_SSE 1
|
|
17
|
-
#else
|
|
18
|
-
#define CONVOLVER_USE_SSE 0
|
|
19
|
-
#endif
|
|
20
|
-
#define LARGEST_RANK 16
|
|
21
|
-
#endif
|
|
3
|
+
#include "raw_config.h"
|
|
22
4
|
|
|
23
5
|
void convolve_raw(
|
|
24
6
|
int in_rank, const size_t *in_shape, const float *in_ptr,
|
|
25
7
|
int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
|
|
26
8
|
int out_rank, const size_t *out_shape, float *out_ptr );
|
|
27
9
|
|
|
10
|
+
void convolve_raw_double(
|
|
11
|
+
int in_rank, const size_t *in_shape, const double *in_ptr,
|
|
12
|
+
int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
|
|
13
|
+
int out_rank, const size_t *out_shape, double *out_ptr );
|
|
28
14
|
#endif
|
data/ext/convolver/convolver.c
CHANGED
|
@@ -4,18 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
#include "correlate_raw.h"
|
|
6
6
|
#include "convolve_raw.h"
|
|
7
|
+
#include "fft_shape.h"
|
|
7
8
|
|
|
8
9
|
static VALUE mConvolver;
|
|
9
10
|
|
|
10
11
|
/* Contiguous views share their backing storage but start at a byte offset. */
|
|
11
|
-
static const
|
|
12
|
-
return (const
|
|
12
|
+
static const void *logical_pointer_for_read(VALUE value) {
|
|
13
|
+
return (const void *)(na_get_pointer_for_read(value) + na_get_offset(value));
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
static VALUE
|
|
16
|
+
static VALUE contiguous_float(VALUE value) {
|
|
16
17
|
narray_t *narray;
|
|
17
18
|
|
|
18
|
-
value = rb_funcall(numo_cSFloat, rb_intern("cast"), 1, value);
|
|
19
19
|
GetNArray(value, narray);
|
|
20
20
|
/* A scalar has no strides; Numo's contiguous check requires an axis. */
|
|
21
21
|
if (narray->ndim > 0 && !RTEST(na_check_contiguous(value))) {
|
|
@@ -54,8 +54,13 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
|
|
|
54
54
|
rb_raise(rb_eArgError, "signal and kernel must be Numo::NArray values");
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
VALUE dtype = CLASS_OF(signal);
|
|
58
|
+
if ((dtype != numo_cSFloat && dtype != numo_cDFloat) || CLASS_OF(kernel) != dtype) {
|
|
59
|
+
rb_raise(rb_eArgError, "native inputs must have matching SFloat or DFloat dtypes");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
signal_value = contiguous_float(signal);
|
|
63
|
+
kernel_value = contiguous_float(kernel);
|
|
59
64
|
GetNArray(signal_value, signal_narray);
|
|
60
65
|
GetNArray(kernel_value, kernel_narray);
|
|
61
66
|
|
|
@@ -82,19 +87,35 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
|
|
|
82
87
|
numo_result_shape[rank - i - 1] = (size_t)result_shape[i];
|
|
83
88
|
}
|
|
84
89
|
|
|
85
|
-
result_value = nary_new(
|
|
86
|
-
if (
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
result_value = nary_new(dtype, rank, numo_result_shape);
|
|
91
|
+
if (dtype == numo_cSFloat) {
|
|
92
|
+
if (operation == CONVOLVER_CORRELATION) {
|
|
93
|
+
correlate_raw(
|
|
94
|
+
rank, signal_shape, logical_pointer_for_read(signal_value),
|
|
95
|
+
rank, kernel_shape, logical_pointer_for_read(kernel_value),
|
|
96
|
+
rank, result_shape, (float *)na_get_pointer_for_write(result_value)
|
|
97
|
+
);
|
|
98
|
+
} else {
|
|
99
|
+
convolve_raw(
|
|
100
|
+
rank, signal_shape, logical_pointer_for_read(signal_value),
|
|
101
|
+
rank, kernel_shape, logical_pointer_for_read(kernel_value),
|
|
102
|
+
rank, result_shape, (float *)na_get_pointer_for_write(result_value)
|
|
103
|
+
);
|
|
104
|
+
}
|
|
92
105
|
} else {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
106
|
+
if (operation == CONVOLVER_CORRELATION) {
|
|
107
|
+
correlate_raw_double(
|
|
108
|
+
rank, signal_shape, logical_pointer_for_read(signal_value),
|
|
109
|
+
rank, kernel_shape, logical_pointer_for_read(kernel_value),
|
|
110
|
+
rank, result_shape, (double *)na_get_pointer_for_write(result_value)
|
|
111
|
+
);
|
|
112
|
+
} else {
|
|
113
|
+
convolve_raw_double(
|
|
114
|
+
rank, signal_shape, logical_pointer_for_read(signal_value),
|
|
115
|
+
rank, kernel_shape, logical_pointer_for_read(kernel_value),
|
|
116
|
+
rank, result_shape, (double *)na_get_pointer_for_write(result_value)
|
|
117
|
+
);
|
|
118
|
+
}
|
|
98
119
|
}
|
|
99
120
|
|
|
100
121
|
return result_value;
|
|
@@ -107,7 +128,7 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
|
|
|
107
128
|
* @overload correlate_basic_valid(signal, kernel)
|
|
108
129
|
* @param signal [Numo::NArray] input values
|
|
109
130
|
* @param kernel [Numo::NArray] correlation kernel
|
|
110
|
-
* @return [Numo::SFloat] valid cross-correlation result
|
|
131
|
+
* @return [Numo::SFloat, Numo::DFloat] valid cross-correlation result
|
|
111
132
|
*/
|
|
112
133
|
static VALUE convolver_correlate_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
|
|
113
134
|
(void)self;
|
|
@@ -121,7 +142,7 @@ static VALUE convolver_correlate_basic_valid(VALUE self, VALUE signal, VALUE ker
|
|
|
121
142
|
* @overload convolve_basic_valid(signal, kernel)
|
|
122
143
|
* @param signal [Numo::NArray] input values
|
|
123
144
|
* @param kernel [Numo::NArray] convolution kernel
|
|
124
|
-
* @return [Numo::SFloat] valid mathematical convolution result
|
|
145
|
+
* @return [Numo::SFloat, Numo::DFloat] valid mathematical convolution result
|
|
125
146
|
*/
|
|
126
147
|
static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
|
|
127
148
|
(void)self;
|
|
@@ -130,6 +151,7 @@ static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kern
|
|
|
130
151
|
|
|
131
152
|
void Init_convolver(void) {
|
|
132
153
|
mConvolver = rb_define_module("Convolver");
|
|
154
|
+
convolver_init_fft_shape(mConvolver);
|
|
133
155
|
rb_define_singleton_method(mConvolver, "correlate_basic_valid", convolver_correlate_basic_valid, 2);
|
|
134
156
|
rb_define_singleton_method(mConvolver, "convolve_basic_valid", convolver_convolve_basic_valid, 2);
|
|
135
157
|
}
|
|
@@ -1,167 +1,57 @@
|
|
|
1
|
-
// ext/convolver/correlate_raw.c
|
|
2
|
-
|
|
3
1
|
#include "correlate_raw.h"
|
|
2
|
+
#include "raw_helpers.h"
|
|
4
3
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}
|
|
9
|
-
return left + right;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
|
|
13
|
-
if ( left != 0 && right > SIZE_MAX / left ) {
|
|
14
|
-
rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
|
|
15
|
-
}
|
|
16
|
-
return left * right;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
|
|
20
|
-
size_t size = 1;
|
|
21
|
-
int i;
|
|
22
|
-
for ( i = 0; i < rank; i++ ) {
|
|
23
|
-
size = checked_multiply( size, shape[i], description );
|
|
24
|
-
}
|
|
25
|
-
return size;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// Sets reverse indices
|
|
29
|
-
static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
|
|
30
|
-
int i;
|
|
31
|
-
for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// Counts indices down, returns number of ranks that reset
|
|
35
|
-
static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
|
|
36
|
-
int i = 0;
|
|
37
|
-
(void) rank;
|
|
38
|
-
while ( ! rev_indices[i]-- ) {
|
|
39
|
-
rev_indices[i] = shape[i] - 1;
|
|
40
|
-
i++;
|
|
41
|
-
}
|
|
42
|
-
return i;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
|
|
46
|
-
static inline void calc_co_increment(
|
|
47
|
-
int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
|
|
48
|
-
size_t factor = 1;
|
|
49
|
-
int i;
|
|
50
|
-
co_increment[0] = 1; // co-increment is always 1 in lowest rank
|
|
51
|
-
for ( i = 0; i < rank; i++ ) {
|
|
52
|
-
size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
|
|
53
|
-
co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
|
|
54
|
-
factor = checked_multiply( factor, outer_shape[i], "array size" );
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
static inline size_t maximum_offset(
|
|
59
|
-
int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
|
|
60
|
-
size_t factor = 1;
|
|
61
|
-
size_t offset = 0;
|
|
62
|
-
int i;
|
|
63
|
-
for ( i = 0; i < rank; i++ ) {
|
|
64
|
-
size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
|
|
65
|
-
offset = checked_add( offset, dimension_offset, description );
|
|
66
|
-
factor = checked_multiply( factor, outer_shape[i], "array size" );
|
|
67
|
-
}
|
|
68
|
-
return offset;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
72
|
-
//
|
|
73
|
-
// Correlate
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
void correlate_raw(
|
|
77
|
-
int in_rank, const size_t *in_shape, const float *in_ptr,
|
|
78
|
-
int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
|
|
79
|
-
int out_rank, const size_t *out_shape, float *out_ptr ) {
|
|
80
|
-
size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
|
|
81
|
-
size_t maximum_input_offset;
|
|
82
|
-
size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
|
|
83
|
-
size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
|
|
84
|
-
size_t *kernel_co_incr_cache;
|
|
85
|
-
VALUE cache_storage = 0;
|
|
86
|
-
|
|
87
|
-
kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
|
|
88
|
-
kernel_aligned = kernel_size - kernel_size % 4;
|
|
89
|
-
out_size = size_from_shape( out_rank, out_shape, "output size" );
|
|
90
|
-
input_size = size_from_shape( in_rank, in_shape, "input size" );
|
|
91
|
-
|
|
92
|
-
calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
|
|
93
|
-
calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
|
|
94
|
-
maximum_input_offset = checked_add(
|
|
95
|
-
maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
|
|
96
|
-
maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
|
|
97
|
-
"input offset"
|
|
98
|
-
);
|
|
99
|
-
if ( maximum_input_offset >= input_size ) {
|
|
100
|
-
rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
|
|
104
|
-
kernel_co_incr_cache[0] = 0;
|
|
105
|
-
|
|
106
|
-
corner_reset( kernel_rank, kernel_shape, ker_q );
|
|
107
|
-
for ( i = 1; i < kernel_size; i++ ) {
|
|
108
|
-
kernel_co_incr_cache[i] = checked_add(
|
|
109
|
-
kernel_co_incr_cache[i-1],
|
|
110
|
-
kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
|
|
111
|
-
"kernel offset"
|
|
112
|
-
);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
offset = 0;
|
|
116
|
-
corner_reset( out_rank, out_shape, out_q );
|
|
117
|
-
|
|
118
|
-
// Main correlation loop
|
|
119
|
-
for ( i = 0; i < out_size; i++ ) {
|
|
120
|
-
#if CONVOLVER_USE_SSE
|
|
121
|
-
__m128 simd_x, simd_y, simd_t;
|
|
122
|
-
float v[4];
|
|
123
|
-
simd_t = _mm_setzero_ps();
|
|
124
|
-
#endif
|
|
125
|
-
float t = 0.0;
|
|
126
|
-
|
|
127
|
-
#if CONVOLVER_USE_SSE
|
|
128
|
-
// Contiguous kernel views need not start on a 16-byte boundary.
|
|
129
|
-
for ( j = 0; j < kernel_aligned; j +=4 ) {
|
|
130
|
-
simd_x = _mm_loadu_ps( kernel_ptr + j );
|
|
131
|
-
// Yes the backwards alignment is correct
|
|
132
|
-
simd_y = _mm_set_ps( in_ptr[ offset + kernel_co_incr_cache[j+3] ], in_ptr[ offset + kernel_co_incr_cache[j+2] ],
|
|
133
|
-
in_ptr[ offset + kernel_co_incr_cache[j+1] ], in_ptr[ offset + kernel_co_incr_cache[j] ] );
|
|
134
|
-
simd_x = _mm_mul_ps( simd_x, simd_y );
|
|
135
|
-
simd_t = _mm_add_ps( simd_x, simd_t );
|
|
136
|
-
}
|
|
137
|
-
_mm_store_ps( v, simd_t );
|
|
4
|
+
#define RAW_INDEX(j) (j)
|
|
5
|
+
#if defined(__SSE__)
|
|
6
|
+
#define RAW_SIMD 1
|
|
138
7
|
#else
|
|
139
|
-
|
|
140
|
-
// Use the same calculation without x86-specific intrinsics.
|
|
141
|
-
for ( j = 0; j < kernel_aligned; j++ ) {
|
|
142
|
-
t += in_ptr[ offset + kernel_co_incr_cache[j] ] * kernel_ptr[j];
|
|
143
|
-
}
|
|
8
|
+
#define RAW_SIMD 0
|
|
144
9
|
#endif
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
#
|
|
152
|
-
|
|
10
|
+
#define RAW_NAME correlate_raw
|
|
11
|
+
#define RAW_TYPE float
|
|
12
|
+
#define RAW_LANES 4
|
|
13
|
+
#define RAW_VECTOR __m128
|
|
14
|
+
#define RAW_ZERO _mm_setzero_ps
|
|
15
|
+
#define RAW_LOAD _mm_loadu_ps
|
|
16
|
+
#define RAW_STORE _mm_storeu_ps
|
|
17
|
+
#define RAW_ADD _mm_add_ps
|
|
18
|
+
#define RAW_MUL _mm_mul_ps
|
|
19
|
+
#include "raw_loop.h"
|
|
20
|
+
#undef RAW_NAME
|
|
21
|
+
#undef RAW_TYPE
|
|
22
|
+
#undef RAW_LANES
|
|
23
|
+
#undef RAW_VECTOR
|
|
24
|
+
#undef RAW_ZERO
|
|
25
|
+
#undef RAW_LOAD
|
|
26
|
+
#undef RAW_STORE
|
|
27
|
+
#undef RAW_ADD
|
|
28
|
+
#undef RAW_MUL
|
|
29
|
+
#undef RAW_SIMD
|
|
30
|
+
|
|
31
|
+
#if defined(__SSE2__)
|
|
32
|
+
#define RAW_SIMD 1
|
|
153
33
|
#else
|
|
154
|
-
|
|
34
|
+
#define RAW_SIMD 0
|
|
155
35
|
#endif
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
36
|
+
#define RAW_NAME correlate_raw_double
|
|
37
|
+
#define RAW_TYPE double
|
|
38
|
+
#define RAW_LANES 2
|
|
39
|
+
#define RAW_VECTOR __m128d
|
|
40
|
+
#define RAW_ZERO _mm_setzero_pd
|
|
41
|
+
#define RAW_LOAD _mm_loadu_pd
|
|
42
|
+
#define RAW_STORE _mm_storeu_pd
|
|
43
|
+
#define RAW_ADD _mm_add_pd
|
|
44
|
+
#define RAW_MUL _mm_mul_pd
|
|
45
|
+
#include "raw_loop.h"
|
|
46
|
+
#undef RAW_NAME
|
|
47
|
+
#undef RAW_TYPE
|
|
48
|
+
#undef RAW_LANES
|
|
49
|
+
#undef RAW_VECTOR
|
|
50
|
+
#undef RAW_ZERO
|
|
51
|
+
#undef RAW_LOAD
|
|
52
|
+
#undef RAW_STORE
|
|
53
|
+
#undef RAW_ADD
|
|
54
|
+
#undef RAW_MUL
|
|
55
|
+
#undef RAW_SIMD
|
|
56
|
+
|
|
57
|
+
#undef RAW_INDEX
|
|
@@ -1,28 +1,14 @@
|
|
|
1
|
-
// ext/convolver/correlate_raw.h
|
|
2
|
-
|
|
3
|
-
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
4
|
-
//
|
|
5
|
-
// Declarations of narray helper functions
|
|
6
|
-
//
|
|
7
|
-
|
|
8
1
|
#ifndef CORRELATE_RAW_H
|
|
9
2
|
#define CORRELATE_RAW_H
|
|
10
|
-
|
|
11
|
-
#ifndef CONVOLVER_RAW_CONFIG
|
|
12
|
-
#define CONVOLVER_RAW_CONFIG
|
|
13
|
-
#include <ruby.h>
|
|
14
|
-
#if defined(__SSE__)
|
|
15
|
-
#include <xmmintrin.h>
|
|
16
|
-
#define CONVOLVER_USE_SSE 1
|
|
17
|
-
#else
|
|
18
|
-
#define CONVOLVER_USE_SSE 0
|
|
19
|
-
#endif
|
|
20
|
-
#define LARGEST_RANK 16
|
|
21
|
-
#endif
|
|
3
|
+
#include "raw_config.h"
|
|
22
4
|
|
|
23
5
|
void correlate_raw(
|
|
24
6
|
int in_rank, const size_t *in_shape, const float *in_ptr,
|
|
25
7
|
int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
|
|
26
8
|
int out_rank, const size_t *out_shape, float *out_ptr );
|
|
27
9
|
|
|
10
|
+
void correlate_raw_double(
|
|
11
|
+
int in_rank, const size_t *in_shape, const double *in_ptr,
|
|
12
|
+
int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
|
|
13
|
+
int out_rank, const size_t *out_shape, double *out_ptr );
|
|
28
14
|
#endif
|
data/ext/convolver/extconf.rb
CHANGED
|
@@ -11,6 +11,7 @@ $LOAD_PATH.each do |load_path|
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
abort 'numo/narray.h not found' unless have_header('numo/narray.h')
|
|
14
|
+
abort 'log2 not found' unless have_func('log2', 'math.h') || have_library('m', 'log2', 'math.h')
|
|
14
15
|
|
|
15
16
|
if RUBY_PLATFORM.match?(/mswin|cygwin|mingw/)
|
|
16
17
|
$LOAD_PATH.each do |load_path|
|