convolver 3.0.1 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,171 +1,57 @@
1
- // ext/convolver/convolve_raw.c
2
-
3
1
  #include "convolve_raw.h"
2
+ #include "raw_helpers.h"
4
3
 
5
- static inline size_t checked_add( size_t left, size_t right, const char *description ) {
6
- if ( right > SIZE_MAX - left ) {
7
- rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
8
- }
9
- return left + right;
10
- }
11
-
12
- static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
13
- if ( left != 0 && right > SIZE_MAX / left ) {
14
- rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
15
- }
16
- return left * right;
17
- }
18
-
19
- static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
20
- size_t size = 1;
21
- int i;
22
- for ( i = 0; i < rank; i++ ) {
23
- size = checked_multiply( size, shape[i], description );
24
- }
25
- return size;
26
- }
27
-
28
- // Sets reverse indices
29
- static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
30
- int i;
31
- for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
32
- }
33
-
34
- // Counts indices down, returns number of ranks that reset
35
- static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
36
- int i = 0;
37
- (void) rank;
38
- while ( ! rev_indices[i]-- ) {
39
- rev_indices[i] = shape[i] - 1;
40
- i++;
41
- }
42
- return i;
43
- }
44
-
45
- // Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
46
- static inline void calc_co_increment(
47
- int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
48
- size_t factor = 1;
49
- int i;
50
- co_increment[0] = 1; // co-increment is always 1 in lowest rank
51
- for ( i = 0; i < rank; i++ ) {
52
- size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
53
- co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
54
- factor = checked_multiply( factor, outer_shape[i], "array size" );
55
- }
56
- }
57
-
58
- static inline size_t maximum_offset(
59
- int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
60
- size_t factor = 1;
61
- size_t offset = 0;
62
- int i;
63
- for ( i = 0; i < rank; i++ ) {
64
- size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
65
- offset = checked_add( offset, dimension_offset, description );
66
- factor = checked_multiply( factor, outer_shape[i], "array size" );
67
- }
68
- return offset;
69
- }
70
-
71
- ////////////////////////////////////////////////////////////////////////////////////////////////////
72
- //
73
- // Convolve
74
- //
75
-
76
- void convolve_raw(
77
- int in_rank, const size_t *in_shape, const float *in_ptr,
78
- int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
79
- int out_rank, const size_t *out_shape, float *out_ptr ) {
80
- size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
81
- size_t maximum_input_offset;
82
- size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
83
- size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
84
- size_t *kernel_co_incr_cache;
85
- VALUE cache_storage = 0;
86
-
87
- kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
88
- kernel_aligned = kernel_size - kernel_size % 4;
89
- out_size = size_from_shape( out_rank, out_shape, "output size" );
90
- input_size = size_from_shape( in_rank, in_shape, "input size" );
91
-
92
- calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
93
- calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
94
- maximum_input_offset = checked_add(
95
- maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
96
- maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
97
- "input offset"
98
- );
99
- if ( maximum_input_offset >= input_size ) {
100
- rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
101
- }
102
-
103
- kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
104
- kernel_co_incr_cache[0] = 0;
105
-
106
- corner_reset( kernel_rank, kernel_shape, ker_q );
107
- for ( i = 1; i < kernel_size; i++ ) {
108
- kernel_co_incr_cache[i] = checked_add(
109
- kernel_co_incr_cache[i-1],
110
- kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
111
- "kernel offset"
112
- );
113
- }
114
-
115
- offset = 0;
116
- corner_reset( out_rank, out_shape, out_q );
117
-
118
- // Main convolution loop
119
- for ( i = 0; i < out_size; i++ ) {
120
- #if CONVOLVER_USE_SSE
121
- __m128 simd_x, simd_y, simd_t;
122
- float v[4];
123
- simd_t = _mm_setzero_ps();
124
- #endif
125
- float t = 0.0;
126
-
127
- #if CONVOLVER_USE_SSE
128
- // Contiguous kernel views need not start on a 16-byte boundary.
129
- for ( j = 0; j < kernel_aligned; j +=4 ) {
130
- simd_x = _mm_loadu_ps( kernel_ptr + j );
131
- // Yes the backwards alignment is correct
132
- simd_y = _mm_set_ps(
133
- in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+3)] ],
134
- in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+2)] ],
135
- in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - (j+1)] ],
136
- in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - j] ]
137
- );
138
- simd_x = _mm_mul_ps( simd_x, simd_y );
139
- simd_t = _mm_add_ps( simd_x, simd_t );
140
- }
141
- _mm_store_ps( v, simd_t );
4
+ #define RAW_INDEX(j) (kernel_size - 1 - (j))
5
+ #if defined(__SSE__)
6
+ #define RAW_SIMD 1
142
7
  #else
143
- // SSE is unavailable on architectures such as Apple Silicon.
144
- // Use the same calculation without x86-specific intrinsics.
145
- for ( j = 0; j < kernel_aligned; j++ ) {
146
- t += in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - j] ] * kernel_ptr[j];
147
- }
8
+ #define RAW_SIMD 0
148
9
  #endif
149
-
150
- // Complete any remaining 1,2 or 3 items one at a time
151
- for ( j = kernel_aligned; j < kernel_size; j++ ) {
152
- t += in_ptr[ offset + kernel_co_incr_cache[kernel_size - 1 - j] ] * kernel_ptr[ j ];
153
- }
154
-
155
- #if CONVOLVER_USE_SSE
156
- out_ptr[i] = v[0] + v[1] + v[2] + v[3] + t;
10
+ #define RAW_NAME convolve_raw
11
+ #define RAW_TYPE float
12
+ #define RAW_LANES 4
13
+ #define RAW_VECTOR __m128
14
+ #define RAW_ZERO _mm_setzero_ps
15
+ #define RAW_LOAD _mm_loadu_ps
16
+ #define RAW_STORE _mm_storeu_ps
17
+ #define RAW_ADD _mm_add_ps
18
+ #define RAW_MUL _mm_mul_ps
19
+ #include "raw_loop.h"
20
+ #undef RAW_NAME
21
+ #undef RAW_TYPE
22
+ #undef RAW_LANES
23
+ #undef RAW_VECTOR
24
+ #undef RAW_ZERO
25
+ #undef RAW_LOAD
26
+ #undef RAW_STORE
27
+ #undef RAW_ADD
28
+ #undef RAW_MUL
29
+ #undef RAW_SIMD
30
+
31
+ #if defined(__SSE2__)
32
+ #define RAW_SIMD 1
157
33
  #else
158
- out_ptr[i] = t;
34
+ #define RAW_SIMD 0
159
35
  #endif
160
-
161
- if ( i + 1 < out_size ) {
162
- offset = checked_add(
163
- offset,
164
- out_co_incr[ corner_dec( out_rank, out_shape, out_q ) ],
165
- "input offset"
166
- );
167
- }
168
- }
169
-
170
- RB_ALLOCV_END( cache_storage );
171
- }
36
+ #define RAW_NAME convolve_raw_double
37
+ #define RAW_TYPE double
38
+ #define RAW_LANES 2
39
+ #define RAW_VECTOR __m128d
40
+ #define RAW_ZERO _mm_setzero_pd
41
+ #define RAW_LOAD _mm_loadu_pd
42
+ #define RAW_STORE _mm_storeu_pd
43
+ #define RAW_ADD _mm_add_pd
44
+ #define RAW_MUL _mm_mul_pd
45
+ #include "raw_loop.h"
46
+ #undef RAW_NAME
47
+ #undef RAW_TYPE
48
+ #undef RAW_LANES
49
+ #undef RAW_VECTOR
50
+ #undef RAW_ZERO
51
+ #undef RAW_LOAD
52
+ #undef RAW_STORE
53
+ #undef RAW_ADD
54
+ #undef RAW_MUL
55
+ #undef RAW_SIMD
56
+
57
+ #undef RAW_INDEX
@@ -1,28 +1,14 @@
1
- // ext/convolver/convolve_raw.h
2
-
3
- ////////////////////////////////////////////////////////////////////////////////////////////////
4
- //
5
- // Declarations of narray helper functions
6
- //
7
-
8
1
  #ifndef CONVOLVE_RAW_H
9
2
  #define CONVOLVE_RAW_H
10
-
11
- #ifndef CONVOLVER_RAW_CONFIG
12
- #define CONVOLVER_RAW_CONFIG
13
- #include <ruby.h>
14
- #if defined(__SSE__)
15
- #include <xmmintrin.h>
16
- #define CONVOLVER_USE_SSE 1
17
- #else
18
- #define CONVOLVER_USE_SSE 0
19
- #endif
20
- #define LARGEST_RANK 16
21
- #endif
3
+ #include "raw_config.h"
22
4
 
23
5
  void convolve_raw(
24
6
  int in_rank, const size_t *in_shape, const float *in_ptr,
25
7
  int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
26
8
  int out_rank, const size_t *out_shape, float *out_ptr );
27
9
 
10
+ void convolve_raw_double(
11
+ int in_rank, const size_t *in_shape, const double *in_ptr,
12
+ int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
13
+ int out_rank, const size_t *out_shape, double *out_ptr );
28
14
  #endif
@@ -4,18 +4,18 @@
4
4
 
5
5
  #include "correlate_raw.h"
6
6
  #include "convolve_raw.h"
7
+ #include "fft_shape.h"
7
8
 
8
9
  static VALUE mConvolver;
9
10
 
10
11
  /* Contiguous views share their backing storage but start at a byte offset. */
11
- static const float *logical_pointer_for_read(VALUE value) {
12
- return (const float *)(na_get_pointer_for_read(value) + na_get_offset(value));
12
+ static const void *logical_pointer_for_read(VALUE value) {
13
+ return (const void *)(na_get_pointer_for_read(value) + na_get_offset(value));
13
14
  }
14
15
 
15
- static VALUE contiguous_sfloat(VALUE value) {
16
+ static VALUE contiguous_float(VALUE value) {
16
17
  narray_t *narray;
17
18
 
18
- value = rb_funcall(numo_cSFloat, rb_intern("cast"), 1, value);
19
19
  GetNArray(value, narray);
20
20
  /* A scalar has no strides; Numo's contiguous check requires an axis. */
21
21
  if (narray->ndim > 0 && !RTEST(na_check_contiguous(value))) {
@@ -54,8 +54,13 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
54
54
  rb_raise(rb_eArgError, "signal and kernel must be Numo::NArray values");
55
55
  }
56
56
 
57
- signal_value = contiguous_sfloat(signal);
58
- kernel_value = contiguous_sfloat(kernel);
57
+ VALUE dtype = CLASS_OF(signal);
58
+ if ((dtype != numo_cSFloat && dtype != numo_cDFloat) || CLASS_OF(kernel) != dtype) {
59
+ rb_raise(rb_eArgError, "native inputs must have matching SFloat or DFloat dtypes");
60
+ }
61
+
62
+ signal_value = contiguous_float(signal);
63
+ kernel_value = contiguous_float(kernel);
59
64
  GetNArray(signal_value, signal_narray);
60
65
  GetNArray(kernel_value, kernel_narray);
61
66
 
@@ -82,19 +87,35 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
82
87
  numo_result_shape[rank - i - 1] = (size_t)result_shape[i];
83
88
  }
84
89
 
85
- result_value = nary_new(numo_cSFloat, rank, numo_result_shape);
86
- if (operation == CONVOLVER_CORRELATION) {
87
- correlate_raw(
88
- rank, signal_shape, logical_pointer_for_read(signal_value),
89
- rank, kernel_shape, logical_pointer_for_read(kernel_value),
90
- rank, result_shape, (float *)na_get_pointer_for_write(result_value)
91
- );
90
+ result_value = nary_new(dtype, rank, numo_result_shape);
91
+ if (dtype == numo_cSFloat) {
92
+ if (operation == CONVOLVER_CORRELATION) {
93
+ correlate_raw(
94
+ rank, signal_shape, logical_pointer_for_read(signal_value),
95
+ rank, kernel_shape, logical_pointer_for_read(kernel_value),
96
+ rank, result_shape, (float *)na_get_pointer_for_write(result_value)
97
+ );
98
+ } else {
99
+ convolve_raw(
100
+ rank, signal_shape, logical_pointer_for_read(signal_value),
101
+ rank, kernel_shape, logical_pointer_for_read(kernel_value),
102
+ rank, result_shape, (float *)na_get_pointer_for_write(result_value)
103
+ );
104
+ }
92
105
  } else {
93
- convolve_raw(
94
- rank, signal_shape, logical_pointer_for_read(signal_value),
95
- rank, kernel_shape, logical_pointer_for_read(kernel_value),
96
- rank, result_shape, (float *)na_get_pointer_for_write(result_value)
97
- );
106
+ if (operation == CONVOLVER_CORRELATION) {
107
+ correlate_raw_double(
108
+ rank, signal_shape, logical_pointer_for_read(signal_value),
109
+ rank, kernel_shape, logical_pointer_for_read(kernel_value),
110
+ rank, result_shape, (double *)na_get_pointer_for_write(result_value)
111
+ );
112
+ } else {
113
+ convolve_raw_double(
114
+ rank, signal_shape, logical_pointer_for_read(signal_value),
115
+ rank, kernel_shape, logical_pointer_for_read(kernel_value),
116
+ rank, result_shape, (double *)na_get_pointer_for_write(result_value)
117
+ );
118
+ }
98
119
  }
99
120
 
100
121
  return result_value;
@@ -107,7 +128,7 @@ static VALUE convolver_basic_valid(VALUE signal, VALUE kernel, enum convolver_op
107
128
  * @overload correlate_basic_valid(signal, kernel)
108
129
  * @param signal [Numo::NArray] input values
109
130
  * @param kernel [Numo::NArray] correlation kernel
110
- * @return [Numo::SFloat] valid cross-correlation result
131
+ * @return [Numo::SFloat, Numo::DFloat] valid cross-correlation result
111
132
  */
112
133
  static VALUE convolver_correlate_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
113
134
  (void)self;
@@ -121,7 +142,7 @@ static VALUE convolver_correlate_basic_valid(VALUE self, VALUE signal, VALUE ker
121
142
  * @overload convolve_basic_valid(signal, kernel)
122
143
  * @param signal [Numo::NArray] input values
123
144
  * @param kernel [Numo::NArray] convolution kernel
124
- * @return [Numo::SFloat] valid mathematical convolution result
145
+ * @return [Numo::SFloat, Numo::DFloat] valid mathematical convolution result
125
146
  */
126
147
  static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kernel) {
127
148
  (void)self;
@@ -130,6 +151,7 @@ static VALUE convolver_convolve_basic_valid(VALUE self, VALUE signal, VALUE kern
130
151
 
131
152
  void Init_convolver(void) {
132
153
  mConvolver = rb_define_module("Convolver");
154
+ convolver_init_fft_shape(mConvolver);
133
155
  rb_define_singleton_method(mConvolver, "correlate_basic_valid", convolver_correlate_basic_valid, 2);
134
156
  rb_define_singleton_method(mConvolver, "convolve_basic_valid", convolver_convolve_basic_valid, 2);
135
157
  }
@@ -1,167 +1,57 @@
1
- // ext/convolver/correlate_raw.c
2
-
3
1
  #include "correlate_raw.h"
2
+ #include "raw_helpers.h"
4
3
 
5
- static inline size_t checked_add( size_t left, size_t right, const char *description ) {
6
- if ( right > SIZE_MAX - left ) {
7
- rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
8
- }
9
- return left + right;
10
- }
11
-
12
- static inline size_t checked_multiply( size_t left, size_t right, const char *description ) {
13
- if ( left != 0 && right > SIZE_MAX / left ) {
14
- rb_raise( rb_eRangeError, "%s exceeds native implementation limit", description );
15
- }
16
- return left * right;
17
- }
18
-
19
- static inline size_t size_from_shape( int rank, const size_t *shape, const char *description ) {
20
- size_t size = 1;
21
- int i;
22
- for ( i = 0; i < rank; i++ ) {
23
- size = checked_multiply( size, shape[i], description );
24
- }
25
- return size;
26
- }
27
-
28
- // Sets reverse indices
29
- static inline void corner_reset( int rank, const size_t *shape, size_t *rev_indices ) {
30
- int i;
31
- for ( i = 0; i < rank; i++ ) { rev_indices[i] = shape[i] - 1; }
32
- }
33
-
34
- // Counts indices down, returns number of ranks that reset
35
- static inline int corner_dec( int rank, const size_t *shape, size_t *rev_indices ) {
36
- int i = 0;
37
- (void) rank;
38
- while ( ! rev_indices[i]-- ) {
39
- rev_indices[i] = shape[i] - 1;
40
- i++;
41
- }
42
- return i;
43
- }
44
-
45
- // Generates co-increment steps by rank boundaries crossed, for the outer position as inner position is incremented by 1
46
- static inline void calc_co_increment(
47
- int rank, const size_t *outer_shape, const size_t *inner_shape, size_t *co_increment ) {
48
- size_t factor = 1;
49
- int i;
50
- co_increment[0] = 1; // co-increment is always 1 in lowest rank
51
- for ( i = 0; i < rank; i++ ) {
52
- size_t skipped = checked_multiply( factor, outer_shape[i] - inner_shape[i], "array offset" );
53
- co_increment[i+1] = checked_add( co_increment[i], skipped, "array offset" );
54
- factor = checked_multiply( factor, outer_shape[i], "array size" );
55
- }
56
- }
57
-
58
- static inline size_t maximum_offset(
59
- int rank, const size_t *outer_shape, const size_t *inner_shape, const char *description ) {
60
- size_t factor = 1;
61
- size_t offset = 0;
62
- int i;
63
- for ( i = 0; i < rank; i++ ) {
64
- size_t dimension_offset = checked_multiply( factor, inner_shape[i] - 1, description );
65
- offset = checked_add( offset, dimension_offset, description );
66
- factor = checked_multiply( factor, outer_shape[i], "array size" );
67
- }
68
- return offset;
69
- }
70
-
71
- ////////////////////////////////////////////////////////////////////////////////////////////////////
72
- //
73
- // Correlate
74
- //
75
-
76
- void correlate_raw(
77
- int in_rank, const size_t *in_shape, const float *in_ptr,
78
- int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
79
- int out_rank, const size_t *out_shape, float *out_ptr ) {
80
- size_t i, j, input_size, kernel_size, kernel_aligned, out_size, offset;
81
- size_t maximum_input_offset;
82
- size_t out_co_incr[LARGEST_RANK + 1], kernel_co_incr[LARGEST_RANK + 1];
83
- size_t ker_q[LARGEST_RANK], out_q[LARGEST_RANK];
84
- size_t *kernel_co_incr_cache;
85
- VALUE cache_storage = 0;
86
-
87
- kernel_size = size_from_shape( kernel_rank, kernel_shape, "kernel size" );
88
- kernel_aligned = kernel_size - kernel_size % 4;
89
- out_size = size_from_shape( out_rank, out_shape, "output size" );
90
- input_size = size_from_shape( in_rank, in_shape, "input size" );
91
-
92
- calc_co_increment( in_rank, in_shape, out_shape, out_co_incr );
93
- calc_co_increment( in_rank, in_shape, kernel_shape, kernel_co_incr );
94
- maximum_input_offset = checked_add(
95
- maximum_offset( in_rank, in_shape, out_shape, "input offset" ),
96
- maximum_offset( in_rank, in_shape, kernel_shape, "input offset" ),
97
- "input offset"
98
- );
99
- if ( maximum_input_offset >= input_size ) {
100
- rb_raise( rb_eRangeError, "input offset exceeds native implementation limit" );
101
- }
102
-
103
- kernel_co_incr_cache = RB_ALLOCV_N( size_t, cache_storage, kernel_size );
104
- kernel_co_incr_cache[0] = 0;
105
-
106
- corner_reset( kernel_rank, kernel_shape, ker_q );
107
- for ( i = 1; i < kernel_size; i++ ) {
108
- kernel_co_incr_cache[i] = checked_add(
109
- kernel_co_incr_cache[i-1],
110
- kernel_co_incr[ corner_dec( kernel_rank, kernel_shape, ker_q ) ],
111
- "kernel offset"
112
- );
113
- }
114
-
115
- offset = 0;
116
- corner_reset( out_rank, out_shape, out_q );
117
-
118
- // Main correlation loop
119
- for ( i = 0; i < out_size; i++ ) {
120
- #if CONVOLVER_USE_SSE
121
- __m128 simd_x, simd_y, simd_t;
122
- float v[4];
123
- simd_t = _mm_setzero_ps();
124
- #endif
125
- float t = 0.0;
126
-
127
- #if CONVOLVER_USE_SSE
128
- // Contiguous kernel views need not start on a 16-byte boundary.
129
- for ( j = 0; j < kernel_aligned; j +=4 ) {
130
- simd_x = _mm_loadu_ps( kernel_ptr + j );
131
- // Yes the backwards alignment is correct
132
- simd_y = _mm_set_ps( in_ptr[ offset + kernel_co_incr_cache[j+3] ], in_ptr[ offset + kernel_co_incr_cache[j+2] ],
133
- in_ptr[ offset + kernel_co_incr_cache[j+1] ], in_ptr[ offset + kernel_co_incr_cache[j] ] );
134
- simd_x = _mm_mul_ps( simd_x, simd_y );
135
- simd_t = _mm_add_ps( simd_x, simd_t );
136
- }
137
- _mm_store_ps( v, simd_t );
4
+ #define RAW_INDEX(j) (j)
5
+ #if defined(__SSE__)
6
+ #define RAW_SIMD 1
138
7
  #else
139
- // SSE is unavailable on architectures such as Apple Silicon.
140
- // Use the same calculation without x86-specific intrinsics.
141
- for ( j = 0; j < kernel_aligned; j++ ) {
142
- t += in_ptr[ offset + kernel_co_incr_cache[j] ] * kernel_ptr[j];
143
- }
8
+ #define RAW_SIMD 0
144
9
  #endif
145
-
146
- // Complete any remaining 1,2 or 3 items one at a time
147
- for ( j = kernel_aligned; j < kernel_size; j++ ) {
148
- t += in_ptr[ offset + kernel_co_incr_cache[j] ] * kernel_ptr[ j ];
149
- }
150
-
151
- #if CONVOLVER_USE_SSE
152
- out_ptr[i] = v[0] + v[1] + v[2] + v[3] + t;
10
+ #define RAW_NAME correlate_raw
11
+ #define RAW_TYPE float
12
+ #define RAW_LANES 4
13
+ #define RAW_VECTOR __m128
14
+ #define RAW_ZERO _mm_setzero_ps
15
+ #define RAW_LOAD _mm_loadu_ps
16
+ #define RAW_STORE _mm_storeu_ps
17
+ #define RAW_ADD _mm_add_ps
18
+ #define RAW_MUL _mm_mul_ps
19
+ #include "raw_loop.h"
20
+ #undef RAW_NAME
21
+ #undef RAW_TYPE
22
+ #undef RAW_LANES
23
+ #undef RAW_VECTOR
24
+ #undef RAW_ZERO
25
+ #undef RAW_LOAD
26
+ #undef RAW_STORE
27
+ #undef RAW_ADD
28
+ #undef RAW_MUL
29
+ #undef RAW_SIMD
30
+
31
+ #if defined(__SSE2__)
32
+ #define RAW_SIMD 1
153
33
  #else
154
- out_ptr[i] = t;
34
+ #define RAW_SIMD 0
155
35
  #endif
156
-
157
- if ( i + 1 < out_size ) {
158
- offset = checked_add(
159
- offset,
160
- out_co_incr[ corner_dec( out_rank, out_shape, out_q ) ],
161
- "input offset"
162
- );
163
- }
164
- }
165
-
166
- RB_ALLOCV_END( cache_storage );
167
- }
36
+ #define RAW_NAME correlate_raw_double
37
+ #define RAW_TYPE double
38
+ #define RAW_LANES 2
39
+ #define RAW_VECTOR __m128d
40
+ #define RAW_ZERO _mm_setzero_pd
41
+ #define RAW_LOAD _mm_loadu_pd
42
+ #define RAW_STORE _mm_storeu_pd
43
+ #define RAW_ADD _mm_add_pd
44
+ #define RAW_MUL _mm_mul_pd
45
+ #include "raw_loop.h"
46
+ #undef RAW_NAME
47
+ #undef RAW_TYPE
48
+ #undef RAW_LANES
49
+ #undef RAW_VECTOR
50
+ #undef RAW_ZERO
51
+ #undef RAW_LOAD
52
+ #undef RAW_STORE
53
+ #undef RAW_ADD
54
+ #undef RAW_MUL
55
+ #undef RAW_SIMD
56
+
57
+ #undef RAW_INDEX
@@ -1,28 +1,14 @@
1
- // ext/convolver/correlate_raw.h
2
-
3
- ////////////////////////////////////////////////////////////////////////////////////////////////
4
- //
5
- // Declarations of narray helper functions
6
- //
7
-
8
1
  #ifndef CORRELATE_RAW_H
9
2
  #define CORRELATE_RAW_H
10
-
11
- #ifndef CONVOLVER_RAW_CONFIG
12
- #define CONVOLVER_RAW_CONFIG
13
- #include <ruby.h>
14
- #if defined(__SSE__)
15
- #include <xmmintrin.h>
16
- #define CONVOLVER_USE_SSE 1
17
- #else
18
- #define CONVOLVER_USE_SSE 0
19
- #endif
20
- #define LARGEST_RANK 16
21
- #endif
3
+ #include "raw_config.h"
22
4
 
23
5
  void correlate_raw(
24
6
  int in_rank, const size_t *in_shape, const float *in_ptr,
25
7
  int kernel_rank, const size_t *kernel_shape, const float *kernel_ptr,
26
8
  int out_rank, const size_t *out_shape, float *out_ptr );
27
9
 
10
+ void correlate_raw_double(
11
+ int in_rank, const size_t *in_shape, const double *in_ptr,
12
+ int kernel_rank, const size_t *kernel_shape, const double *kernel_ptr,
13
+ int out_rank, const size_t *out_shape, double *out_ptr );
28
14
  #endif
@@ -11,6 +11,7 @@ $LOAD_PATH.each do |load_path|
11
11
  end
12
12
 
13
13
  abort 'numo/narray.h not found' unless have_header('numo/narray.h')
14
+ abort 'log2 not found' unless have_func('log2', 'math.h') || have_library('m', 'log2', 'math.h')
14
15
 
15
16
  if RUBY_PLATFORM.match?(/mswin|cygwin|mingw/)
16
17
  $LOAD_PATH.each do |load_path|