tilework-hip 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +81 -0
- data/ext/tilework/hip/bridge.cpp +1067 -0
- data/ext/tilework/hip/extconf.rb +66 -0
- data/ext/tilework/hip/gpu_bridge.c +840 -0
- data/lib/tilework/backend/hip.rb +21 -0
- metadata +65 -0
|
@@ -0,0 +1,1067 @@
|
|
|
1
|
+
#include <hip/hip_runtime_api.h>
|
|
2
|
+
#include <hip/hiprtc.h>
|
|
3
|
+
#ifdef TILEWORK_HAS_HIPBLAS
|
|
4
|
+
#include <hipblas/hipblas.h>
|
|
5
|
+
#endif
|
|
6
|
+
#ifdef TILEWORK_HAS_HIPRAND
|
|
7
|
+
#include <hiprand/hiprand.h>
|
|
8
|
+
#endif
|
|
9
|
+
#ifdef TILEWORK_HAS_HIPFFT
|
|
10
|
+
#include <hipfft/hipfft.h>
|
|
11
|
+
#endif
|
|
12
|
+
#ifdef TILEWORK_HAS_MIOPEN
|
|
13
|
+
#include <miopen/miopen.h>
|
|
14
|
+
#endif
|
|
15
|
+
|
|
16
|
+
#include <charconv>
|
|
17
|
+
#include <algorithm>
|
|
18
|
+
#include <cstdio>
|
|
19
|
+
#include <cstdlib>
|
|
20
|
+
#include <cstring>
|
|
21
|
+
#include <mutex>
|
|
22
|
+
#include <atomic>
|
|
23
|
+
#include <map>
|
|
24
|
+
#include <string>
|
|
25
|
+
#include <tuple>
|
|
26
|
+
#include <thread>
|
|
27
|
+
#include <vector>
|
|
28
|
+
|
|
29
|
+
namespace {
|
|
30
|
+
struct Pipeline;
|
|
31
|
+
std::mutex g_mutex;
|
|
32
|
+
bool g_failed = false;
|
|
33
|
+
bool g_shutting_down = false;
|
|
34
|
+
int g_device = -1;
|
|
35
|
+
hipDeviceProp_t g_props{};
|
|
36
|
+
hipStream_t g_stream = nullptr;
|
|
37
|
+
std::string g_error;
|
|
38
|
+
constexpr size_t HOST_STAGING_BYTES = 1 << 20;
|
|
39
|
+
std::mutex g_staging_mutex;
|
|
40
|
+
void *g_staging = nullptr;
|
|
41
|
+
bool g_staging_attempted = false;
|
|
42
|
+
std::mutex g_matrix_mutex;
|
|
43
|
+
bool g_matrix_probed = false;
|
|
44
|
+
bool g_matrix_available = false;
|
|
45
|
+
Pipeline *g_matrix_pipeline = nullptr;
|
|
46
|
+
#ifdef TILEWORK_HAS_HIPBLAS
|
|
47
|
+
hipblasHandle_t g_blas = nullptr;
|
|
48
|
+
bool g_blas_failed = false;
|
|
49
|
+
bool g_blas_probed = false;
|
|
50
|
+
bool g_blas_available = false;
|
|
51
|
+
#endif
|
|
52
|
+
#ifdef TILEWORK_HAS_HIPFFT
|
|
53
|
+
std::map<std::tuple<int, long, long>, hipfftHandle> g_fft_plans;
|
|
54
|
+
bool g_fft_probed = false;
|
|
55
|
+
bool g_fft_available = false;
|
|
56
|
+
#endif
|
|
57
|
+
#ifdef TILEWORK_HAS_HIPRAND
|
|
58
|
+
hiprandGenerator_t g_rng = nullptr;
|
|
59
|
+
bool g_rng_failed = false;
|
|
60
|
+
bool g_rng_probed = false;
|
|
61
|
+
bool g_rng_available = false;
|
|
62
|
+
#endif
|
|
63
|
+
#ifdef TILEWORK_HAS_MIOPEN
|
|
64
|
+
struct MIOpenPlan {
|
|
65
|
+
miopenTensorDescriptor_t x = nullptr, w = nullptr, y = nullptr;
|
|
66
|
+
miopenConvolutionDescriptor_t conv = nullptr;
|
|
67
|
+
miopenConvFwdAlgorithm_t algorithm{};
|
|
68
|
+
void *workspace = nullptr;
|
|
69
|
+
size_t workspace_bytes = 0;
|
|
70
|
+
miopenConvBwdDataAlgorithm_t backward_data_algorithm{};
|
|
71
|
+
void *backward_data_workspace = nullptr;
|
|
72
|
+
size_t backward_data_workspace_bytes = 0;
|
|
73
|
+
bool backward_data_ready = false;
|
|
74
|
+
miopenConvBwdWeightsAlgorithm_t backward_weights_algorithm{};
|
|
75
|
+
void *backward_weights_workspace = nullptr;
|
|
76
|
+
size_t backward_weights_workspace_bytes = 0;
|
|
77
|
+
bool backward_weights_ready = false;
|
|
78
|
+
};
|
|
79
|
+
miopenHandle_t g_miopen = nullptr;
|
|
80
|
+
bool g_miopen_failed = false;
|
|
81
|
+
bool g_miopen_probed = false;
|
|
82
|
+
bool g_miopen_available = false;
|
|
83
|
+
bool g_miopen_backward_probed = false;
|
|
84
|
+
bool g_miopen_backward_available = false;
|
|
85
|
+
std::map<std::string, MIOpenPlan *> g_miopen_plans;
|
|
86
|
+
#endif
|
|
87
|
+
|
|
88
|
+
#ifdef TILEWORK_HAS_HIPFFT
|
|
89
|
+
bool ensure_runtime();
|
|
90
|
+
std::string hipfft_error(const char *operation, hipfftResult status) {
|
|
91
|
+
return std::string(operation) + ": hipFFT status " + std::to_string(static_cast<int>(status));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
bool fft_plan(int kind, long n, long batch, hipfftHandle *out) {
|
|
95
|
+
if (!ensure_runtime()) return false;
|
|
96
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
97
|
+
auto key = std::make_tuple(kind, n, batch);
|
|
98
|
+
auto found = g_fft_plans.find(key);
|
|
99
|
+
if (found != g_fft_plans.end()) { *out = found->second; return true; }
|
|
100
|
+
hipfftType type = kind == 0 ? HIPFFT_R2C : (kind == 1 ? HIPFFT_C2R : HIPFFT_C2C);
|
|
101
|
+
hipfftHandle plan{};
|
|
102
|
+
int length = static_cast<int>(n);
|
|
103
|
+
hipfftResult status = hipfftPlanMany(&plan, 1, &length, nullptr, 1, 0, nullptr, 1, 0, type, static_cast<int>(batch));
|
|
104
|
+
if (status == HIPFFT_SUCCESS) status = hipfftSetStream(plan, g_stream);
|
|
105
|
+
if (status != HIPFFT_SUCCESS) {
|
|
106
|
+
if (plan) (void)hipfftDestroy(plan);
|
|
107
|
+
g_error = hipfft_error("hipFFT plan creation", status);
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
g_fft_plans.emplace(key, plan);
|
|
111
|
+
*out = plan;
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
#endif
|
|
115
|
+
|
|
116
|
+
bool ensure_runtime();
|
|
117
|
+
std::string hip_error(const char *operation, hipError_t status);
|
|
118
|
+
|
|
119
|
+
bool ensure_staging() {
|
|
120
|
+
std::lock_guard<std::mutex> lock(g_staging_mutex);
|
|
121
|
+
if (g_staging) return true;
|
|
122
|
+
if (g_staging_attempted) return false;
|
|
123
|
+
g_staging_attempted = true;
|
|
124
|
+
hipError_t status = hipHostMalloc(&g_staging, HOST_STAGING_BYTES, hipHostMallocDefault);
|
|
125
|
+
if (status != hipSuccess) {
|
|
126
|
+
g_staging = nullptr;
|
|
127
|
+
g_error = hip_error("HIP pinned staging allocation", status);
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
#ifdef TILEWORK_HAS_MIOPEN
|
|
134
|
+
std::string miopen_error(const char *operation, miopenStatus_t status) {
|
|
135
|
+
return std::string(operation) + ": MIOpen status " + std::to_string(static_cast<int>(status));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
void miopen_plan_free(MIOpenPlan *plan) {
|
|
139
|
+
if (!plan) return;
|
|
140
|
+
if (plan->workspace) (void)hipFree(plan->workspace);
|
|
141
|
+
if (plan->backward_data_workspace) (void)hipFree(plan->backward_data_workspace);
|
|
142
|
+
if (plan->backward_weights_workspace) (void)hipFree(plan->backward_weights_workspace);
|
|
143
|
+
if (plan->x) (void)miopenDestroyTensorDescriptor(plan->x);
|
|
144
|
+
if (plan->w) (void)miopenDestroyTensorDescriptor(plan->w);
|
|
145
|
+
if (plan->y) (void)miopenDestroyTensorDescriptor(plan->y);
|
|
146
|
+
if (plan->conv) (void)miopenDestroyConvolutionDescriptor(plan->conv);
|
|
147
|
+
delete plan;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
bool ensure_miopen() {
|
|
151
|
+
if (!ensure_runtime()) return false;
|
|
152
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
153
|
+
if (g_miopen) return true;
|
|
154
|
+
if (g_miopen_failed) return false;
|
|
155
|
+
miopenStatus_t status = miopenCreateWithStream(&g_miopen, reinterpret_cast<miopenAcceleratorQueue_t>(g_stream));
|
|
156
|
+
if (status != miopenStatusSuccess) {
|
|
157
|
+
g_miopen = nullptr;
|
|
158
|
+
g_miopen_failed = true;
|
|
159
|
+
g_error = miopen_error("miopenCreateWithStream", status);
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
std::string miopen_key(int n, int cin, int h, int width, int cout, int kh, int kw,
|
|
166
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
167
|
+
int dilation_h, int dilation_w, int groups, int dtype) {
|
|
168
|
+
return std::to_string(n) + ":" + std::to_string(cin) + ":" + std::to_string(h) + ":" + std::to_string(width) + ":" +
|
|
169
|
+
std::to_string(cout) + ":" + std::to_string(kh) + ":" + std::to_string(kw) + ":" +
|
|
170
|
+
std::to_string(pad_h) + ":" + std::to_string(pad_w) + ":" + std::to_string(stride_h) + ":" + std::to_string(stride_w) + ":" +
|
|
171
|
+
std::to_string(dilation_h) + ":" + std::to_string(dilation_w) + ":" + std::to_string(groups) + ":" + std::to_string(dtype);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
bool miopen_plan(MIOpenPlan **out, void *x, void *w, void *y,
|
|
175
|
+
int n, int cin, int h, int width, int cout, int kh, int kw,
|
|
176
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
177
|
+
int dilation_h, int dilation_w, int groups, int dtype) {
|
|
178
|
+
if (!ensure_miopen()) return false;
|
|
179
|
+
const std::string key = miopen_key(n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
180
|
+
dilation_h, dilation_w, groups, dtype);
|
|
181
|
+
{
|
|
182
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
183
|
+
auto found = g_miopen_plans.find(key);
|
|
184
|
+
if (found != g_miopen_plans.end()) { *out = found->second; return true; }
|
|
185
|
+
}
|
|
186
|
+
auto *plan = new MIOpenPlan;
|
|
187
|
+
const miopenDataType_t data_type = dtype == 0 ? miopenFloat :
|
|
188
|
+
(dtype == 1 ? miopenHalf : miopenBFloat16);
|
|
189
|
+
int oh = 0, ow = 0, on = 0, oc = 0, returned = 0;
|
|
190
|
+
miopenStatus_t status = miopenCreateTensorDescriptor(&plan->x);
|
|
191
|
+
if (status == miopenStatusSuccess) status = miopenCreateTensorDescriptor(&plan->w);
|
|
192
|
+
if (status == miopenStatusSuccess) status = miopenCreateTensorDescriptor(&plan->y);
|
|
193
|
+
if (status == miopenStatusSuccess) status = miopenCreateConvolutionDescriptor(&plan->conv);
|
|
194
|
+
if (status == miopenStatusSuccess) status = miopenSet4dTensorDescriptor(plan->x, data_type, n, cin, h, width);
|
|
195
|
+
if (status == miopenStatusSuccess) status = miopenSet4dTensorDescriptor(plan->w, data_type, cout, cin / groups, kh, kw);
|
|
196
|
+
if (status == miopenStatusSuccess) status = miopenInitConvolutionDescriptor(plan->conv, miopenConvolution,
|
|
197
|
+
pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w);
|
|
198
|
+
if (status == miopenStatusSuccess && groups != 1) status = miopenSetConvolutionGroupCount(plan->conv, groups);
|
|
199
|
+
if (status == miopenStatusSuccess) status = miopenGetConvolutionForwardOutputDim(plan->conv, plan->x, plan->w, &on, &oc, &oh, &ow);
|
|
200
|
+
if (status == miopenStatusSuccess && (on != n || oc != cout || oh <= 0 || ow <= 0)) status = miopenStatusBadParm;
|
|
201
|
+
if (status == miopenStatusSuccess) status = miopenSet4dTensorDescriptor(plan->y, data_type, on, oc, oh, ow);
|
|
202
|
+
if (status == miopenStatusSuccess) status = miopenConvolutionForwardGetWorkSpaceSize(g_miopen, plan->w, plan->x, plan->conv, plan->y, &plan->workspace_bytes);
|
|
203
|
+
hipError_t hip_status = hipSuccess;
|
|
204
|
+
if (status == miopenStatusSuccess && plan->workspace_bytes != 0) hip_status = hipMalloc(&plan->workspace, plan->workspace_bytes);
|
|
205
|
+
miopenConvAlgoPerf_t perf{};
|
|
206
|
+
if (status == miopenStatusSuccess && hip_status == hipSuccess)
|
|
207
|
+
status = miopenFindConvolutionForwardAlgorithm(g_miopen, plan->x, x, plan->w, w, plan->conv, plan->y, y,
|
|
208
|
+
1, &returned, &perf, plan->workspace, plan->workspace_bytes, false);
|
|
209
|
+
if (status == miopenStatusSuccess && returned != 1) status = miopenStatusUnknownError;
|
|
210
|
+
if (status != miopenStatusSuccess || hip_status != hipSuccess) {
|
|
211
|
+
g_error = status != miopenStatusSuccess ? miopen_error("MIOpen forward algorithm search", status)
|
|
212
|
+
: hip_error("MIOpen workspace allocation", hip_status);
|
|
213
|
+
miopen_plan_free(plan);
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
plan->algorithm = perf.fwd_algo;
|
|
217
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
218
|
+
auto [where, inserted] = g_miopen_plans.emplace(key, plan);
|
|
219
|
+
if (!inserted) { miopen_plan_free(plan); }
|
|
220
|
+
*out = where->second;
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
bool miopen_prepare_backward_data(MIOpenPlan *plan, void *dy, void *w, void *dx) {
|
|
225
|
+
if (plan->backward_data_ready) return true;
|
|
226
|
+
miopenStatus_t status = miopenConvolutionBackwardDataGetWorkSpaceSize(g_miopen, plan->y, plan->w, plan->conv, plan->x,
|
|
227
|
+
&plan->backward_data_workspace_bytes);
|
|
228
|
+
hipError_t hip_status = hipSuccess;
|
|
229
|
+
if (status == miopenStatusSuccess && plan->backward_data_workspace_bytes != 0)
|
|
230
|
+
hip_status = hipMalloc(&plan->backward_data_workspace, plan->backward_data_workspace_bytes);
|
|
231
|
+
miopenConvAlgoPerf_t perf{};
|
|
232
|
+
int returned = 0;
|
|
233
|
+
if (status == miopenStatusSuccess && hip_status == hipSuccess)
|
|
234
|
+
status = miopenFindConvolutionBackwardDataAlgorithm(g_miopen, plan->y, dy, plan->w, w, plan->conv, plan->x, dx,
|
|
235
|
+
1, &returned, &perf, plan->backward_data_workspace,
|
|
236
|
+
plan->backward_data_workspace_bytes, false);
|
|
237
|
+
if (status == miopenStatusSuccess && returned != 1) status = miopenStatusUnknownError;
|
|
238
|
+
if (status != miopenStatusSuccess || hip_status != hipSuccess) {
|
|
239
|
+
g_error = status != miopenStatusSuccess ? miopen_error("MIOpen backward-data algorithm search", status)
|
|
240
|
+
: hip_error("MIOpen backward-data workspace allocation", hip_status);
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
plan->backward_data_algorithm = perf.bwd_data_algo;
|
|
244
|
+
plan->backward_data_ready = true;
|
|
245
|
+
return true;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
bool miopen_prepare_backward_weights(MIOpenPlan *plan, void *x, void *dy, void *dw) {
|
|
249
|
+
if (plan->backward_weights_ready) return true;
|
|
250
|
+
miopenStatus_t status = miopenConvolutionBackwardWeightsGetWorkSpaceSize(g_miopen, plan->y, plan->x, plan->conv, plan->w,
|
|
251
|
+
&plan->backward_weights_workspace_bytes);
|
|
252
|
+
hipError_t hip_status = hipSuccess;
|
|
253
|
+
if (status == miopenStatusSuccess && plan->backward_weights_workspace_bytes != 0)
|
|
254
|
+
hip_status = hipMalloc(&plan->backward_weights_workspace, plan->backward_weights_workspace_bytes);
|
|
255
|
+
miopenConvAlgoPerf_t perf{};
|
|
256
|
+
int returned = 0;
|
|
257
|
+
if (status == miopenStatusSuccess && hip_status == hipSuccess)
|
|
258
|
+
status = miopenFindConvolutionBackwardWeightsAlgorithm(g_miopen, plan->y, dy, plan->x, x, plan->conv, plan->w, dw,
|
|
259
|
+
1, &returned, &perf, plan->backward_weights_workspace,
|
|
260
|
+
plan->backward_weights_workspace_bytes, false);
|
|
261
|
+
if (status == miopenStatusSuccess && returned != 1) status = miopenStatusUnknownError;
|
|
262
|
+
if (status != miopenStatusSuccess || hip_status != hipSuccess) {
|
|
263
|
+
g_error = status != miopenStatusSuccess ? miopen_error("MIOpen backward-filter algorithm search", status)
|
|
264
|
+
: hip_error("MIOpen backward-filter workspace allocation", hip_status);
|
|
265
|
+
return false;
|
|
266
|
+
}
|
|
267
|
+
plan->backward_weights_algorithm = perf.bwd_weights_algo;
|
|
268
|
+
plan->backward_weights_ready = true;
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
MIOpenPlan *miopen_existing_plan(int n, int cin, int h, int width, int cout, int kh, int kw,
|
|
273
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
274
|
+
int dilation_h, int dilation_w, int groups, int dtype) {
|
|
275
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
276
|
+
auto found = g_miopen_plans.find(miopen_key(n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
277
|
+
dilation_h, dilation_w, groups, dtype));
|
|
278
|
+
if (found == g_miopen_plans.end()) {
|
|
279
|
+
g_error = "MIOpen convolution backward requires the matching forward plan to run first";
|
|
280
|
+
return nullptr;
|
|
281
|
+
}
|
|
282
|
+
return found->second;
|
|
283
|
+
}
|
|
284
|
+
#endif
|
|
285
|
+
|
|
286
|
+
const char *rocm_include_dir() {
|
|
287
|
+
static std::string path;
|
|
288
|
+
if (path.empty()) {
|
|
289
|
+
const char *home = getenv("ROCM_HOME");
|
|
290
|
+
path = std::string((home && *home) ? home : "/opt/rocm") + "/include";
|
|
291
|
+
}
|
|
292
|
+
return path.c_str();
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
char *dup(const std::string &s) {
|
|
296
|
+
char *out = static_cast<char *>(malloc(s.size() + 1));
|
|
297
|
+
if (out) memcpy(out, s.c_str(), s.size() + 1);
|
|
298
|
+
return out;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
std::string hip_error(const char *operation, hipError_t status) {
|
|
302
|
+
return std::string(operation) + ": " + hipGetErrorString(status) +
|
|
303
|
+
" (" + std::to_string(static_cast<int>(status)) + ")";
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
bool ensure_runtime();
|
|
307
|
+
|
|
308
|
+
#ifdef TILEWORK_HAS_HIPBLAS
|
|
309
|
+
std::string hipblas_error(const char *operation, hipblasStatus_t status) {
|
|
310
|
+
return std::string(operation) + ": hipBLAS status " + std::to_string(static_cast<int>(status));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
bool ensure_blas() {
|
|
314
|
+
if (!ensure_runtime()) return false;
|
|
315
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
316
|
+
if (g_blas) return true;
|
|
317
|
+
if (g_blas_failed) return false;
|
|
318
|
+
hipblasStatus_t status = hipblasCreate(&g_blas);
|
|
319
|
+
if (status != HIPBLAS_STATUS_SUCCESS) {
|
|
320
|
+
g_blas = nullptr;
|
|
321
|
+
g_blas_failed = true;
|
|
322
|
+
g_error = hipblas_error("hipblasCreate", status);
|
|
323
|
+
return false;
|
|
324
|
+
}
|
|
325
|
+
status = hipblasSetStream(g_blas, g_stream);
|
|
326
|
+
if (status != HIPBLAS_STATUS_SUCCESS) {
|
|
327
|
+
(void)hipblasDestroy(g_blas);
|
|
328
|
+
g_blas = nullptr;
|
|
329
|
+
g_blas_failed = true;
|
|
330
|
+
g_error = hipblas_error("hipblasSetStream", status);
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
return true;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
bool run_blas_probe() {
|
|
337
|
+
if (g_blas_probed) return g_blas_available;
|
|
338
|
+
g_blas_probed = true;
|
|
339
|
+
if (!ensure_blas()) return false;
|
|
340
|
+
const float a[4] = {1, 2, 3, 4};
|
|
341
|
+
const float b[4] = {5, 6, 7, 8};
|
|
342
|
+
float c[4] = {};
|
|
343
|
+
float *da = nullptr, *db = nullptr, *dc = nullptr;
|
|
344
|
+
hipError_t hip_status = hipMalloc(&da, sizeof(a));
|
|
345
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&db, sizeof(b));
|
|
346
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&dc, sizeof(c));
|
|
347
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(da, a, sizeof(a), hipMemcpyHostToDevice, g_stream);
|
|
348
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(db, b, sizeof(b), hipMemcpyHostToDevice, g_stream);
|
|
349
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
350
|
+
hipblasStatus_t blas_status = hip_status == hipSuccess
|
|
351
|
+
? hipblasSgemm(g_blas, HIPBLAS_OP_N, HIPBLAS_OP_N, 2, 2, 2,
|
|
352
|
+
&alpha, db, 2, da, 2, &beta, dc, 2)
|
|
353
|
+
: HIPBLAS_STATUS_INTERNAL_ERROR;
|
|
354
|
+
if (hip_status == hipSuccess && blas_status == HIPBLAS_STATUS_SUCCESS)
|
|
355
|
+
hip_status = hipMemcpyAsync(c, dc, sizeof(c), hipMemcpyDeviceToHost, g_stream);
|
|
356
|
+
if (hip_status == hipSuccess && blas_status == HIPBLAS_STATUS_SUCCESS)
|
|
357
|
+
hip_status = hipStreamSynchronize(g_stream);
|
|
358
|
+
if (da) (void)hipFree(da);
|
|
359
|
+
if (db) (void)hipFree(db);
|
|
360
|
+
if (dc) (void)hipFree(dc);
|
|
361
|
+
if (hip_status != hipSuccess) g_error = hip_error("hipBLAS execution probe", hip_status);
|
|
362
|
+
else if (blas_status != HIPBLAS_STATUS_SUCCESS) g_error = hipblas_error("hipblasSgemm execution probe", blas_status);
|
|
363
|
+
else if (c[0] != 19 || c[1] != 22 || c[2] != 43 || c[3] != 50)
|
|
364
|
+
g_error = "hipBLAS execution probe returned an incorrect result";
|
|
365
|
+
else g_blas_available = true;
|
|
366
|
+
return g_blas_available;
|
|
367
|
+
}
|
|
368
|
+
#endif
|
|
369
|
+
|
|
370
|
+
#ifdef TILEWORK_HAS_HIPRAND
|
|
371
|
+
std::string hiprand_error(const char *operation, hiprandStatus_t status) {
|
|
372
|
+
return std::string(operation) + ": hipRAND status " + std::to_string(static_cast<int>(status));
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
bool ensure_rng() {
|
|
376
|
+
if (!ensure_runtime()) return false;
|
|
377
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
378
|
+
if (g_rng) return true;
|
|
379
|
+
if (g_rng_failed) return false;
|
|
380
|
+
hiprandStatus_t status = hiprandCreateGenerator(&g_rng, HIPRAND_RNG_PSEUDO_PHILOX4_32_10);
|
|
381
|
+
if (status == HIPRAND_STATUS_SUCCESS) status = hiprandSetStream(g_rng, g_stream);
|
|
382
|
+
if (status != HIPRAND_STATUS_SUCCESS) {
|
|
383
|
+
if (g_rng) (void)hiprandDestroyGenerator(g_rng);
|
|
384
|
+
g_rng = nullptr;
|
|
385
|
+
g_rng_failed = true;
|
|
386
|
+
g_error = hiprand_error("hipRAND initialization", status);
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
bool run_rng_probe() {
|
|
393
|
+
if (g_rng_probed) return g_rng_available;
|
|
394
|
+
g_rng_probed = true;
|
|
395
|
+
if (!ensure_rng()) return false;
|
|
396
|
+
float *device = nullptr;
|
|
397
|
+
float host[4] = {};
|
|
398
|
+
hipError_t hip_status = hipMalloc(&device, sizeof(host));
|
|
399
|
+
hiprandStatus_t rand_status = hip_status == hipSuccess
|
|
400
|
+
? hiprandSetPseudoRandomGeneratorSeed(g_rng, 1234) : HIPRAND_STATUS_INTERNAL_ERROR;
|
|
401
|
+
if (rand_status == HIPRAND_STATUS_SUCCESS) rand_status = hiprandSetGeneratorOffset(g_rng, 0);
|
|
402
|
+
if (rand_status == HIPRAND_STATUS_SUCCESS) rand_status = hiprandGenerateUniform(g_rng, device, 4);
|
|
403
|
+
if (hip_status == hipSuccess && rand_status == HIPRAND_STATUS_SUCCESS)
|
|
404
|
+
hip_status = hipMemcpyAsync(host, device, sizeof(host), hipMemcpyDeviceToHost, g_stream);
|
|
405
|
+
if (hip_status == hipSuccess && rand_status == HIPRAND_STATUS_SUCCESS)
|
|
406
|
+
hip_status = hipStreamSynchronize(g_stream);
|
|
407
|
+
if (device) (void)hipFree(device);
|
|
408
|
+
if (hip_status != hipSuccess) g_error = hip_error("hipRAND execution probe", hip_status);
|
|
409
|
+
else if (rand_status != HIPRAND_STATUS_SUCCESS) g_error = hiprand_error("hipRAND execution probe", rand_status);
|
|
410
|
+
else if (!std::all_of(std::begin(host), std::end(host), [](float x) { return x > 0.0f && x <= 1.0f; }))
|
|
411
|
+
g_error = "hipRAND execution probe returned values outside (0, 1]";
|
|
412
|
+
else g_rng_available = true;
|
|
413
|
+
return g_rng_available;
|
|
414
|
+
}
|
|
415
|
+
#endif
|
|
416
|
+
bool selected_device(int *out) {
|
|
417
|
+
const char *value = getenv("TILEWORK_DEVICE");
|
|
418
|
+
if (!value || !*value) { *out = 0; return true; }
|
|
419
|
+
const char *end = value + strlen(value);
|
|
420
|
+
int selected = 0;
|
|
421
|
+
auto result = std::from_chars(value, end, selected, 10);
|
|
422
|
+
if (result.ec != std::errc() || result.ptr != end || selected < 0) {
|
|
423
|
+
g_error = std::string("invalid TILEWORK_DEVICE=") + value + "; expected a non-negative decimal index";
|
|
424
|
+
return false;
|
|
425
|
+
}
|
|
426
|
+
*out = selected;
|
|
427
|
+
return true;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
bool ensure_runtime() {
|
|
431
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
432
|
+
if (g_shutting_down) return false;
|
|
433
|
+
if (g_stream) return hipSetDevice(g_device) == hipSuccess;
|
|
434
|
+
if (g_failed) return false;
|
|
435
|
+
|
|
436
|
+
int requested = 0, count = 0;
|
|
437
|
+
if (!selected_device(&requested)) { g_failed = true; return false; }
|
|
438
|
+
hipError_t status = hipGetDeviceCount(&count);
|
|
439
|
+
if (status != hipSuccess) g_error = hip_error("hipGetDeviceCount", status);
|
|
440
|
+
else if (requested >= count) {
|
|
441
|
+
std::string names;
|
|
442
|
+
for (int i = 0; i < count; ++i) {
|
|
443
|
+
hipDeviceProp_t p{};
|
|
444
|
+
if (hipGetDeviceProperties(&p, i) == hipSuccess) {
|
|
445
|
+
if (!names.empty()) names += ", ";
|
|
446
|
+
names += p.name;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
g_error = "TILEWORK_DEVICE=" + std::to_string(requested) +
|
|
450
|
+
" is outside the visible HIP device count (" + std::to_string(count) +
|
|
451
|
+
"; devices: " + names + ")";
|
|
452
|
+
} else if ((status = hipSetDevice(requested)) != hipSuccess) g_error = hip_error("hipSetDevice", status);
|
|
453
|
+
else if ((status = hipGetDeviceProperties(&g_props, requested)) != hipSuccess) g_error = hip_error("hipGetDeviceProperties", status);
|
|
454
|
+
else if ((status = hipStreamCreateWithFlags(&g_stream, hipStreamNonBlocking)) != hipSuccess) g_error = hip_error("hipStreamCreateWithFlags", status);
|
|
455
|
+
else { g_device = requested; return true; }
|
|
456
|
+
g_failed = true;
|
|
457
|
+
if (g_stream) { (void)hipStreamDestroy(g_stream); g_stream = nullptr; }
|
|
458
|
+
return false;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
struct Pipeline { hipModule_t module; hipFunction_t function; };
|
|
462
|
+
struct HostBacking { void *data; long bytes; };
|
|
463
|
+
struct HostTransfer {
|
|
464
|
+
std::thread worker;
|
|
465
|
+
std::atomic<bool> done{false};
|
|
466
|
+
int status = 0;
|
|
467
|
+
std::string error;
|
|
468
|
+
};
|
|
469
|
+
std::atomic<int> g_live_host_transfers{0};
|
|
470
|
+
constexpr int MAX_HOST_TRANSFERS = 1;
|
|
471
|
+
|
|
472
|
+
void set_error(char **out, const std::string &message) { if (out) *out = dup(message); }
|
|
473
|
+
} // namespace
|
|
474
|
+
|
|
475
|
+
extern "C" {
|
|
476
|
+
char *gpu_bridge_device_name(void) {
|
|
477
|
+
return ensure_runtime() ? dup(g_props.name) : nullptr;
|
|
478
|
+
}
|
|
479
|
+
int gpu_bridge_device_available(void) { return ensure_runtime() ? 1 : 0; }
|
|
480
|
+
char *gpu_bridge_last_error(void) { return g_error.empty() ? nullptr : dup(g_error); }
|
|
481
|
+
char *gpu_bridge_device_architecture(void) {
|
|
482
|
+
return ensure_runtime() ? dup(g_props.gcnArchName) : nullptr;
|
|
483
|
+
}
|
|
484
|
+
int gpu_bridge_device_index(void) { return ensure_runtime() ? g_device : -1; }
|
|
485
|
+
|
|
486
|
+
void *gpu_bridge_buffer_alloc(long bytes) {
|
|
487
|
+
if (bytes <= 0 || !ensure_runtime()) return nullptr;
|
|
488
|
+
void *buffer = nullptr;
|
|
489
|
+
if (hipMalloc(&buffer, static_cast<size_t>(bytes)) != hipSuccess) return nullptr;
|
|
490
|
+
// The zero fill is queued on g_stream, and every kernel, copy and free this
|
|
491
|
+
// handle can reach goes to that same stream, so ordering already guarantees
|
|
492
|
+
// nobody observes the uninitialized memory. Waiting here would put a host
|
|
493
|
+
// synchronization in front of *every* allocation, which is exactly the cost
|
|
494
|
+
// Tilework.submit exists to remove (an eager op allocates its output).
|
|
495
|
+
if (hipMemsetAsync(buffer, 0, static_cast<size_t>(bytes), g_stream) != hipSuccess) {
|
|
496
|
+
(void)hipFree(buffer);
|
|
497
|
+
return nullptr;
|
|
498
|
+
}
|
|
499
|
+
return buffer;
|
|
500
|
+
}
|
|
501
|
+
void gpu_bridge_buffer_write(void *handle, const void *src, long bytes, long offset) {
|
|
502
|
+
if (ensure_runtime() &&
|
|
503
|
+
hipMemcpyAsync(static_cast<char *>(handle) + offset, src, bytes, hipMemcpyHostToDevice, g_stream) == hipSuccess) {
|
|
504
|
+
// The Ruby ABI releases the source String after this call. Complete the
|
|
505
|
+
// pageable-host copy before returning so it cannot outlive that String.
|
|
506
|
+
(void)hipStreamSynchronize(g_stream);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
void gpu_bridge_buffer_read(void *handle, void *dst, long bytes, long offset) {
|
|
510
|
+
if (ensure_runtime() && hipMemcpyAsync(dst, static_cast<char *>(handle) + offset, bytes, hipMemcpyDeviceToHost, g_stream) == hipSuccess)
|
|
511
|
+
(void)hipStreamSynchronize(g_stream);
|
|
512
|
+
}
|
|
513
|
+
void gpu_bridge_buffer_free(void *handle) { if (handle && ensure_runtime()) (void)hipFree(handle); }
|
|
514
|
+
|
|
515
|
+
int gpu_bridge_host_backing_available(void) { return ensure_runtime() ? 1 : 0; }
|
|
516
|
+
void *gpu_bridge_host_backing_alloc(long bytes) {
|
|
517
|
+
if (bytes <= 0) { g_error = "host backing size must be positive"; return nullptr; }
|
|
518
|
+
auto *backing = static_cast<HostBacking *>(malloc(sizeof(HostBacking)));
|
|
519
|
+
if (!backing) { g_error = "failed to allocate host backing metadata"; return nullptr; }
|
|
520
|
+
backing->data = malloc(static_cast<size_t>(bytes));
|
|
521
|
+
backing->bytes = bytes;
|
|
522
|
+
if (!backing->data) { free(backing); g_error = "failed to allocate pageable host backing"; return nullptr; }
|
|
523
|
+
return backing;
|
|
524
|
+
}
|
|
525
|
+
void gpu_bridge_host_backing_free(void *handle) {
|
|
526
|
+
auto *backing = static_cast<HostBacking *>(handle);
|
|
527
|
+
if (!backing) return;
|
|
528
|
+
free(backing->data);
|
|
529
|
+
backing->data = nullptr;
|
|
530
|
+
free(backing);
|
|
531
|
+
}
|
|
532
|
+
int gpu_bridge_host_backing_store(void *host, void *device, long bytes, long offset, char **error_out) {
|
|
533
|
+
auto *backing = static_cast<HostBacking *>(host);
|
|
534
|
+
if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
|
|
535
|
+
set_error(error_out, "host backing store range is out of bounds"); return 1;
|
|
536
|
+
}
|
|
537
|
+
if (!ensure_runtime()) { set_error(error_out, g_error); return 1; }
|
|
538
|
+
hipError_t status = hipSuccess;
|
|
539
|
+
if (ensure_staging()) {
|
|
540
|
+
std::lock_guard<std::mutex> lock(g_staging_mutex);
|
|
541
|
+
for (long copied = 0; copied < bytes && status == hipSuccess; ) {
|
|
542
|
+
const size_t chunk = std::min(static_cast<size_t>(bytes - copied), HOST_STAGING_BYTES);
|
|
543
|
+
status = hipMemcpyAsync(g_staging, static_cast<char *>(device) + copied, chunk, hipMemcpyDeviceToHost, g_stream);
|
|
544
|
+
if (status == hipSuccess) status = hipStreamSynchronize(g_stream);
|
|
545
|
+
if (status == hipSuccess) memcpy(static_cast<char *>(backing->data) + offset + copied, g_staging, chunk);
|
|
546
|
+
copied += static_cast<long>(chunk);
|
|
547
|
+
}
|
|
548
|
+
} else {
|
|
549
|
+
status = hipMemcpyAsync(static_cast<char *>(backing->data) + offset, device, static_cast<size_t>(bytes), hipMemcpyDeviceToHost, g_stream);
|
|
550
|
+
if (status == hipSuccess) status = hipStreamSynchronize(g_stream);
|
|
551
|
+
}
|
|
552
|
+
if (status != hipSuccess) { set_error(error_out, hip_error("host backing device-to-host copy", status)); return 1; }
|
|
553
|
+
return 0;
|
|
554
|
+
}
|
|
555
|
+
int gpu_bridge_host_backing_restore(void *host, void *device, long bytes, long offset, char **error_out) {
|
|
556
|
+
auto *backing = static_cast<HostBacking *>(host);
|
|
557
|
+
if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
|
|
558
|
+
set_error(error_out, "host backing restore range is out of bounds"); return 1;
|
|
559
|
+
}
|
|
560
|
+
if (!ensure_runtime()) { set_error(error_out, g_error); return 1; }
|
|
561
|
+
hipError_t status = hipSuccess;
|
|
562
|
+
if (ensure_staging()) {
|
|
563
|
+
std::lock_guard<std::mutex> lock(g_staging_mutex);
|
|
564
|
+
for (long copied = 0; copied < bytes && status == hipSuccess; ) {
|
|
565
|
+
const size_t chunk = std::min(static_cast<size_t>(bytes - copied), HOST_STAGING_BYTES);
|
|
566
|
+
memcpy(g_staging, static_cast<char *>(backing->data) + offset + copied, chunk);
|
|
567
|
+
status = hipMemcpyAsync(static_cast<char *>(device) + copied, g_staging, chunk, hipMemcpyHostToDevice, g_stream);
|
|
568
|
+
if (status == hipSuccess) status = hipStreamSynchronize(g_stream);
|
|
569
|
+
copied += static_cast<long>(chunk);
|
|
570
|
+
}
|
|
571
|
+
} else {
|
|
572
|
+
status = hipMemcpyAsync(device, static_cast<char *>(backing->data) + offset, static_cast<size_t>(bytes), hipMemcpyHostToDevice, g_stream);
|
|
573
|
+
if (status == hipSuccess) status = hipStreamSynchronize(g_stream);
|
|
574
|
+
}
|
|
575
|
+
if (status != hipSuccess) { set_error(error_out, hip_error("host backing host-to-device copy", status)); return 1; }
|
|
576
|
+
return 0;
|
|
577
|
+
}
|
|
578
|
+
void *gpu_bridge_host_backing_transfer_begin(void *host, void *device, long bytes, long offset, char **error_out) {
|
|
579
|
+
if (g_live_host_transfers.fetch_add(1) >= MAX_HOST_TRANSFERS) {
|
|
580
|
+
g_live_host_transfers.fetch_sub(1);
|
|
581
|
+
set_error(error_out, "the HIP host-backing transfer limit is reached");
|
|
582
|
+
return nullptr;
|
|
583
|
+
}
|
|
584
|
+
auto *transfer = new HostTransfer;
|
|
585
|
+
transfer->worker = std::thread([transfer, host, device, bytes, offset] {
|
|
586
|
+
char *error = nullptr;
|
|
587
|
+
transfer->status = gpu_bridge_host_backing_restore(host, device, bytes, offset, &error);
|
|
588
|
+
if (error) { transfer->error = error; free(error); }
|
|
589
|
+
transfer->done.store(true, std::memory_order_release);
|
|
590
|
+
});
|
|
591
|
+
return transfer;
|
|
592
|
+
}
|
|
593
|
+
int gpu_bridge_host_backing_transfer_poll(void *handle) {
|
|
594
|
+
auto *transfer = static_cast<HostTransfer *>(handle);
|
|
595
|
+
return transfer && transfer->done.load(std::memory_order_acquire) ? 1 : 0;
|
|
596
|
+
}
|
|
597
|
+
int gpu_bridge_host_backing_transfer_end(void *handle, char **error_out) {
|
|
598
|
+
auto *transfer = static_cast<HostTransfer *>(handle);
|
|
599
|
+
if (!transfer) { set_error(error_out, "invalid HIP host-backing transfer token"); return 1; }
|
|
600
|
+
if (transfer->worker.joinable()) transfer->worker.join();
|
|
601
|
+
const int status = transfer->status;
|
|
602
|
+
if (status != 0) set_error(error_out, transfer->error.empty() ? "HIP host-backing transfer failed" : transfer->error);
|
|
603
|
+
delete transfer;
|
|
604
|
+
g_live_host_transfers.fetch_sub(1);
|
|
605
|
+
return status;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
void *gpu_bridge_pipeline_create(const char *source, const char *name, char **error_out) {
|
|
609
|
+
if (!ensure_runtime()) { set_error(error_out, g_error); return nullptr; }
|
|
610
|
+
hiprtcProgram program{};
|
|
611
|
+
hiprtcResult rc = hiprtcCreateProgram(&program, source, "tilework.hip", 0, nullptr, nullptr);
|
|
612
|
+
if (rc != HIPRTC_SUCCESS) { set_error(error_out, "hiprtcCreateProgram failed"); return nullptr; }
|
|
613
|
+
std::string arch = std::string("--gpu-architecture=") + g_props.gcnArchName;
|
|
614
|
+
std::string include = std::string("-I") + rocm_include_dir();
|
|
615
|
+
const char *options[] = { "--std=c++17", arch.c_str(), include.c_str() };
|
|
616
|
+
rc = hiprtcCompileProgram(program, 3, options);
|
|
617
|
+
size_t log_size = 0; hiprtcGetProgramLogSize(program, &log_size);
|
|
618
|
+
std::string log(log_size, '\0'); if (log_size) hiprtcGetProgramLog(program, log.data());
|
|
619
|
+
if (rc != HIPRTC_SUCCESS) { hiprtcDestroyProgram(&program); set_error(error_out, "HIPRTC compilation failed for " + arch + ":\n" + log); return nullptr; }
|
|
620
|
+
size_t code_size = 0;
|
|
621
|
+
if (hiprtcGetCodeSize(program, &code_size) != HIPRTC_SUCCESS) { hiprtcDestroyProgram(&program); set_error(error_out, "hiprtcGetCodeSize failed"); return nullptr; }
|
|
622
|
+
std::vector<char> code(code_size);
|
|
623
|
+
if (hiprtcGetCode(program, code.data()) != HIPRTC_SUCCESS) { hiprtcDestroyProgram(&program); set_error(error_out, "hiprtcGetCode failed"); return nullptr; }
|
|
624
|
+
hiprtcDestroyProgram(&program);
|
|
625
|
+
Pipeline *pipeline = new Pipeline{};
|
|
626
|
+
hipError_t status = hipModuleLoadData(&pipeline->module, code.data());
|
|
627
|
+
if (status == hipSuccess) status = hipModuleGetFunction(&pipeline->function, pipeline->module, name);
|
|
628
|
+
if (status != hipSuccess) { if (pipeline->module) (void)hipModuleUnload(pipeline->module); delete pipeline; set_error(error_out, hip_error("HIP module/function load", status)); return nullptr; }
|
|
629
|
+
return pipeline;
|
|
630
|
+
}
|
|
631
|
+
long gpu_bridge_pipeline_max_threads(void *handle) {
|
|
632
|
+
// HIP's module API does not expose CUDA's per-function max-thread query on
|
|
633
|
+
// all supported ROCm versions. The device limit remains a safe upper bound;
|
|
634
|
+
// HIP validates any stricter kernel-specific limit at launch.
|
|
635
|
+
(void)handle;
|
|
636
|
+
return g_props.maxThreadsPerBlock;
|
|
637
|
+
}
|
|
638
|
+
void gpu_bridge_pipeline_free(void *handle) { auto *p = static_cast<Pipeline *>(handle); if (p) { (void)hipModuleUnload(p->module); delete p; } }
|
|
639
|
+
|
|
640
|
+
int gpu_bridge_dispatch(void *pipeline, const int32_t *kinds, void *const *ptrs, const long *lens, int32_t count,
|
|
641
|
+
long gx, long gy, long gz, long bx, long by, long bz, int32_t wait, char **error_out) {
|
|
642
|
+
if (!ensure_runtime()) { set_error(error_out, g_error); return 1; }
|
|
643
|
+
std::vector<void *> values(count), params(count);
|
|
644
|
+
for (int i = 0; i < count; ++i) {
|
|
645
|
+
if (kinds[i] == 0) { values[i] = static_cast<char *>(ptrs[i]) + lens[i]; params[i] = &values[i]; }
|
|
646
|
+
else params[i] = ptrs[i];
|
|
647
|
+
}
|
|
648
|
+
hipError_t status = hipModuleLaunchKernel(static_cast<Pipeline *>(pipeline)->function,
|
|
649
|
+
gx, gy, gz, bx, by, bz, 0, g_stream, params.data(), nullptr);
|
|
650
|
+
if (status == hipSuccess && wait) status = hipStreamSynchronize(g_stream);
|
|
651
|
+
if (status != hipSuccess) { set_error(error_out, hip_error("hipModuleLaunchKernel", status)); return 1; }
|
|
652
|
+
return 0;
|
|
653
|
+
}
|
|
654
|
+
void gpu_bridge_synchronize(void) { if (ensure_runtime()) (void)hipStreamSynchronize(g_stream); }
|
|
655
|
+
void gpu_bridge_shutdown(void) {
|
|
656
|
+
std::lock_guard<std::mutex> lock(g_mutex);
|
|
657
|
+
g_shutting_down = true;
|
|
658
|
+
// All lazy library handles and plans are bound to this stream. Drain it
|
|
659
|
+
// before destroying any of them; hipFFT in particular can otherwise retain
|
|
660
|
+
// work that touches plan-owned state during Ruby's end-proc sequence.
|
|
661
|
+
if (g_stream) (void)hipStreamSynchronize(g_stream);
|
|
662
|
+
{
|
|
663
|
+
std::lock_guard<std::mutex> lock(g_staging_mutex);
|
|
664
|
+
if (g_staging) {
|
|
665
|
+
(void)hipHostFree(g_staging);
|
|
666
|
+
g_staging = nullptr;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
#ifdef TILEWORK_HAS_MIOPEN
|
|
670
|
+
// Descriptor/workspace ownership belongs to MIOpen and must be released
|
|
671
|
+
// while its handle and the HIP stream are still valid.
|
|
672
|
+
for (auto &[_, plan] : g_miopen_plans) miopen_plan_free(plan);
|
|
673
|
+
g_miopen_plans.clear();
|
|
674
|
+
if (g_miopen) {
|
|
675
|
+
(void)miopenDestroy(g_miopen);
|
|
676
|
+
g_miopen = nullptr;
|
|
677
|
+
}
|
|
678
|
+
#endif
|
|
679
|
+
#ifdef TILEWORK_HAS_HIPFFT
|
|
680
|
+
// hipFFT must be released before the remaining ROCm library handles on this
|
|
681
|
+
// WSL2 stack. MIOpen owns descriptor workspaces, so it is drained first.
|
|
682
|
+
for (auto &[_, plan] : g_fft_plans) (void)hipfftDestroy(plan);
|
|
683
|
+
g_fft_plans.clear();
|
|
684
|
+
#endif
|
|
685
|
+
#ifdef TILEWORK_HAS_HIPBLAS
|
|
686
|
+
if (g_blas) {
|
|
687
|
+
(void)hipblasDestroy(g_blas);
|
|
688
|
+
g_blas = nullptr;
|
|
689
|
+
}
|
|
690
|
+
#endif
|
|
691
|
+
#ifdef TILEWORK_HAS_HIPRAND
|
|
692
|
+
if (g_rng) {
|
|
693
|
+
(void)hiprandDestroyGenerator(g_rng);
|
|
694
|
+
g_rng = nullptr;
|
|
695
|
+
}
|
|
696
|
+
#endif
|
|
697
|
+
if (g_stream) {
|
|
698
|
+
(void)hipStreamDestroy(g_stream);
|
|
699
|
+
g_stream = nullptr;
|
|
700
|
+
}
|
|
701
|
+
{
|
|
702
|
+
std::lock_guard<std::mutex> matrix_lock(g_matrix_mutex);
|
|
703
|
+
if (g_matrix_pipeline) {
|
|
704
|
+
gpu_bridge_pipeline_free(g_matrix_pipeline);
|
|
705
|
+
g_matrix_pipeline = nullptr;
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
// MIOpen keeps process-global ROCm state whose own DSO teardown runs after
|
|
709
|
+
// Ruby end-procs. Resetting the device here races that teardown on ROCm
|
|
710
|
+
// 7.2/WSL2; all extension-owned resources above are already released.
|
|
711
|
+
// Leave final device destruction to the HIP runtime's process shutdown.
|
|
712
|
+
g_device = -1;
|
|
713
|
+
}
|
|
714
|
+
int gpu_bridge_device_capabilities(int32_t *warp, int32_t *major, int32_t *minor, int32_t *integrated, long *budget) {
|
|
715
|
+
if (!ensure_runtime()) return 1;
|
|
716
|
+
size_t free_bytes = 0, total = 0;
|
|
717
|
+
if (hipMemGetInfo(&free_bytes, &total) != hipSuccess) return 1;
|
|
718
|
+
*warp = g_props.warpSize; *major = 0; *minor = 0; *integrated = g_props.integrated; *budget = static_cast<long>(free_bytes);
|
|
719
|
+
return 0;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
static const char *matrix_core_source() {
|
|
723
|
+
return R"HIP(
|
|
724
|
+
#include <rocwmma/rocwmma.hpp>
|
|
725
|
+
using namespace rocwmma;
|
|
726
|
+
extern "C" __global__ void tilework_rocwmma_gemm(const float16_t* a, const float16_t* b,
|
|
727
|
+
float* c, int m, int k, int n) {
|
|
728
|
+
const int row = blockIdx.y * 16;
|
|
729
|
+
const int col = blockIdx.x * 16;
|
|
730
|
+
using FragA = fragment<matrix_a, 16, 16, 16, float16_t, row_major>;
|
|
731
|
+
using FragB = fragment<matrix_b, 16, 16, 16, float16_t, row_major>;
|
|
732
|
+
using Accum = fragment<accumulator, 16, 16, 16, float, row_major>;
|
|
733
|
+
Accum acc;
|
|
734
|
+
fill_fragment(acc, 0.0f);
|
|
735
|
+
for (int kk = 0; kk < k; kk += 16) {
|
|
736
|
+
FragA fa;
|
|
737
|
+
FragB fb;
|
|
738
|
+
load_matrix_sync(fa, a + row * k + kk, k);
|
|
739
|
+
load_matrix_sync(fb, b + kk * n + col, n);
|
|
740
|
+
mma_sync(acc, fa, fb, acc);
|
|
741
|
+
}
|
|
742
|
+
store_matrix_sync(c + row * n + col, acc, n);
|
|
743
|
+
}
|
|
744
|
+
)HIP";
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
bool ensure_matrix_pipeline(char **error_out) {
|
|
748
|
+
std::lock_guard<std::mutex> lock(g_matrix_mutex);
|
|
749
|
+
if (g_matrix_pipeline) return true;
|
|
750
|
+
void *pipeline = gpu_bridge_pipeline_create(matrix_core_source(), "tilework_rocwmma_gemm", error_out);
|
|
751
|
+
if (!pipeline) return false;
|
|
752
|
+
g_matrix_pipeline = static_cast<Pipeline *>(pipeline);
|
|
753
|
+
return true;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
int gpu_bridge_matrix_core_gemm(void *a, long a_off, void *b, long b_off,
|
|
757
|
+
void *c, long c_off, long m, long k, long n,
|
|
758
|
+
char **error_out) {
|
|
759
|
+
if (m <= 0 || k <= 0 || n <= 0 || m % 16 || k % 16 || n % 16) {
|
|
760
|
+
set_error(error_out, "rocWMMA GEMM requires positive m, k, and n that are multiples of 16");
|
|
761
|
+
return 1;
|
|
762
|
+
}
|
|
763
|
+
if (!ensure_runtime()) { set_error(error_out, g_error); return 1; }
|
|
764
|
+
if (!ensure_matrix_pipeline(error_out)) return 1;
|
|
765
|
+
int32_t kinds[] = {0, 0, 0, 1, 1, 1};
|
|
766
|
+
int32_t dims[] = {static_cast<int32_t>(m), static_cast<int32_t>(k), static_cast<int32_t>(n)};
|
|
767
|
+
void *args[] = {a, b, c, &dims[0], &dims[1], &dims[2]};
|
|
768
|
+
long lens[] = {a_off, b_off, c_off, static_cast<long>(sizeof(int32_t)),
|
|
769
|
+
static_cast<long>(sizeof(int32_t)), static_cast<long>(sizeof(int32_t))};
|
|
770
|
+
return gpu_bridge_dispatch(g_matrix_pipeline, kinds, args, lens, 6,
|
|
771
|
+
n / 16, m / 16, 1, 32, 1, 1, 1, error_out);
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
int gpu_bridge_matrix_core_available(void) {
|
|
775
|
+
{
|
|
776
|
+
std::lock_guard<std::mutex> lock(g_matrix_mutex);
|
|
777
|
+
if (g_matrix_probed) return g_matrix_available ? 1 : 0;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
char *compile_error = nullptr;
|
|
781
|
+
bool ok = ensure_matrix_pipeline(&compile_error);
|
|
782
|
+
if (!ok) {
|
|
783
|
+
g_error = compile_error ? compile_error : "rocWMMA pipeline compilation failed";
|
|
784
|
+
if (compile_error) free(compile_error);
|
|
785
|
+
} else {
|
|
786
|
+
std::vector<uint16_t> input(16 * 16, 0x3c00); // float16 1.0
|
|
787
|
+
std::vector<float> output(16 * 16, 0.0f);
|
|
788
|
+
void *a = nullptr, *b = nullptr, *c = nullptr;
|
|
789
|
+
hipError_t status = hipMalloc(&a, input.size() * sizeof(uint16_t));
|
|
790
|
+
if (status == hipSuccess) status = hipMalloc(&b, input.size() * sizeof(uint16_t));
|
|
791
|
+
if (status == hipSuccess) status = hipMalloc(&c, output.size() * sizeof(float));
|
|
792
|
+
if (status == hipSuccess) status = hipMemcpyAsync(a, input.data(), input.size() * sizeof(uint16_t), hipMemcpyHostToDevice, g_stream);
|
|
793
|
+
if (status == hipSuccess) status = hipMemcpyAsync(b, input.data(), input.size() * sizeof(uint16_t), hipMemcpyHostToDevice, g_stream);
|
|
794
|
+
char *dispatch_error = nullptr;
|
|
795
|
+
if (status == hipSuccess)
|
|
796
|
+
status = gpu_bridge_matrix_core_gemm(a, 0, b, 0, c, 0, 16, 16, 16, &dispatch_error) == 0 ? hipSuccess : hipErrorUnknown;
|
|
797
|
+
if (status == hipSuccess) status = hipMemcpyAsync(output.data(), c, output.size() * sizeof(float), hipMemcpyDeviceToHost, g_stream);
|
|
798
|
+
if (status == hipSuccess) status = hipStreamSynchronize(g_stream);
|
|
799
|
+
if (a) (void)hipFree(a);
|
|
800
|
+
if (b) (void)hipFree(b);
|
|
801
|
+
if (c) (void)hipFree(c);
|
|
802
|
+
if (status != hipSuccess) {
|
|
803
|
+
g_error = dispatch_error ? dispatch_error : hip_error("rocWMMA execution probe", status);
|
|
804
|
+
if (dispatch_error) free(dispatch_error);
|
|
805
|
+
} else {
|
|
806
|
+
ok = std::all_of(output.begin(), output.end(), [](float x) { return x == 16.0f; });
|
|
807
|
+
if (!ok) g_error = "rocWMMA execution probe returned an incorrect result";
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
std::lock_guard<std::mutex> lock(g_matrix_mutex);
|
|
811
|
+
g_matrix_probed = true;
|
|
812
|
+
g_matrix_available = ok;
|
|
813
|
+
return ok ? 1 : 0;
|
|
814
|
+
}
|
|
815
|
+
#ifdef TILEWORK_HAS_HIPBLAS
|
|
816
|
+
int gpu_bridge_gemm_available(void) { return run_blas_probe() ? 1 : 0; }
|
|
817
|
+
int gpu_bridge_gemm(void *a, long a_off, void *b, long b_off, void *c, long c_off,
|
|
818
|
+
long m, long k, long n, int /*allow_tf32*/, int in_dtype, char **error_out) {
|
|
819
|
+
if (!run_blas_probe()) { set_error(error_out, g_error); return 1; }
|
|
820
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
821
|
+
void *ap = static_cast<char *>(a) + a_off;
|
|
822
|
+
void *bp = static_cast<char *>(b) + b_off;
|
|
823
|
+
void *cp = static_cast<char *>(c) + c_off;
|
|
824
|
+
hipblasStatus_t status;
|
|
825
|
+
// Tilework buffers are row-major. C^T = B^T * A^T lets hipBLAS consume
|
|
826
|
+
// those same bytes as column-major matrices without a transpose kernel.
|
|
827
|
+
if (in_dtype == 0) {
|
|
828
|
+
status = hipblasSgemm(g_blas, HIPBLAS_OP_N, HIPBLAS_OP_N,
|
|
829
|
+
static_cast<int>(n), static_cast<int>(m), static_cast<int>(k),
|
|
830
|
+
&alpha, static_cast<const float *>(bp), static_cast<int>(n),
|
|
831
|
+
static_cast<const float *>(ap), static_cast<int>(k),
|
|
832
|
+
&beta, static_cast<float *>(cp), static_cast<int>(n));
|
|
833
|
+
} else if (in_dtype == 1) {
|
|
834
|
+
status = hipblasGemmEx(g_blas, HIPBLAS_OP_N, HIPBLAS_OP_N,
|
|
835
|
+
static_cast<int>(n), static_cast<int>(m), static_cast<int>(k),
|
|
836
|
+
&alpha, bp, HIP_R_16F, static_cast<int>(n),
|
|
837
|
+
ap, HIP_R_16F, static_cast<int>(k),
|
|
838
|
+
&beta, cp, HIP_R_32F, static_cast<int>(n),
|
|
839
|
+
HIPBLAS_COMPUTE_32F, HIPBLAS_GEMM_DEFAULT);
|
|
840
|
+
} else if (in_dtype == 2) {
|
|
841
|
+
status = hipblasGemmEx(g_blas, HIPBLAS_OP_N, HIPBLAS_OP_N,
|
|
842
|
+
static_cast<int>(n), static_cast<int>(m), static_cast<int>(k),
|
|
843
|
+
&alpha, bp, HIP_R_16BF, static_cast<int>(n),
|
|
844
|
+
ap, HIP_R_16BF, static_cast<int>(k),
|
|
845
|
+
&beta, cp, HIP_R_32F, static_cast<int>(n),
|
|
846
|
+
HIPBLAS_COMPUTE_32F, HIPBLAS_GEMM_DEFAULT);
|
|
847
|
+
} else {
|
|
848
|
+
set_error(error_out, "hipBLAS GEMM received an unsupported input dtype");
|
|
849
|
+
return 1;
|
|
850
|
+
}
|
|
851
|
+
if (status != HIPBLAS_STATUS_SUCCESS) {
|
|
852
|
+
set_error(error_out, hipblas_error("hipBLAS GEMM", status));
|
|
853
|
+
return 1;
|
|
854
|
+
}
|
|
855
|
+
return 0;
|
|
856
|
+
}
|
|
857
|
+
#endif
|
|
858
|
+
#ifdef TILEWORK_HAS_HIPRAND
|
|
859
|
+
int gpu_bridge_random_available(void) { return run_rng_probe() ? 1 : 0; }
|
|
860
|
+
int gpu_bridge_random(void *handle, long n, int kind, unsigned long long seed,
|
|
861
|
+
float mean, float stddev, char **error_out) {
|
|
862
|
+
if (n <= 0) { set_error(error_out, "hipRAND draw length must be positive"); return 1; }
|
|
863
|
+
if (!run_rng_probe()) { set_error(error_out, g_error); return 1; }
|
|
864
|
+
hiprandStatus_t status = hiprandSetPseudoRandomGeneratorSeed(g_rng, seed);
|
|
865
|
+
if (status == HIPRAND_STATUS_SUCCESS) status = hiprandSetGeneratorOffset(g_rng, 0);
|
|
866
|
+
if (status != HIPRAND_STATUS_SUCCESS) { set_error(error_out, hiprand_error("hipRAND seed setup", status)); return 1; }
|
|
867
|
+
if (kind == 0) {
|
|
868
|
+
status = hiprandGenerateUniform(g_rng, static_cast<float *>(handle), static_cast<size_t>(n));
|
|
869
|
+
} else if (kind == 1) {
|
|
870
|
+
if ((n & 1) == 0) {
|
|
871
|
+
status = hiprandGenerateNormal(g_rng, static_cast<float *>(handle), static_cast<size_t>(n), mean, stddev);
|
|
872
|
+
} else {
|
|
873
|
+
float *scratch = nullptr;
|
|
874
|
+
hipError_t hip_status = hipMalloc(&scratch, static_cast<size_t>(n + 1) * sizeof(float));
|
|
875
|
+
if (hip_status != hipSuccess) { set_error(error_out, hip_error("hipRAND normal scratch allocation", hip_status)); return 1; }
|
|
876
|
+
status = hiprandGenerateNormal(g_rng, scratch, static_cast<size_t>(n + 1), mean, stddev);
|
|
877
|
+
if (status == HIPRAND_STATUS_SUCCESS)
|
|
878
|
+
hip_status = hipMemcpyAsync(handle, scratch, static_cast<size_t>(n) * sizeof(float), hipMemcpyDeviceToDevice, g_stream);
|
|
879
|
+
(void)hipFree(scratch);
|
|
880
|
+
if (hip_status != hipSuccess) { set_error(error_out, hip_error("hipRAND normal scratch copy", hip_status)); return 1; }
|
|
881
|
+
}
|
|
882
|
+
} else {
|
|
883
|
+
set_error(error_out, "hipRAND received an unknown distribution kind");
|
|
884
|
+
return 1;
|
|
885
|
+
}
|
|
886
|
+
if (status != HIPRAND_STATUS_SUCCESS) { set_error(error_out, hiprand_error("hipRAND generation", status)); return 1; }
|
|
887
|
+
return 0;
|
|
888
|
+
}
|
|
889
|
+
#endif
|
|
890
|
+
#ifdef TILEWORK_HAS_HIPFFT
|
|
891
|
+
int gpu_bridge_fft(void *in, void *out, long n, long batch, int kind, int direction, char **error_out) {
|
|
892
|
+
if (n <= 0 || batch <= 0 || kind < 0 || kind > 2) { set_error(error_out, "hipFFT received invalid transform dimensions or kind"); return 1; }
|
|
893
|
+
hipfftHandle plan{};
|
|
894
|
+
if (!fft_plan(kind, n, batch, &plan)) { set_error(error_out, g_error); return 1; }
|
|
895
|
+
hipfftResult status;
|
|
896
|
+
if (kind == 0) status = hipfftExecR2C(plan, static_cast<hipfftReal *>(in), static_cast<hipfftComplex *>(out));
|
|
897
|
+
else if (kind == 1) status = hipfftExecC2R(plan, static_cast<hipfftComplex *>(in), static_cast<hipfftReal *>(out));
|
|
898
|
+
else status = hipfftExecC2C(plan, static_cast<hipfftComplex *>(in), static_cast<hipfftComplex *>(out),
|
|
899
|
+
direction < 0 ? HIPFFT_BACKWARD : HIPFFT_FORWARD);
|
|
900
|
+
if (status != HIPFFT_SUCCESS) { set_error(error_out, hipfft_error("hipFFT execution", status)); return 1; }
|
|
901
|
+
return 0;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
int gpu_bridge_fft_available(void) {
|
|
905
|
+
if (g_fft_probed) return g_fft_available ? 1 : 0;
|
|
906
|
+
g_fft_probed = true;
|
|
907
|
+
float input[4] = {1, 0, 0, 0};
|
|
908
|
+
float output[6] = {};
|
|
909
|
+
void *in = nullptr, *out = nullptr;
|
|
910
|
+
hipError_t hip_status = hipMalloc(&in, sizeof(input));
|
|
911
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&out, sizeof(output));
|
|
912
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(in, input, sizeof(input), hipMemcpyHostToDevice, g_stream);
|
|
913
|
+
char *fft_error_text = nullptr;
|
|
914
|
+
if (hip_status == hipSuccess) hip_status = gpu_bridge_fft(in, out, 4, 1, 0, 1, &fft_error_text) == 0 ? hipSuccess : hipErrorUnknown;
|
|
915
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(output, out, sizeof(output), hipMemcpyDeviceToHost, g_stream);
|
|
916
|
+
if (hip_status == hipSuccess) hip_status = hipStreamSynchronize(g_stream);
|
|
917
|
+
if (in) (void)hipFree(in);
|
|
918
|
+
if (out) (void)hipFree(out);
|
|
919
|
+
if (hip_status != hipSuccess) {
|
|
920
|
+
g_error = fft_error_text ? fft_error_text : hip_error("hipFFT execution probe", hip_status);
|
|
921
|
+
if (fft_error_text) free(fft_error_text);
|
|
922
|
+
return 0;
|
|
923
|
+
}
|
|
924
|
+
g_fft_available = output[0] == 1.0f && output[1] == 0.0f &&
|
|
925
|
+
output[2] == 1.0f && output[3] == 0.0f &&
|
|
926
|
+
output[4] == 1.0f && output[5] == 0.0f;
|
|
927
|
+
if (!g_fft_available) g_error = "hipFFT execution probe returned an incorrect spectrum";
|
|
928
|
+
return g_fft_available ? 1 : 0;
|
|
929
|
+
}
|
|
930
|
+
#endif
|
|
931
|
+
#ifdef TILEWORK_HAS_MIOPEN
|
|
932
|
+
int gpu_bridge_conv2d(void *x, void *w, void *y,
|
|
933
|
+
int n, int cin, int h, int width, int cout, int kh, int kw,
|
|
934
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
935
|
+
int dilation_h, int dilation_w, int groups, int dtype, char **error_out) {
|
|
936
|
+
if (n <= 0 || cin <= 0 || h <= 0 || width <= 0 || cout <= 0 || kh <= 0 || kw <= 0 ||
|
|
937
|
+
pad_h < 0 || pad_w < 0 || stride_h <= 0 || stride_w <= 0 || dilation_h <= 0 || dilation_w <= 0 ||
|
|
938
|
+
groups <= 0 || cin % groups != 0 || cout % groups != 0 || (dtype != 0 && dtype != 1 && dtype != 2)) {
|
|
939
|
+
set_error(error_out, "MIOpen convolution received invalid dimensions or dtype");
|
|
940
|
+
return 1;
|
|
941
|
+
}
|
|
942
|
+
MIOpenPlan *plan = nullptr;
|
|
943
|
+
if (!miopen_plan(&plan, x, w, y, n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
944
|
+
dilation_h, dilation_w, groups, dtype)) {
|
|
945
|
+
set_error(error_out, g_error);
|
|
946
|
+
return 1;
|
|
947
|
+
}
|
|
948
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
949
|
+
miopenStatus_t status = miopenConvolutionForward(g_miopen, &alpha, plan->x, x, plan->w, w, plan->conv,
|
|
950
|
+
plan->algorithm, &beta, plan->y, y,
|
|
951
|
+
plan->workspace, plan->workspace_bytes);
|
|
952
|
+
if (status != miopenStatusSuccess) {
|
|
953
|
+
set_error(error_out, miopen_error("MIOpen convolution forward", status));
|
|
954
|
+
return 1;
|
|
955
|
+
}
|
|
956
|
+
return 0;
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
int gpu_bridge_conv2d_backward(void *a, void *b, void *out,
|
|
960
|
+
int n, int cin, int h, int width, int cout, int kh, int kw,
|
|
961
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
962
|
+
int dilation_h, int dilation_w, int groups, int dtype, int mode, char **error_out) {
|
|
963
|
+
if (!ensure_miopen() || (mode != 0 && mode != 1)) {
|
|
964
|
+
set_error(error_out, mode != 0 && mode != 1 ? "MIOpen convolution backward received an invalid mode" : g_error);
|
|
965
|
+
return 1;
|
|
966
|
+
}
|
|
967
|
+
MIOpenPlan *plan = miopen_existing_plan(n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
968
|
+
dilation_h, dilation_w, groups, dtype);
|
|
969
|
+
if (!plan) { set_error(error_out, g_error); return 1; }
|
|
970
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
971
|
+
miopenStatus_t status;
|
|
972
|
+
if (mode == 0) {
|
|
973
|
+
if (!miopen_prepare_backward_data(plan, b, a, out)) { set_error(error_out, g_error); return 1; }
|
|
974
|
+
status = miopenConvolutionBackwardData(g_miopen, &alpha, plan->y, b, plan->w, a, plan->conv,
|
|
975
|
+
plan->backward_data_algorithm, &beta, plan->x, out,
|
|
976
|
+
plan->backward_data_workspace, plan->backward_data_workspace_bytes);
|
|
977
|
+
} else {
|
|
978
|
+
if (!miopen_prepare_backward_weights(plan, a, b, out)) { set_error(error_out, g_error); return 1; }
|
|
979
|
+
status = miopenConvolutionBackwardWeights(g_miopen, &alpha, plan->y, b, plan->x, a, plan->conv,
|
|
980
|
+
plan->backward_weights_algorithm, &beta, plan->w, out,
|
|
981
|
+
plan->backward_weights_workspace, plan->backward_weights_workspace_bytes);
|
|
982
|
+
}
|
|
983
|
+
if (status != miopenStatusSuccess) {
|
|
984
|
+
set_error(error_out, miopen_error(mode == 0 ? "MIOpen convolution backward-data" : "MIOpen convolution backward-filter", status));
|
|
985
|
+
return 1;
|
|
986
|
+
}
|
|
987
|
+
return 0;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
int gpu_bridge_conv2d_available(void) {
|
|
991
|
+
if (g_miopen_probed) return g_miopen_available ? 1 : 0;
|
|
992
|
+
g_miopen_probed = true;
|
|
993
|
+
if (!ensure_miopen()) return 0;
|
|
994
|
+
const float input[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9};
|
|
995
|
+
const float weight[1] = {2};
|
|
996
|
+
float output[9] = {};
|
|
997
|
+
void *x = nullptr, *w = nullptr, *y = nullptr;
|
|
998
|
+
hipError_t hip_status = hipMalloc(&x, sizeof(input));
|
|
999
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&w, sizeof(weight));
|
|
1000
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&y, sizeof(output));
|
|
1001
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(x, input, sizeof(input), hipMemcpyHostToDevice, g_stream);
|
|
1002
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(w, weight, sizeof(weight), hipMemcpyHostToDevice, g_stream);
|
|
1003
|
+
char *conv_error = nullptr;
|
|
1004
|
+
if (hip_status == hipSuccess)
|
|
1005
|
+
hip_status = gpu_bridge_conv2d(x, w, y, 1, 1, 3, 3, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, &conv_error) == 0
|
|
1006
|
+
? hipSuccess : hipErrorUnknown;
|
|
1007
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(output, y, sizeof(output), hipMemcpyDeviceToHost, g_stream);
|
|
1008
|
+
if (hip_status == hipSuccess) hip_status = hipStreamSynchronize(g_stream);
|
|
1009
|
+
if (x) (void)hipFree(x);
|
|
1010
|
+
if (w) (void)hipFree(w);
|
|
1011
|
+
if (y) (void)hipFree(y);
|
|
1012
|
+
if (hip_status != hipSuccess) {
|
|
1013
|
+
g_error = conv_error ? conv_error : hip_error("MIOpen execution probe", hip_status);
|
|
1014
|
+
if (conv_error) free(conv_error);
|
|
1015
|
+
return 0;
|
|
1016
|
+
}
|
|
1017
|
+
g_miopen_available = std::equal(std::begin(output), std::end(output), std::begin(input),
|
|
1018
|
+
[](float got, float original) { return got == original * 2.0f; });
|
|
1019
|
+
if (!g_miopen_available) g_error = "MIOpen execution probe returned an incorrect result";
|
|
1020
|
+
return g_miopen_available ? 1 : 0;
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
int gpu_bridge_conv2d_backward_available(void) {
|
|
1024
|
+
if (g_miopen_backward_probed) return g_miopen_backward_available ? 1 : 0;
|
|
1025
|
+
g_miopen_backward_probed = true;
|
|
1026
|
+
if (!gpu_bridge_conv2d_available()) return 0;
|
|
1027
|
+
const float input[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9};
|
|
1028
|
+
const float weight[1] = {2};
|
|
1029
|
+
const float gradient[9] = {1, 1, 1, 1, 1, 1, 1, 1, 1};
|
|
1030
|
+
float forward[9] = {}, dx[9] = {}, dw[1] = {};
|
|
1031
|
+
void *x = nullptr, *w = nullptr, *y = nullptr, *dy = nullptr, *device_dx = nullptr, *device_dw = nullptr;
|
|
1032
|
+
hipError_t hip_status = hipMalloc(&x, sizeof(input));
|
|
1033
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&w, sizeof(weight));
|
|
1034
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&y, sizeof(forward));
|
|
1035
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&dy, sizeof(gradient));
|
|
1036
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&device_dx, sizeof(dx));
|
|
1037
|
+
if (hip_status == hipSuccess) hip_status = hipMalloc(&device_dw, sizeof(dw));
|
|
1038
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(x, input, sizeof(input), hipMemcpyHostToDevice, g_stream);
|
|
1039
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(w, weight, sizeof(weight), hipMemcpyHostToDevice, g_stream);
|
|
1040
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(dy, gradient, sizeof(gradient), hipMemcpyHostToDevice, g_stream);
|
|
1041
|
+
char *error = nullptr;
|
|
1042
|
+
if (hip_status == hipSuccess)
|
|
1043
|
+
hip_status = gpu_bridge_conv2d(x, w, y, 1, 1, 3, 3, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, &error) == 0 ? hipSuccess : hipErrorUnknown;
|
|
1044
|
+
if (hip_status == hipSuccess)
|
|
1045
|
+
hip_status = gpu_bridge_conv2d_backward(w, dy, device_dx, 1, 1, 3, 3, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, &error) == 0 ? hipSuccess : hipErrorUnknown;
|
|
1046
|
+
if (hip_status == hipSuccess)
|
|
1047
|
+
hip_status = gpu_bridge_conv2d_backward(x, dy, device_dw, 1, 1, 3, 3, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, &error) == 0 ? hipSuccess : hipErrorUnknown;
|
|
1048
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(dx, device_dx, sizeof(dx), hipMemcpyDeviceToHost, g_stream);
|
|
1049
|
+
if (hip_status == hipSuccess) hip_status = hipMemcpyAsync(dw, device_dw, sizeof(dw), hipMemcpyDeviceToHost, g_stream);
|
|
1050
|
+
if (hip_status == hipSuccess) hip_status = hipStreamSynchronize(g_stream);
|
|
1051
|
+
if (x) (void)hipFree(x);
|
|
1052
|
+
if (w) (void)hipFree(w);
|
|
1053
|
+
if (y) (void)hipFree(y);
|
|
1054
|
+
if (dy) (void)hipFree(dy);
|
|
1055
|
+
if (device_dx) (void)hipFree(device_dx);
|
|
1056
|
+
if (device_dw) (void)hipFree(device_dw);
|
|
1057
|
+
if (hip_status != hipSuccess) {
|
|
1058
|
+
g_error = error ? error : hip_error("MIOpen backward execution probe", hip_status);
|
|
1059
|
+
if (error) free(error);
|
|
1060
|
+
return 0;
|
|
1061
|
+
}
|
|
1062
|
+
g_miopen_backward_available = std::all_of(std::begin(dx), std::end(dx), [](float value) { return value == 2.0f; }) && dw[0] == 45.0f;
|
|
1063
|
+
if (!g_miopen_backward_available) g_error = "MIOpen backward execution probe returned an incorrect result";
|
|
1064
|
+
return g_miopen_backward_available ? 1 : 0;
|
|
1065
|
+
}
|
|
1066
|
+
#endif
|
|
1067
|
+
}
|