executorch 0.1.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1131 @@
1
+ #include "module_core.hpp"
2
+ #include "tensor_core.hpp"
3
+
4
+ #include <algorithm>
5
+ #include <cstdlib>
6
+ #include <cstring>
7
+ #include <deque>
8
+ #include <limits>
9
+ #include <new>
10
+ #include <sys/stat.h>
11
+
12
+ #include <executorch/extension/data_loader/buffer_data_loader.h>
13
+ #include <executorch/runtime/backend/interface.h>
14
+ #include <executorch/runtime/core/error.h>
15
+ #include <executorch/runtime/kernel/operator_registry.h>
16
+ #include <executorch/runtime/kernel/thread_parallel_interface.h>
17
+ #include <executorch/runtime/platform/platform.h>
18
+ #include <executorch/runtime/platform/runtime.h>
19
+
20
+ #ifdef ET_USE_THREADPOOL
21
+ namespace executorch::extension::threadpool {
22
+ class ThreadPool {
23
+ public:
24
+ void _unsafe_reset_threadpool(uint32_t num_threads);
25
+ size_t get_thread_count() const;
26
+ };
27
+ ThreadPool* get_threadpool();
28
+ } // namespace executorch::extension::threadpool
29
+ #endif
30
+
31
+ namespace etrb {
32
+
33
+ namespace {
34
+
35
+ thread_local std::string* g_pal_log_target = nullptr;
36
+ std::once_flag g_runtime_init_flag;
37
+ std::mutex g_threadpool_mu;
38
+ pal_emit_log_message_method g_default_pal_emit = nullptr;
39
+
40
+ void etrb_pal_emit_log(
41
+ et_timestamp_t timestamp,
42
+ et_pal_log_level_t level,
43
+ const char* filename,
44
+ const char* function,
45
+ size_t line,
46
+ const char* message,
47
+ size_t length) {
48
+ if (g_pal_log_target != nullptr &&
49
+ (level == kError || level == kFatal)) {
50
+ try {
51
+ if (!g_pal_log_target->empty()) {
52
+ g_pal_log_target->push_back('\n');
53
+ }
54
+ if (message != nullptr && length > 0) {
55
+ g_pal_log_target->append(message, length);
56
+ }
57
+ } catch (...) {
58
+ // Never throw across C PAL boundary.
59
+ }
60
+ return;
61
+ }
62
+ if (g_default_pal_emit != nullptr && (level == kError || level == kFatal)) {
63
+ // Only forward uncaptured errors if user opted in via EXECUTORCH_LOG_STDERR.
64
+ static const bool forward_stderr =
65
+ std::getenv("EXECUTORCH_LOG_STDERR") != nullptr;
66
+ if (forward_stderr) {
67
+ g_default_pal_emit(
68
+ timestamp, level, filename, function, line, message, length);
69
+ }
70
+ }
71
+ }
72
+
73
+ bool mul_overflow_size(size_t a, size_t b, size_t* out) noexcept {
74
+ #if defined(__GNUC__) || defined(__clang__)
75
+ return __builtin_mul_overflow(a, b, out);
76
+ #else
77
+ if (a != 0 && b > std::numeric_limits<size_t>::max() / a) {
78
+ return true;
79
+ }
80
+ *out = a * b;
81
+ return false;
82
+ #endif
83
+ }
84
+
85
+ } // namespace
86
+
87
+ void init_runtime_once() noexcept {
88
+ std::call_once(g_runtime_init_flag, []() {
89
+ executorch::runtime::runtime_init();
90
+ const auto* existing = executorch::runtime::get_pal_impl();
91
+ if (existing != nullptr) {
92
+ g_default_pal_emit = existing->emit_log_message;
93
+ }
94
+ auto impl = executorch::runtime::PalImpl::create(
95
+ &etrb_pal_emit_log, __FILE__);
96
+ executorch::runtime::register_pal(impl);
97
+ });
98
+ }
99
+
100
+ PalLogScope::PalLogScope() noexcept {
101
+ prev_buf_ = g_pal_log_target;
102
+ g_pal_log_target = &buf_;
103
+ }
104
+
105
+ PalLogScope::~PalLogScope() noexcept {
106
+ g_pal_log_target = prev_buf_;
107
+ }
108
+
109
+ std::string PalLogScope::take() noexcept {
110
+ return std::move(buf_);
111
+ }
112
+
113
+ Status status_from_et_error(
114
+ executorch::runtime::Error err,
115
+ const char* context,
116
+ std::string pal_log,
117
+ ErrorKind default_kind) noexcept {
118
+ if (err == executorch::runtime::Error::Ok) {
119
+ return Status::Ok();
120
+ }
121
+ ErrorKind kind = default_kind;
122
+ switch (err) {
123
+ case executorch::runtime::Error::OperatorMissing:
124
+ kind = ErrorKind::OperatorMissing;
125
+ break;
126
+ case executorch::runtime::Error::DelegateInvalidCompatibility:
127
+ case executorch::runtime::Error::DelegateMemoryAllocationFailed:
128
+ case executorch::runtime::Error::DelegateInvalidHandle:
129
+ kind = ErrorKind::Delegate;
130
+ break;
131
+ case executorch::runtime::Error::MemoryAllocationFailed:
132
+ case executorch::runtime::Error::OutOfResources:
133
+ kind = ErrorKind::Memory;
134
+ break;
135
+ case executorch::runtime::Error::AccessFailed:
136
+ case executorch::runtime::Error::InvalidProgram:
137
+ case executorch::runtime::Error::InvalidExternalData:
138
+ kind = ErrorKind::Load;
139
+ break;
140
+ case executorch::runtime::Error::NotFound:
141
+ kind = (default_kind == ErrorKind::MethodNotFound)
142
+ ? ErrorKind::MethodNotFound
143
+ : ErrorKind::Load;
144
+ break;
145
+ case executorch::runtime::Error::InvalidArgument:
146
+ if (default_kind == ErrorKind::Load) {
147
+ kind = ErrorKind::Load;
148
+ } else if (default_kind == ErrorKind::MethodNotFound ||
149
+ pal_log.find("No method named") != std::string::npos) {
150
+ kind = ErrorKind::MethodNotFound;
151
+ } else {
152
+ kind = ErrorKind::Execution;
153
+ }
154
+ break;
155
+ case executorch::runtime::Error::InvalidType:
156
+ kind = ErrorKind::Type;
157
+ break;
158
+ default:
159
+ kind = default_kind;
160
+ break;
161
+ }
162
+
163
+ try {
164
+ const char* ename = executorch::runtime::to_string(err);
165
+ std::string msg = std::string(context) + ": " + ename + " (0x" +
166
+ [&]() {
167
+ char hex[16];
168
+ std::snprintf(
169
+ hex, sizeof(hex), "%x", static_cast<unsigned>(err));
170
+ return std::string(hex);
171
+ }() +
172
+ ")";
173
+ if (!pal_log.empty()) {
174
+ msg += " — " + pal_log;
175
+ }
176
+ return Status::Make(
177
+ kind, std::move(msg), static_cast<uint32_t>(err), std::move(pal_log));
178
+ } catch (...) {
179
+ return Status::Make(
180
+ ErrorKind::Memory, "out of memory formatting ExecuTorch error");
181
+ }
182
+ }
183
+
184
+ // ---------------------------------------------------------------------------
185
+ // TensorCore helpers
186
+ // ---------------------------------------------------------------------------
187
+
188
+ AlignedBuffer::~AlignedBuffer() noexcept {
189
+ if (ptr != nullptr) {
190
+ std::free(ptr);
191
+ ptr = nullptr;
192
+ nbytes = 0;
193
+ }
194
+ }
195
+
196
+ AlignedBuffer::AlignedBuffer(AlignedBuffer&& other) noexcept
197
+ : ptr(other.ptr), nbytes(other.nbytes) {
198
+ other.ptr = nullptr;
199
+ other.nbytes = 0;
200
+ }
201
+
202
+ AlignedBuffer& AlignedBuffer::operator=(AlignedBuffer&& other) noexcept {
203
+ if (this != &other) {
204
+ if (ptr != nullptr) {
205
+ std::free(ptr);
206
+ }
207
+ ptr = other.ptr;
208
+ nbytes = other.nbytes;
209
+ other.ptr = nullptr;
210
+ other.nbytes = 0;
211
+ }
212
+ return *this;
213
+ }
214
+
215
+ StatusOr<AlignedBuffer> AlignedBuffer::allocate(size_t nbytes) noexcept {
216
+ AlignedBuffer buf;
217
+ buf.nbytes = nbytes;
218
+ if (nbytes == 0) {
219
+ return StatusOr<AlignedBuffer>::Ok(std::move(buf));
220
+ }
221
+ void* mem = nullptr;
222
+ if (::posix_memalign(&mem, 64, nbytes) != 0 || mem == nullptr) {
223
+ return StatusOr<AlignedBuffer>::Err(
224
+ Status::Make(ErrorKind::Memory, "failed to allocate tensor buffer"));
225
+ }
226
+ buf.ptr = mem;
227
+ return StatusOr<AlignedBuffer>::Ok(std::move(buf));
228
+ }
229
+
230
+ StatusOr<AlignedBuffer> AlignedBuffer::allocate_zeroed(size_t nbytes) noexcept {
231
+ auto res = allocate(nbytes);
232
+ if (!res.ok()) {
233
+ return res;
234
+ }
235
+ if (nbytes > 0 && res.value.ptr != nullptr) {
236
+ std::memset(res.value.ptr, 0, nbytes);
237
+ }
238
+ return res;
239
+ }
240
+
241
+ StatusOr<ShapeInfo> validate_shape(
242
+ const std::vector<int64_t>& raw_dims,
243
+ executorch::aten::ScalarType dtype) noexcept {
244
+ try {
245
+ if (!is_supported_dtype(dtype)) {
246
+ return StatusOr<ShapeInfo>::Err(Status::Make(
247
+ ErrorKind::DType,
248
+ std::string("unsupported dtype: ") + scalar_type_name(dtype)));
249
+ }
250
+ const size_t item_size = scalar_type_size(dtype);
251
+ ShapeInfo info;
252
+ info.sizes.reserve(raw_dims.size());
253
+
254
+ constexpr int64_t kMaxDim =
255
+ static_cast<int64_t>(std::numeric_limits<executorch::aten::SizesType>::max());
256
+ size_t numel = 1;
257
+ bool has_zero = false;
258
+
259
+ for (size_t i = 0; i < raw_dims.size(); ++i) {
260
+ const int64_t d = raw_dims[i];
261
+ if (d < 0) {
262
+ return StatusOr<ShapeInfo>::Err(Status::Make(
263
+ ErrorKind::Shape,
264
+ "shape dimension at axis " + std::to_string(i) +
265
+ " must be non-negative, got " + std::to_string(d)));
266
+ }
267
+ if (d > kMaxDim) {
268
+ return StatusOr<ShapeInfo>::Err(Status::Make(
269
+ ErrorKind::Shape,
270
+ "shape dimension at axis " + std::to_string(i) +
271
+ " exceeds INT32_MAX: " + std::to_string(d)));
272
+ }
273
+ info.sizes.push_back(static_cast<executorch::aten::SizesType>(d));
274
+ if (d == 0) {
275
+ has_zero = true;
276
+ } else if (!has_zero) {
277
+ size_t next = 0;
278
+ if (mul_overflow_size(numel, static_cast<size_t>(d), &next)) {
279
+ return StatusOr<ShapeInfo>::Err(Status::Make(
280
+ ErrorKind::Shape, "tensor element count overflows size_t"));
281
+ }
282
+ numel = next;
283
+ }
284
+ }
285
+
286
+ info.numel = has_zero ? 0 : numel;
287
+ if (mul_overflow_size(info.numel, item_size, &info.nbytes)) {
288
+ return StatusOr<ShapeInfo>::Err(Status::Make(
289
+ ErrorKind::Shape, "tensor byte size overflows size_t"));
290
+ }
291
+ if (info.nbytes > static_cast<size_t>(std::numeric_limits<ssize_t>::max())) {
292
+ return StatusOr<ShapeInfo>::Err(Status::Make(
293
+ ErrorKind::Shape, "tensor byte size exceeds SSIZE_MAX"));
294
+ }
295
+ return StatusOr<ShapeInfo>::Ok(std::move(info));
296
+ } catch (const std::bad_alloc&) {
297
+ return StatusOr<ShapeInfo>::Err(
298
+ Status::Make(ErrorKind::Memory, "out of memory validating shape"));
299
+ } catch (...) {
300
+ return StatusOr<ShapeInfo>::Err(
301
+ Status::Make(ErrorKind::Shape, "unexpected error validating shape"));
302
+ }
303
+ }
304
+
305
+ bool is_standard_contiguous(const executorch::aten::Tensor& tensor) noexcept {
306
+ const ssize_t ndim = tensor.dim();
307
+ if (ndim <= 0 || tensor.numel() <= 1) {
308
+ return true;
309
+ }
310
+ const auto sizes = tensor.sizes();
311
+ const auto strides = tensor.strides();
312
+ if (static_cast<ssize_t>(strides.size()) != ndim) {
313
+ return false;
314
+ }
315
+ int64_t expected_stride = 1;
316
+ for (ssize_t i = ndim - 1; i >= 0; --i) {
317
+ const int64_t sz = static_cast<int64_t>(sizes[i]);
318
+ if (sz == 0) {
319
+ return true;
320
+ }
321
+ if (sz > 1 && static_cast<int64_t>(strides[i]) != expected_stride) {
322
+ return false;
323
+ }
324
+ expected_stride *= sz;
325
+ }
326
+ return true;
327
+ }
328
+
329
+ Status copy_to_contiguous_bytes(
330
+ const executorch::aten::Tensor& tensor,
331
+ void* dst,
332
+ size_t dst_nbytes) noexcept {
333
+ const size_t elem_size = static_cast<size_t>(tensor.element_size());
334
+ const size_t numel = static_cast<size_t>(tensor.numel());
335
+ size_t needed = 0;
336
+ if (mul_overflow_size(numel, elem_size, &needed) || needed != dst_nbytes) {
337
+ return Status::Make(
338
+ ErrorKind::Shape, "destination buffer size does not match tensor nbytes");
339
+ }
340
+ if (needed == 0) {
341
+ return Status::Ok();
342
+ }
343
+ const auto* src = static_cast<const uint8_t*>(tensor.const_data_ptr());
344
+ if (src == nullptr || dst == nullptr) {
345
+ return Status::Make(ErrorKind::Memory, "null tensor data pointer");
346
+ }
347
+ if (is_standard_contiguous(tensor)) {
348
+ std::memcpy(dst, src, needed);
349
+ return Status::Ok();
350
+ }
351
+
352
+ // General strided copy handling NHWC / channels-last / transposed layouts.
353
+ try {
354
+ const ssize_t ndim = tensor.dim();
355
+ const auto sizes = tensor.sizes();
356
+ const auto strides = tensor.strides();
357
+ auto* out = static_cast<uint8_t*>(dst);
358
+
359
+ std::vector<int64_t> coords(static_cast<size_t>(ndim), 0);
360
+ for (size_t linear = 0; linear < numel; ++linear) {
361
+ int64_t elem_offset = 0;
362
+ for (ssize_t d = 0; d < ndim; ++d) {
363
+ elem_offset += coords[static_cast<size_t>(d)] *
364
+ static_cast<int64_t>(strides[d]);
365
+ }
366
+ std::memcpy(
367
+ out + linear * elem_size,
368
+ src + static_cast<size_t>(elem_offset) * elem_size,
369
+ elem_size);
370
+
371
+ for (ssize_t d = ndim - 1; d >= 0; --d) {
372
+ coords[static_cast<size_t>(d)]++;
373
+ if (coords[static_cast<size_t>(d)] < static_cast<int64_t>(sizes[d])) {
374
+ break;
375
+ }
376
+ coords[static_cast<size_t>(d)] = 0;
377
+ }
378
+ }
379
+ return Status::Ok();
380
+ } catch (const std::bad_alloc&) {
381
+ return Status::Make(
382
+ ErrorKind::Memory, "out of memory copying strided tensor");
383
+ } catch (...) {
384
+ return Status::Make(
385
+ ErrorKind::Execution, "unexpected error copying strided tensor");
386
+ }
387
+ }
388
+
389
+ StatusOr<executorch::extension::TensorPtr> make_owned_tensor_ptr(
390
+ std::vector<executorch::aten::SizesType> sizes,
391
+ executorch::aten::ScalarType dtype,
392
+ AlignedBuffer buf) noexcept {
393
+ try {
394
+ void* raw = buf.ptr;
395
+ auto holder = std::make_shared<AlignedBuffer>(std::move(buf));
396
+ auto ptr = executorch::extension::make_tensor_ptr(
397
+ std::move(sizes),
398
+ raw,
399
+ dtype,
400
+ executorch::aten::Device(executorch::aten::DeviceType::CPU),
401
+ executorch::aten::TensorShapeDynamism::DYNAMIC_BOUND,
402
+ [holder = std::move(holder)](void*) {});
403
+ return StatusOr<executorch::extension::TensorPtr>::Ok(std::move(ptr));
404
+ } catch (const std::bad_alloc&) {
405
+ return StatusOr<executorch::extension::TensorPtr>::Err(
406
+ Status::Make(ErrorKind::Memory, "out of memory creating tensor"));
407
+ } catch (const std::exception& ex) {
408
+ return StatusOr<executorch::extension::TensorPtr>::Err(
409
+ Status::Make(ErrorKind::Execution, ex.what()));
410
+ } catch (...) {
411
+ return StatusOr<executorch::extension::TensorPtr>::Err(
412
+ Status::Make(ErrorKind::Execution, "failed creating tensor"));
413
+ }
414
+ }
415
+
416
+ StatusOr<executorch::extension::TensorPtr> make_borrowed_tensor_ptr(
417
+ std::vector<executorch::aten::SizesType> sizes,
418
+ executorch::aten::ScalarType dtype,
419
+ void* data) noexcept {
420
+ try {
421
+ auto ptr = executorch::extension::make_tensor_ptr(
422
+ std::move(sizes),
423
+ data,
424
+ dtype,
425
+ executorch::aten::Device(executorch::aten::DeviceType::CPU),
426
+ executorch::aten::TensorShapeDynamism::DYNAMIC_BOUND,
427
+ nullptr);
428
+ return StatusOr<executorch::extension::TensorPtr>::Ok(std::move(ptr));
429
+ } catch (const std::bad_alloc&) {
430
+ return StatusOr<executorch::extension::TensorPtr>::Err(
431
+ Status::Make(ErrorKind::Memory, "out of memory creating borrowed tensor"));
432
+ } catch (const std::exception& ex) {
433
+ return StatusOr<executorch::extension::TensorPtr>::Err(
434
+ Status::Make(ErrorKind::Execution, ex.what()));
435
+ } catch (...) {
436
+ return StatusOr<executorch::extension::TensorPtr>::Err(
437
+ Status::Make(ErrorKind::Execution, "failed creating borrowed tensor"));
438
+ }
439
+ }
440
+
441
+ StatusOr<executorch::extension::TensorPtr> clone_contiguous_tensor_ptr(
442
+ const executorch::aten::Tensor& tensor) noexcept {
443
+ try {
444
+ const auto dtype = tensor.scalar_type();
445
+ const size_t nbytes =
446
+ static_cast<size_t>(tensor.numel()) * static_cast<size_t>(tensor.element_size());
447
+ auto buf_res = AlignedBuffer::allocate(nbytes);
448
+ if (!buf_res.ok()) {
449
+ return StatusOr<executorch::extension::TensorPtr>::Err(
450
+ std::move(buf_res.status));
451
+ }
452
+ auto cp = copy_to_contiguous_bytes(tensor, buf_res.value.ptr, nbytes);
453
+ if (!cp.ok()) {
454
+ return StatusOr<executorch::extension::TensorPtr>::Err(std::move(cp));
455
+ }
456
+ std::vector<executorch::aten::SizesType> sizes(
457
+ tensor.sizes().begin(), tensor.sizes().end());
458
+ return make_owned_tensor_ptr(
459
+ std::move(sizes), dtype, std::move(buf_res.value));
460
+ } catch (const std::bad_alloc&) {
461
+ return StatusOr<executorch::extension::TensorPtr>::Err(
462
+ Status::Make(ErrorKind::Memory, "out of memory cloning tensor"));
463
+ } catch (...) {
464
+ return StatusOr<executorch::extension::TensorPtr>::Err(
465
+ Status::Make(ErrorKind::Execution, "failed cloning tensor"));
466
+ }
467
+ }
468
+
469
+ // ---------------------------------------------------------------------------
470
+ // ModuleCore
471
+ // ---------------------------------------------------------------------------
472
+
473
+ StatusOr<std::unique_ptr<ModuleCore>> ModuleCore::open_file(
474
+ const std::string& path,
475
+ const std::vector<std::string>& data_files,
476
+ LoadMode load_mode) noexcept {
477
+ init_runtime_once();
478
+ PalLogScope pal;
479
+ try {
480
+ struct stat st {};
481
+ if (::stat(path.c_str(), &st) != 0) {
482
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(Status::Make(
483
+ ErrorKind::Load,
484
+ "Cannot open ExecuTorch program '" + path + "': " +
485
+ std::strerror(errno)));
486
+ }
487
+ if (S_ISDIR(st.st_mode)) {
488
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(Status::Make(
489
+ ErrorKind::Load,
490
+ "ExecuTorch program path is a directory: '" + path + "'"));
491
+ }
492
+
493
+ size_t est = static_cast<size_t>(st.st_size);
494
+ for (const auto& df : data_files) {
495
+ struct stat dst {};
496
+ if (::stat(df.c_str(), &dst) != 0 || S_ISDIR(dst.st_mode)) {
497
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(Status::Make(
498
+ ErrorKind::Load,
499
+ "Cannot open external data file '" + df + "'"));
500
+ }
501
+ est += static_cast<size_t>(dst.st_size);
502
+ }
503
+
504
+ std::unique_ptr<ModuleCore> core(new ModuleCore());
505
+ core->source_label_ = path;
506
+ core->estimated_bytes_ = est;
507
+
508
+ if (data_files.empty()) {
509
+ core->module_ = std::make_unique<executorch::extension::Module>(
510
+ path, load_mode);
511
+ } else {
512
+ core->module_ = std::make_unique<executorch::extension::Module>(
513
+ path, data_files, load_mode);
514
+ }
515
+
516
+ auto err = core->module_->load(
517
+ executorch::runtime::Program::Verification::Minimal);
518
+ if (err != executorch::runtime::Error::Ok) {
519
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(status_from_et_error(
520
+ err,
521
+ ("Failed to load ExecuTorch program '" + path + "'").c_str(),
522
+ pal.take(),
523
+ ErrorKind::Load));
524
+ }
525
+
526
+ return StatusOr<std::unique_ptr<ModuleCore>>::Ok(std::move(core));
527
+ } catch (const std::bad_alloc&) {
528
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
529
+ Status::Make(ErrorKind::Memory, "out of memory loading module"));
530
+ } catch (const std::exception& ex) {
531
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
532
+ Status::Make(ErrorKind::Load, ex.what(), 0, pal.take()));
533
+ } catch (...) {
534
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
535
+ Status::Make(ErrorKind::Load, "unknown error loading module", 0, pal.take()));
536
+ }
537
+ }
538
+
539
+ StatusOr<std::unique_ptr<ModuleCore>> ModuleCore::open_bytes(
540
+ std::vector<uint8_t> bytes) noexcept {
541
+ init_runtime_once();
542
+ PalLogScope pal;
543
+ try {
544
+ if (bytes.empty()) {
545
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(Status::Make(
546
+ ErrorKind::Load, "Cannot load ExecuTorch program from empty bytes"));
547
+ }
548
+ std::unique_ptr<ModuleCore> core(new ModuleCore());
549
+ core->source_label_ = "<bytes:" + std::to_string(bytes.size()) + ">";
550
+ core->estimated_bytes_ = bytes.size();
551
+ core->owned_bytes_ = std::move(bytes);
552
+
553
+ auto loader = std::make_unique<executorch::extension::BufferDataLoader>(
554
+ core->owned_bytes_.data(), core->owned_bytes_.size());
555
+ core->module_ = std::make_unique<executorch::extension::Module>(
556
+ std::move(loader));
557
+
558
+ auto err = core->module_->load(
559
+ executorch::runtime::Program::Verification::Minimal);
560
+ if (err != executorch::runtime::Error::Ok) {
561
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(status_from_et_error(
562
+ err,
563
+ "Failed to load ExecuTorch program from bytes",
564
+ pal.take(),
565
+ ErrorKind::Load));
566
+ }
567
+ return StatusOr<std::unique_ptr<ModuleCore>>::Ok(std::move(core));
568
+ } catch (const std::bad_alloc&) {
569
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
570
+ Status::Make(ErrorKind::Memory, "out of memory loading module from bytes"));
571
+ } catch (const std::exception& ex) {
572
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
573
+ Status::Make(ErrorKind::Load, ex.what(), 0, pal.take()));
574
+ } catch (...) {
575
+ return StatusOr<std::unique_ptr<ModuleCore>>::Err(
576
+ Status::Make(ErrorKind::Load, "unknown error loading bytes", 0, pal.take()));
577
+ }
578
+ }
579
+
580
+ void ModuleCore::close() noexcept {
581
+ std::unique_lock<std::shared_mutex> lock(lifecycle_mu_);
582
+ if (closed_) {
583
+ return;
584
+ }
585
+ closed_ = true;
586
+ // Destroy module BEFORE clearing owned_bytes_ (BufferDataLoader borrows it).
587
+ module_.reset();
588
+ owned_bytes_.clear();
589
+ owned_bytes_.shrink_to_fit();
590
+ estimated_bytes_ = 0;
591
+ }
592
+
593
+ bool ModuleCore::is_closed() const noexcept {
594
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
595
+ return closed_;
596
+ }
597
+
598
+ std::string ModuleCore::source_label() const noexcept {
599
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
600
+ return source_label_;
601
+ }
602
+
603
+ size_t ModuleCore::estimated_memory_bytes() const noexcept {
604
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
605
+ return estimated_bytes_;
606
+ }
607
+
608
+ std::mutex& ModuleCore::mutex_for_method(const std::string& name) {
609
+ std::lock_guard<std::mutex> g(method_mutexes_mu_);
610
+ auto& ptr = method_mutexes_[name];
611
+ if (!ptr) {
612
+ ptr = std::make_unique<std::mutex>();
613
+ }
614
+ return *ptr;
615
+ }
616
+
617
+ StatusOr<std::vector<std::string>> ModuleCore::method_names() noexcept {
618
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
619
+ if (closed_ || !module_) {
620
+ return StatusOr<std::vector<std::string>>::Err(
621
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
622
+ }
623
+ PalLogScope pal;
624
+ try {
625
+ auto res = module_->method_names();
626
+ if (!res.ok()) {
627
+ return StatusOr<std::vector<std::string>>::Err(status_from_et_error(
628
+ res.error(), "Failed to list method names", pal.take()));
629
+ }
630
+ std::vector<std::string> names(res->begin(), res->end());
631
+ std::sort(names.begin(), names.end());
632
+ return StatusOr<std::vector<std::string>>::Ok(std::move(names));
633
+ } catch (const std::bad_alloc&) {
634
+ return StatusOr<std::vector<std::string>>::Err(
635
+ Status::Make(ErrorKind::Memory, "out of memory listing methods"));
636
+ } catch (...) {
637
+ return StatusOr<std::vector<std::string>>::Err(
638
+ Status::Make(ErrorKind::Execution, "unexpected error listing methods"));
639
+ }
640
+ }
641
+
642
+ StatusOr<MethodMetaSnapshot> ModuleCore::method_meta(
643
+ const std::string& name) noexcept {
644
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
645
+ if (closed_ || !module_) {
646
+ return StatusOr<MethodMetaSnapshot>::Err(
647
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
648
+ }
649
+ PalLogScope pal;
650
+ try {
651
+ auto res = module_->method_meta(name);
652
+ if (!res.ok()) {
653
+ return StatusOr<MethodMetaSnapshot>::Err(status_from_et_error(
654
+ res.error(),
655
+ ("Failed to read metadata for method '" + name + "'").c_str(),
656
+ pal.take(),
657
+ ErrorKind::MethodNotFound));
658
+ }
659
+ const auto& meta = res.get();
660
+ MethodMetaSnapshot snap;
661
+ snap.name = meta.name() ? meta.name() : name;
662
+
663
+ const size_t n_in = meta.num_inputs();
664
+ snap.inputs.reserve(n_in);
665
+ for (size_t i = 0; i < n_in; ++i) {
666
+ ValueMetaSnapshot vm;
667
+ auto tag_res = meta.input_tag(i);
668
+ if (tag_res.ok()) {
669
+ vm.tag = tag_res.get();
670
+ if (vm.tag == executorch::runtime::Tag::Tensor) {
671
+ auto tm_res = meta.input_tensor_meta(i);
672
+ if (tm_res.ok()) {
673
+ const auto& tm = tm_res.get();
674
+ vm.tensor.name = std::string(tm.name());
675
+ vm.tensor.dtype = tm.scalar_type();
676
+ vm.tensor.sizes.assign(tm.sizes().begin(), tm.sizes().end());
677
+ vm.tensor.dim_order.assign(
678
+ tm.dim_order().begin(), tm.dim_order().end());
679
+ vm.tensor.nbytes = tm.nbytes();
680
+ vm.tensor.memory_planned = tm.is_memory_planned();
681
+ }
682
+ }
683
+ }
684
+ snap.inputs.push_back(std::move(vm));
685
+ }
686
+
687
+ const size_t n_out = meta.num_outputs();
688
+ snap.outputs.reserve(n_out);
689
+ for (size_t i = 0; i < n_out; ++i) {
690
+ ValueMetaSnapshot vm;
691
+ auto tag_res = meta.output_tag(i);
692
+ if (tag_res.ok()) {
693
+ vm.tag = tag_res.get();
694
+ if (vm.tag == executorch::runtime::Tag::Tensor) {
695
+ auto tm_res = meta.output_tensor_meta(i);
696
+ if (tm_res.ok()) {
697
+ const auto& tm = tm_res.get();
698
+ vm.tensor.name = std::string(tm.name());
699
+ vm.tensor.dtype = tm.scalar_type();
700
+ vm.tensor.sizes.assign(tm.sizes().begin(), tm.sizes().end());
701
+ vm.tensor.dim_order.assign(
702
+ tm.dim_order().begin(), tm.dim_order().end());
703
+ vm.tensor.nbytes = tm.nbytes();
704
+ vm.tensor.memory_planned = tm.is_memory_planned();
705
+ }
706
+ }
707
+ }
708
+ snap.outputs.push_back(std::move(vm));
709
+ }
710
+
711
+ const size_t n_backends = meta.num_backends();
712
+ snap.backends.reserve(n_backends);
713
+ for (size_t i = 0; i < n_backends; ++i) {
714
+ auto b_res = meta.get_backend_name(i);
715
+ if (b_res.ok() && b_res.get() != nullptr) {
716
+ snap.backends.emplace_back(b_res.get());
717
+ }
718
+ }
719
+
720
+ const size_t n_bufs = meta.num_memory_planned_buffers();
721
+ snap.planned_buffer_sizes.reserve(n_bufs);
722
+ int64_t total = 0;
723
+ for (size_t i = 0; i < n_bufs; ++i) {
724
+ auto sz_res = meta.memory_planned_buffer_size(i);
725
+ if (sz_res.ok()) {
726
+ snap.planned_buffer_sizes.push_back(sz_res.get());
727
+ if (sz_res.get() > 0) {
728
+ total += sz_res.get();
729
+ }
730
+ }
731
+ }
732
+ snap.planned_memory_bytes = total;
733
+
734
+ return StatusOr<MethodMetaSnapshot>::Ok(std::move(snap));
735
+ } catch (const std::bad_alloc&) {
736
+ return StatusOr<MethodMetaSnapshot>::Err(
737
+ Status::Make(ErrorKind::Memory, "out of memory reading method_meta"));
738
+ } catch (...) {
739
+ return StatusOr<MethodMetaSnapshot>::Err(
740
+ Status::Make(ErrorKind::Execution, "unexpected error reading method_meta"));
741
+ }
742
+ }
743
+
744
+ Status ModuleCore::load_method(const std::string& name) noexcept {
745
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
746
+ if (closed_ || !module_) {
747
+ return Status::Make(ErrorKind::Closed, "ExecuTorch module is closed");
748
+ }
749
+ PalLogScope pal;
750
+ try {
751
+ // Module::methods_ map is mutated on load_method, so serialize on method_mutexes_mu_
752
+ // while checking/loading the method.
753
+ std::mutex& m_mu = mutex_for_method(name);
754
+ std::lock_guard<std::mutex> mg(m_mu);
755
+ auto err = module_->load_method(name);
756
+ if (err != executorch::runtime::Error::Ok) {
757
+ return status_from_et_error(
758
+ err,
759
+ ("Failed to load method '" + name + "'").c_str(),
760
+ pal.take(),
761
+ ErrorKind::MethodNotFound);
762
+ }
763
+ return Status::Ok();
764
+ } catch (const std::bad_alloc&) {
765
+ return Status::Make(ErrorKind::Memory, "out of memory loading method");
766
+ } catch (const std::exception& ex) {
767
+ return Status::Make(ErrorKind::Execution, ex.what(), 0, pal.take());
768
+ } catch (...) {
769
+ return Status::Make(
770
+ ErrorKind::Execution, "unknown error loading method", 0, pal.take());
771
+ }
772
+ }
773
+
774
+ StatusOr<bool> ModuleCore::unload_method(const std::string& name) noexcept {
775
+ // Unique lock ensures no concurrent execute() is running on any method
776
+ // while Module::methods_ hash map is modified.
777
+ std::unique_lock<std::shared_mutex> lock(lifecycle_mu_);
778
+ if (closed_ || !module_) {
779
+ return StatusOr<bool>::Err(
780
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
781
+ }
782
+ try {
783
+ return StatusOr<bool>::Ok(module_->unload_method(name));
784
+ } catch (...) {
785
+ return StatusOr<bool>::Err(
786
+ Status::Make(ErrorKind::Execution, "failed unloading method"));
787
+ }
788
+ }
789
+
790
+ bool ModuleCore::is_method_loaded(const std::string& name) noexcept {
791
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
792
+ if (closed_ || !module_) {
793
+ return false;
794
+ }
795
+ try {
796
+ std::mutex& m_mu = mutex_for_method(name);
797
+ std::lock_guard<std::mutex> mg(m_mu);
798
+ return module_->is_method_loaded(name);
799
+ } catch (...) {
800
+ return false;
801
+ }
802
+ }
803
+
804
+ StatusOr<std::vector<CoreValue>> ModuleCore::execute(
805
+ const std::string& method_name,
806
+ const std::vector<CoreValue>& inputs,
807
+ const std::vector<executorch::extension::TensorPtr>& out_targets) noexcept {
808
+ std::shared_lock<std::shared_mutex> lock(lifecycle_mu_);
809
+ if (closed_ || !module_) {
810
+ return StatusOr<std::vector<CoreValue>>::Err(
811
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
812
+ }
813
+ PalLogScope pal;
814
+ try {
815
+ // Ensure method is loaded under an exclusive check if not already loaded,
816
+ // because Module::methods_ (std::unordered_map) is mutated on first load.
817
+ {
818
+ std::mutex& m_mu = mutex_for_method(method_name);
819
+ std::lock_guard<std::mutex> mg(m_mu);
820
+ if (!module_->is_method_loaded(method_name)) {
821
+ lock.unlock();
822
+ {
823
+ std::unique_lock<std::shared_mutex> ulock(lifecycle_mu_);
824
+ if (closed_ || !module_) {
825
+ return StatusOr<std::vector<CoreValue>>::Err(
826
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
827
+ }
828
+ if (!module_->is_method_loaded(method_name)) {
829
+ auto load_err = module_->load_method(method_name);
830
+ if (load_err != executorch::runtime::Error::Ok) {
831
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
832
+ load_err,
833
+ ("Failed to load method '" + method_name + "'").c_str(),
834
+ pal.take(),
835
+ ErrorKind::MethodNotFound));
836
+ }
837
+ }
838
+ }
839
+ lock.lock();
840
+ if (closed_ || !module_) {
841
+ return StatusOr<std::vector<CoreValue>>::Err(
842
+ Status::Make(ErrorKind::Closed, "ExecuTorch module is closed"));
843
+ }
844
+ }
845
+ }
846
+
847
+ // Per-method lock: ExecuTorch Method owns a single mutable planned arena.
848
+ std::mutex& m_mu = mutex_for_method(method_name);
849
+ std::lock_guard<std::mutex> mg(m_mu);
850
+
851
+ // Backing containers (std::deque guarantees pointer stability on push_back).
852
+ std::deque<executorch::aten::ArrayRef<char>> str_refs;
853
+ std::deque<executorch::aten::ArrayRef<double>> dbl_list_refs;
854
+ std::deque<std::vector<bool>> bool_vecs;
855
+ std::deque<std::unique_ptr<bool[]>> bool_bufs;
856
+ std::deque<executorch::aten::ArrayRef<bool>> bool_list_refs;
857
+
858
+ std::vector<executorch::runtime::EValue> evalues;
859
+ evalues.reserve(inputs.size());
860
+
861
+ for (size_t i = 0; i < inputs.size(); ++i) {
862
+ const auto& in = inputs[i];
863
+ switch (in.kind) {
864
+ case ValueKind::None:
865
+ evalues.emplace_back();
866
+ break;
867
+ case ValueKind::Int:
868
+ evalues.emplace_back(in.int_val);
869
+ break;
870
+ case ValueKind::Double:
871
+ evalues.emplace_back(in.double_val);
872
+ break;
873
+ case ValueKind::Bool:
874
+ evalues.emplace_back(in.bool_val);
875
+ break;
876
+ case ValueKind::String: {
877
+ str_refs.emplace_back(in.string_val.data(), in.string_val.size());
878
+ evalues.emplace_back(&str_refs.back());
879
+ break;
880
+ }
881
+ case ValueKind::Tensor: {
882
+ if (!in.tensor_val) {
883
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
884
+ ErrorKind::Closed,
885
+ "Input tensor " + std::to_string(i) + " is closed"));
886
+ }
887
+ evalues.emplace_back(*in.tensor_val);
888
+ break;
889
+ }
890
+ case ValueKind::DoubleList: {
891
+ dbl_list_refs.emplace_back(in.double_list.data(), in.double_list.size());
892
+ evalues.emplace_back(&dbl_list_refs.back());
893
+ break;
894
+ }
895
+ case ValueKind::BoolList: {
896
+ auto arr = std::make_unique<bool[]>(in.bool_list.size());
897
+ for (size_t j = 0; j < in.bool_list.size(); ++j) {
898
+ arr[j] = in.bool_list[j];
899
+ }
900
+ bool_list_refs.emplace_back(arr.get(), in.bool_list.size());
901
+ bool_bufs.push_back(std::move(arr));
902
+ evalues.emplace_back(&bool_list_refs.back());
903
+ break;
904
+ }
905
+ default:
906
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
907
+ ErrorKind::Type,
908
+ "Unsupported input EValue kind at index " + std::to_string(i)));
909
+ }
910
+ }
911
+
912
+ auto res = module_->execute(method_name, evalues);
913
+ if (!res.ok()) {
914
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
915
+ res.error(),
916
+ ("Execution of method '" + method_name + "' failed").c_str(),
917
+ pal.take(),
918
+ ErrorKind::Execution));
919
+ }
920
+
921
+ const auto& raw_outputs = res.get();
922
+ if (!out_targets.empty() && out_targets.size() != raw_outputs.size()) {
923
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
924
+ ErrorKind::Shape,
925
+ "out: expected " + std::to_string(raw_outputs.size()) +
926
+ " output tensor(s), got " + std::to_string(out_targets.size())));
927
+ }
928
+
929
+ std::vector<CoreValue> outputs;
930
+ outputs.reserve(raw_outputs.size());
931
+
932
+ for (size_t i = 0; i < raw_outputs.size(); ++i) {
933
+ const auto& ev = raw_outputs[i];
934
+ if (ev.isTensor()) {
935
+ const auto& src = ev.toTensor();
936
+ if (!out_targets.empty()) {
937
+ const auto& dst = out_targets[i];
938
+ if (!dst) {
939
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
940
+ ErrorKind::Closed,
941
+ "out: tensor " + std::to_string(i) + " is closed"));
942
+ }
943
+ if (dst->scalar_type() != src.scalar_type()) {
944
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
945
+ ErrorKind::DType,
946
+ std::string("out: dtype mismatch at output ") +
947
+ std::to_string(i) + ": expected " +
948
+ scalar_type_name(src.scalar_type()) + ", got " +
949
+ scalar_type_name(dst->scalar_type())));
950
+ }
951
+ bool same_shape = (dst->dim() == src.dim());
952
+ if (same_shape) {
953
+ for (ssize_t d = 0; d < src.dim(); ++d) {
954
+ if (dst->size(d) != src.size(d)) {
955
+ same_shape = false;
956
+ break;
957
+ }
958
+ }
959
+ }
960
+ if (!same_shape) {
961
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
962
+ ErrorKind::Shape,
963
+ "out: shape mismatch at output " + std::to_string(i)));
964
+ }
965
+ auto cp = copy_to_contiguous_bytes(
966
+ src, dst->mutable_data_ptr(), dst->nbytes());
967
+ if (!cp.ok()) {
968
+ return StatusOr<std::vector<CoreValue>>::Err(std::move(cp));
969
+ }
970
+ outputs.push_back(CoreValue::Tensor(dst));
971
+ } else {
972
+ auto cloned = clone_contiguous_tensor_ptr(src);
973
+ if (!cloned.ok()) {
974
+ return StatusOr<std::vector<CoreValue>>::Err(
975
+ std::move(cloned.status));
976
+ }
977
+ outputs.push_back(CoreValue::Tensor(std::move(cloned.value)));
978
+ }
979
+ } else if (!out_targets.empty()) {
980
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
981
+ ErrorKind::Type,
982
+ "out: target provided for non-tensor output " + std::to_string(i)));
983
+ } else if (ev.isInt()) {
984
+ outputs.push_back(CoreValue::Int(ev.toInt()));
985
+ } else if (ev.isDouble()) {
986
+ outputs.push_back(CoreValue::Double(ev.toDouble()));
987
+ } else if (ev.isBool()) {
988
+ outputs.push_back(CoreValue::Bool(ev.toBool()));
989
+ } else if (ev.isNone()) {
990
+ outputs.push_back(CoreValue::None());
991
+ } else if (ev.isString()) {
992
+ auto sv = ev.tryToString();
993
+ if (!sv.ok()) {
994
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
995
+ sv.error(), "Failed to extract String output", pal.take()));
996
+ }
997
+ outputs.push_back(CoreValue::String(std::string(sv.get())));
998
+ } else if (ev.isIntList()) {
999
+ auto lr = ev.tryToIntList();
1000
+ if (!lr.ok()) {
1001
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
1002
+ lr.error(), "Failed to extract IntList output", pal.take()));
1003
+ }
1004
+ CoreValue cv;
1005
+ cv.kind = ValueKind::IntList;
1006
+ cv.int_list.assign(lr.get().begin(), lr.get().end());
1007
+ outputs.push_back(std::move(cv));
1008
+ } else if (ev.isDoubleList()) {
1009
+ auto lr = ev.tryToDoubleList();
1010
+ if (!lr.ok()) {
1011
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
1012
+ lr.error(), "Failed to extract DoubleList output", pal.take()));
1013
+ }
1014
+ CoreValue cv;
1015
+ cv.kind = ValueKind::DoubleList;
1016
+ cv.double_list.assign(lr.get().begin(), lr.get().end());
1017
+ outputs.push_back(std::move(cv));
1018
+ } else if (ev.isBoolList()) {
1019
+ auto lr = ev.tryToBoolList();
1020
+ if (!lr.ok()) {
1021
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
1022
+ lr.error(), "Failed to extract BoolList output", pal.take()));
1023
+ }
1024
+ CoreValue cv;
1025
+ cv.kind = ValueKind::BoolList;
1026
+ cv.bool_list.assign(lr.get().begin(), lr.get().end());
1027
+ outputs.push_back(std::move(cv));
1028
+ } else if (ev.isTensorList()) {
1029
+ auto lr = ev.tryToTensorList();
1030
+ if (!lr.ok()) {
1031
+ return StatusOr<std::vector<CoreValue>>::Err(status_from_et_error(
1032
+ lr.error(), "Failed to extract TensorList output", pal.take()));
1033
+ }
1034
+ CoreValue cv;
1035
+ cv.kind = ValueKind::TensorList;
1036
+ for (const auto& t : lr.get()) {
1037
+ auto cl = clone_contiguous_tensor_ptr(t);
1038
+ if (!cl.ok()) {
1039
+ return StatusOr<std::vector<CoreValue>>::Err(std::move(cl.status));
1040
+ }
1041
+ cv.tensor_list.push_back(std::move(cl.value));
1042
+ }
1043
+ outputs.push_back(std::move(cv));
1044
+ } else {
1045
+ return StatusOr<std::vector<CoreValue>>::Err(Status::Make(
1046
+ ErrorKind::Type,
1047
+ "Unsupported output EValue tag at output " + std::to_string(i)));
1048
+ }
1049
+ }
1050
+
1051
+ return StatusOr<std::vector<CoreValue>>::Ok(std::move(outputs));
1052
+ } catch (const std::bad_alloc&) {
1053
+ return StatusOr<std::vector<CoreValue>>::Err(
1054
+ Status::Make(ErrorKind::Memory, "out of memory executing method"));
1055
+ } catch (const std::exception& ex) {
1056
+ return StatusOr<std::vector<CoreValue>>::Err(
1057
+ Status::Make(ErrorKind::Execution, ex.what(), 0, pal.take()));
1058
+ } catch (...) {
1059
+ return StatusOr<std::vector<CoreValue>>::Err(
1060
+ Status::Make(ErrorKind::Execution, "unknown error executing method", 0, pal.take()));
1061
+ }
1062
+ }
1063
+
1064
+ // ---------------------------------------------------------------------------
1065
+ // Runtime introspection & threadpool controls
1066
+ // ---------------------------------------------------------------------------
1067
+
1068
+ std::vector<std::string> registered_backends() noexcept {
1069
+ init_runtime_once();
1070
+ std::vector<std::string> out;
1071
+ try {
1072
+ const size_t n = executorch::runtime::get_num_registered_backends();
1073
+ out.reserve(n);
1074
+ for (size_t i = 0; i < n; ++i) {
1075
+ auto res = executorch::runtime::get_backend_name(i);
1076
+ if (res.ok() && res.get() != nullptr) {
1077
+ out.emplace_back(res.get());
1078
+ }
1079
+ }
1080
+ std::sort(out.begin(), out.end());
1081
+ } catch (...) {
1082
+ }
1083
+ return out;
1084
+ }
1085
+
1086
+ size_t registered_kernel_count() noexcept {
1087
+ init_runtime_once();
1088
+ return executorch::runtime::get_registered_kernels().size();
1089
+ }
1090
+
1091
+ int64_t get_num_threads() noexcept {
1092
+ init_runtime_once();
1093
+ #ifdef ET_USE_THREADPOOL
1094
+ std::lock_guard<std::mutex> g(g_threadpool_mu);
1095
+ auto* tp = executorch::extension::threadpool::get_threadpool();
1096
+ if (tp != nullptr) {
1097
+ return static_cast<int64_t>(tp->get_thread_count());
1098
+ }
1099
+ #endif
1100
+ return executorch::extension::get_thread_count();
1101
+ }
1102
+
1103
+ Status set_num_threads(int64_t n) noexcept {
1104
+ init_runtime_once();
1105
+ if (n < 1 || n > 1024) {
1106
+ return Status::Make(
1107
+ ErrorKind::Argument,
1108
+ "num_threads must be between 1 and 1024, got " + std::to_string(n));
1109
+ }
1110
+ #ifdef ET_USE_THREADPOOL
1111
+ try {
1112
+ std::lock_guard<std::mutex> g(g_threadpool_mu);
1113
+ auto* tp = executorch::extension::threadpool::get_threadpool();
1114
+ if (tp == nullptr) {
1115
+ return Status::Make(
1116
+ ErrorKind::Execution, "ExecuTorch threadpool is unavailable");
1117
+ }
1118
+ tp->_unsafe_reset_threadpool(static_cast<uint32_t>(n));
1119
+ return Status::Ok();
1120
+ } catch (...) {
1121
+ return Status::Make(
1122
+ ErrorKind::Execution, "failed resetting ExecuTorch threadpool");
1123
+ }
1124
+ #else
1125
+ return Status::Make(
1126
+ ErrorKind::Execution,
1127
+ "ExecuTorch was built without threadpool support (ET_USE_THREADPOOL)");
1128
+ #endif
1129
+ }
1130
+
1131
+ } // namespace etrb