com.github.asus4.onnxruntime 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Plugins/Android/onnxruntime.aar +0 -0
- package/Plugins/Linux/x64/libonnxruntime.so +0 -0
- package/Plugins/Windows/arm64/onnxruntime.dll +0 -0
- package/Plugins/Windows/x64/onnxruntime.dll +0 -0
- package/Plugins/Windows/x86/onnxruntime.dll +0 -0
- package/Plugins/iOS~/onnxruntime.xcframework/Info.plist +11 -5
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers/onnxruntime_c_api.h +1307 -64
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers/onnxruntime_cxx_api.h +425 -24
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers/onnxruntime_cxx_inline.h +614 -6
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers/onnxruntime_run_options_config_keys.h +3 -0
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers/onnxruntime_session_options_config_keys.h +29 -5
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Info.plist +2 -2
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/onnxruntime +0 -0
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Headers/onnxruntime_c_api.h +1307 -64
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Headers/onnxruntime_cxx_api.h +425 -24
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Headers/onnxruntime_cxx_inline.h +614 -6
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Headers/onnxruntime_run_options_config_keys.h +3 -0
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Headers/onnxruntime_session_options_config_keys.h +29 -5
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/Resources/Info.plist +2 -2
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-maccatalyst/onnxruntime.framework/Versions/A/onnxruntime +0 -0
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Headers/onnxruntime_c_api.h +1307 -64
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Headers/onnxruntime_cxx_api.h +425 -24
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Headers/onnxruntime_cxx_inline.h +614 -6
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Headers/onnxruntime_run_options_config_keys.h +3 -0
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Headers/onnxruntime_session_options_config_keys.h +29 -5
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/Info.plist +2 -2
- package/Plugins/iOS~/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.framework/onnxruntime +0 -0
- package/Plugins/macOS/arm64/libonnxruntime.dylib +0 -0
- package/Plugins/macOS/x64/libonnxruntime.dylib +0 -0
- package/README.md +7 -6
- package/Runtime/ManagedProjections.shared.cs +1 -2
- package/Runtime/NativeMethods.shared.cs +20 -2
- package/Runtime/OrtValue.shared.cs +1 -1
- package/Runtime/SessionOptions.shared.cs +10 -0
- package/Runtime/Training/NativeTrainingMethods.shared.cs +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
1
|
+
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
2
2
|
// Licensed under the MIT License.
|
|
3
3
|
|
|
4
4
|
// See docs\c_cxx\README.md on generating the Doxygen documentation from this file
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
*
|
|
39
39
|
* This value is used by some API functions to behave as this version of the header expects.
|
|
40
40
|
*/
|
|
41
|
-
#define ORT_API_VERSION
|
|
41
|
+
#define ORT_API_VERSION 22
|
|
42
42
|
|
|
43
43
|
#ifdef __cplusplus
|
|
44
44
|
extern "C" {
|
|
@@ -255,6 +255,8 @@ typedef enum OrtErrorCode {
|
|
|
255
255
|
ORT_NOT_IMPLEMENTED,
|
|
256
256
|
ORT_INVALID_GRAPH,
|
|
257
257
|
ORT_EP_FAIL,
|
|
258
|
+
ORT_MODEL_LOAD_CANCELED,
|
|
259
|
+
ORT_MODEL_REQUIRES_COMPILATION,
|
|
258
260
|
} OrtErrorCode;
|
|
259
261
|
|
|
260
262
|
typedef enum OrtOpAttrType {
|
|
@@ -297,6 +299,7 @@ ORT_RUNTIME_CLASS(ThreadingOptions);
|
|
|
297
299
|
ORT_RUNTIME_CLASS(ArenaCfg);
|
|
298
300
|
ORT_RUNTIME_CLASS(PrepackedWeightsContainer);
|
|
299
301
|
ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2);
|
|
302
|
+
ORT_RUNTIME_CLASS(NvTensorRtRtxProviderOptions);
|
|
300
303
|
ORT_RUNTIME_CLASS(CUDAProviderOptionsV2);
|
|
301
304
|
ORT_RUNTIME_CLASS(CANNProviderOptions);
|
|
302
305
|
ORT_RUNTIME_CLASS(DnnlProviderOptions);
|
|
@@ -305,8 +308,16 @@ ORT_RUNTIME_CLASS(OpAttr);
|
|
|
305
308
|
ORT_RUNTIME_CLASS(Logger);
|
|
306
309
|
ORT_RUNTIME_CLASS(ShapeInferContext);
|
|
307
310
|
ORT_RUNTIME_CLASS(LoraAdapter);
|
|
308
|
-
|
|
309
|
-
|
|
311
|
+
ORT_RUNTIME_CLASS(ValueInfo);
|
|
312
|
+
ORT_RUNTIME_CLASS(Node);
|
|
313
|
+
ORT_RUNTIME_CLASS(Graph);
|
|
314
|
+
ORT_RUNTIME_CLASS(Model);
|
|
315
|
+
ORT_RUNTIME_CLASS(ModelCompilationOptions);
|
|
316
|
+
ORT_RUNTIME_CLASS(HardwareDevice);
|
|
317
|
+
ORT_RUNTIME_CLASS(EpDevice);
|
|
318
|
+
ORT_RUNTIME_CLASS(KeyValuePairs);
|
|
319
|
+
|
|
320
|
+
#ifdef _MSC_VER
|
|
310
321
|
typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
|
|
311
322
|
#else
|
|
312
323
|
typedef OrtStatus* OrtStatusPtr;
|
|
@@ -396,6 +407,52 @@ typedef enum OrtMemoryInfoDeviceType {
|
|
|
396
407
|
OrtMemoryInfoDeviceType_FPGA = 2
|
|
397
408
|
} OrtMemoryInfoDeviceType;
|
|
398
409
|
|
|
410
|
+
typedef enum OrtHardwareDeviceType {
|
|
411
|
+
OrtHardwareDeviceType_CPU,
|
|
412
|
+
OrtHardwareDeviceType_GPU,
|
|
413
|
+
OrtHardwareDeviceType_NPU
|
|
414
|
+
} OrtHardwareDeviceType;
|
|
415
|
+
|
|
416
|
+
/** \brief These are the default EP selection policies used by ORT when doing automatic EP selection.
|
|
417
|
+
*/
|
|
418
|
+
typedef enum OrtExecutionProviderDevicePolicy {
|
|
419
|
+
OrtExecutionProviderDevicePolicy_DEFAULT,
|
|
420
|
+
OrtExecutionProviderDevicePolicy_PREFER_CPU,
|
|
421
|
+
OrtExecutionProviderDevicePolicy_PREFER_NPU,
|
|
422
|
+
OrtExecutionProviderDevicePolicy_PREFER_GPU,
|
|
423
|
+
OrtExecutionProviderDevicePolicy_MAX_PERFORMANCE,
|
|
424
|
+
OrtExecutionProviderDevicePolicy_MAX_EFFICIENCY,
|
|
425
|
+
OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER,
|
|
426
|
+
} OrtExecutionProviderDevicePolicy;
|
|
427
|
+
|
|
428
|
+
/** \brief Delegate to allow providing custom OrtEpDevice selection logic
|
|
429
|
+
*
|
|
430
|
+
* This delegate is called by the EP selection code to allow the user to provide custom device selection logic.
|
|
431
|
+
* The user can use this to select OrtEpDevice instances from the list of available devices.
|
|
432
|
+
*
|
|
433
|
+
* \param ep_devices The list of available devices.
|
|
434
|
+
* \param num_devices The number of available devices.
|
|
435
|
+
* \param model_metadata The model metadata.
|
|
436
|
+
* \param runtime_metadata The runtime metadata. May be nullptr.
|
|
437
|
+
* \param selected Pre-allocated array to populate with selected OrtEpDevice pointers from ep_devices.
|
|
438
|
+
* \param max_ep_devices The maximum number of devices that can be selected in the pre-allocated array.
|
|
439
|
+
Currently the maximum is 8.
|
|
440
|
+
* \param num_ep_devices The number of selected devices.
|
|
441
|
+
* \param state Opaque pointer. Required to use the delegate from other languages like C# and python.
|
|
442
|
+
*
|
|
443
|
+
* \return OrtStatus* Selection status. Return nullptr on success.
|
|
444
|
+
* Use CreateStatus to provide error info. Use ORT_FAIL as the error code.
|
|
445
|
+
* ORT will release the OrtStatus* if not null.
|
|
446
|
+
*/
|
|
447
|
+
typedef OrtStatus* (*EpSelectionDelegate)(_In_ const OrtEpDevice** ep_devices,
|
|
448
|
+
_In_ size_t num_devices,
|
|
449
|
+
_In_ const OrtKeyValuePairs* model_metadata,
|
|
450
|
+
_In_opt_ const OrtKeyValuePairs* runtime_metadata,
|
|
451
|
+
_Inout_ const OrtEpDevice** selected,
|
|
452
|
+
_In_ size_t max_selected,
|
|
453
|
+
_Out_ size_t* num_selected,
|
|
454
|
+
_In_ void* state);
|
|
455
|
+
|
|
399
456
|
/** \brief Algorithm to use for cuDNN Convolution Op
|
|
400
457
|
*/
|
|
401
458
|
typedef enum OrtCudnnConvAlgoSearch {
|
|
@@ -665,6 +722,15 @@ typedef struct OrtApi OrtApi;
|
|
|
665
722
|
struct OrtTrainingApi;
|
|
666
723
|
typedef struct OrtTrainingApi OrtTrainingApi;
|
|
667
724
|
|
|
725
|
+
struct OrtModelEditorApi;
|
|
726
|
+
typedef struct OrtModelEditorApi OrtModelEditorApi;
|
|
727
|
+
|
|
728
|
+
struct OrtCompileApi;
|
|
729
|
+
typedef struct OrtCompileApi OrtCompileApi;
|
|
730
|
+
|
|
731
|
+
struct OrtEpApi;
|
|
732
|
+
typedef struct OrtEpApi OrtEpApi;
|
|
733
|
+
|
|
668
734
|
/** \brief The helper interface to get the right version of OrtApi
|
|
669
735
|
*
|
|
670
736
|
* Get a pointer to this structure through ::OrtGetApiBase
|
|
@@ -847,7 +913,8 @@ struct OrtApi {
|
|
|
847
913
|
*
|
|
848
914
|
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
849
915
|
*/
|
|
850
|
-
ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env,
|
|
916
|
+
ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env,
|
|
917
|
+
_In_ const void* model_data, size_t model_data_length,
|
|
851
918
|
_In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
|
|
852
919
|
|
|
853
920
|
/** \brief Run the model in an ::OrtSession
|
|
@@ -1340,6 +1407,8 @@ struct OrtApi {
|
|
|
1340
1407
|
* Create a tensor with user's buffer. You can fill the buffer either before calling this function or after.
|
|
1341
1408
|
* p_data is owned by caller. ReleaseValue won't release p_data.
|
|
1342
1409
|
*
|
|
1410
|
+
* If you wish to transfer ownership of p_data to ORT use CreateTensorWithDataAndDeleterAsOrtValue.
|
|
1411
|
+
*
|
|
1343
1412
|
* \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc).
|
|
1344
1413
|
* \param[in] p_data Pointer to the data buffer.
|
|
1345
1414
|
* \param[in] p_data_len The number of bytes in the data buffer.
|
|
@@ -1997,7 +2066,8 @@ struct OrtApi {
|
|
|
1997
2066
|
/** \brief Get the value type from an ::OrtMapTypeInfo
|
|
1998
2067
|
*
|
|
1999
2068
|
* \param[in] map_type_info
|
|
2000
|
-
* \param[out] type_info
|
|
2069
|
+
* \param[out] type_info A copy of the OrtTypeInfo for the map value type.
|
|
2070
|
+
* The user must free this value with ReleaseTypeInfo.
|
|
2001
2071
|
*
|
|
2002
2072
|
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
2003
2073
|
*/
|
|
@@ -2012,7 +2082,8 @@ struct OrtApi {
|
|
|
2012
2082
|
* This is used by WinML to support model reflection APIs.
|
|
2013
2083
|
*
|
|
2014
2084
|
* \param[in] sequence_type_info
|
|
2015
|
-
* \param[out] type_info
|
|
2085
|
+
* \param[out] type_info A copy of the OrtTypeInfo for the sequence element type.
|
|
2086
|
+
* The user must free this value with ReleaseTypeInfo.
|
|
2016
2087
|
*
|
|
2017
2088
|
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
2018
2089
|
*/
|
|
@@ -2887,7 +2958,8 @@ struct OrtApi {
|
|
|
2887
2958
|
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
2888
2959
|
*/
|
|
2889
2960
|
ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,
|
|
2890
|
-
_In_ const OrtSessionOptions* options,
|
|
2961
|
+
_In_ const OrtSessionOptions* options,
|
|
2962
|
+
_Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container,
|
|
2891
2963
|
_Outptr_ OrtSession** out);
|
|
2892
2964
|
|
|
2893
2965
|
/** \brief Create session from memory with prepacked weights container
|
|
@@ -2910,7 +2982,8 @@ struct OrtApi {
|
|
|
2910
2982
|
*/
|
|
2911
2983
|
ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env,
|
|
2912
2984
|
_In_ const void* model_data, size_t model_data_length,
|
|
2913
|
-
_In_ const OrtSessionOptions* options,
|
|
2985
|
+
_In_ const OrtSessionOptions* options,
|
|
2986
|
+
_Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container,
|
|
2914
2987
|
_Outptr_ OrtSession** out);
|
|
2915
2988
|
|
|
2916
2989
|
/// @}
|
|
@@ -3623,77 +3696,97 @@ struct OrtApi {
|
|
|
3623
3696
|
* \param[in] provider_options_values - values to configure the provider options
|
|
3624
3697
|
* \param[in] num_keys - number of keys passed in
|
|
3625
3698
|
*
|
|
3626
|
-
* Currently supported
|
|
3627
|
-
* QNN
|
|
3628
|
-
*
|
|
3629
|
-
* XNNPACK
|
|
3699
|
+
* Currently supported provider names:
|
|
3700
|
+
* QNNExecutionProvider (or QNN)
|
|
3701
|
+
* OpenVINOExecutionProvider (or OpenVINO)
|
|
3702
|
+
* XnnpackExecutionProvider (or XNNPACK)
|
|
3703
|
+
* WebNNExecutionProvider (or WEBNN)
|
|
3704
|
+
* WebGpuExecutionProvider (or WebGPU)
|
|
3705
|
+
* AzureExecutionProvider (or AZURE)
|
|
3706
|
+
* JsExecutionProvider (or JS)
|
|
3707
|
+
* VitisAIExecutionProvider (or VitisAI)
|
|
3708
|
+
* CoreMLExecutionProvider (or CoreML)
|
|
3630
3709
|
*
|
|
3631
3710
|
* Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_<provider name> function
|
|
3632
3711
|
* that should be used to add it.
|
|
3633
3712
|
*
|
|
3634
3713
|
* QNN supported keys:
|
|
3635
|
-
* "
|
|
3636
|
-
*
|
|
3714
|
+
* "backend_type": Type of QNN backend. Specifies a backend path that is the associated QNN backend library file
|
|
3715
|
+
* name. E.g., given backend type "htp", on Windows, the backend path would be "QnnHtp.dll", and on other
|
|
3716
|
+
* platforms, it would be "libQnnHtp.so". Mutually exclusive with "backend_path".
|
|
3717
|
+
* Available options:
|
|
3718
|
+
* -# "cpu"
|
|
3719
|
+
* -# "gpu"
|
|
3720
|
+
* -# "htp": Default.
|
|
3721
|
+
* -# "saver"
|
|
3722
|
+
* "backend_path": File path to QNN backend library. Mutually exclusive with "backend_type".
|
|
3723
|
+
* "profiling_level": QNN profiling level.
|
|
3724
|
+
* Available options:
|
|
3725
|
+
* -# "off": Default.
|
|
3726
|
+
* -# "basic"
|
|
3727
|
+
* -# "detailed"
|
|
3637
3728
|
* "profiling_file_path": QNN profiling file path if ETW not enabled.
|
|
3638
3729
|
* "rpc_control_latency": QNN RPC control latency.
|
|
3639
3730
|
* "vtcm_mb": QNN VTCM size in MB. default to 0(not set).
|
|
3640
|
-
* "htp_performance_mode": QNN performance mode
|
|
3641
|
-
*
|
|
3731
|
+
* "htp_performance_mode": QNN performance mode.
|
|
3732
|
+
* Available options:
|
|
3733
|
+
* -# "burst"
|
|
3734
|
+
* -# "balanced"
|
|
3735
|
+
* -# "default": Default.
|
|
3736
|
+
* -# "high_performance"
|
|
3737
|
+
* -# "high_power_saver"
|
|
3738
|
+
* -# "low_balanced"
|
|
3739
|
+
* -# "extreme_power_saver"
|
|
3740
|
+
* -# "low_power_saver"
|
|
3741
|
+
* -# "power_saver"
|
|
3742
|
+
* -# "sustained_high_performance"
|
|
3642
3743
|
* "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will
|
|
3643
|
-
*
|
|
3644
|
-
*
|
|
3645
|
-
* "qnn_context_priority": QNN context priority
|
|
3646
|
-
*
|
|
3647
|
-
*
|
|
3648
|
-
*
|
|
3649
|
-
*
|
|
3650
|
-
*
|
|
3651
|
-
* "
|
|
3652
|
-
*
|
|
3653
|
-
*
|
|
3654
|
-
*
|
|
3655
|
-
*
|
|
3656
|
-
*
|
|
3657
|
-
*
|
|
3744
|
+
* dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and
|
|
3745
|
+
* may alter model/EP partitioning. Use only for debugging.
|
|
3746
|
+
* "qnn_context_priority": QNN context priority.
|
|
3747
|
+
* Available options:
|
|
3748
|
+
* -# "low"
|
|
3749
|
+
* -# "normal": Default.
|
|
3750
|
+
* -# "normal_high"
|
|
3751
|
+
* -# "high"
|
|
3752
|
+
* "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend.
|
|
3753
|
+
* Available options:
|
|
3754
|
+
* -# "0": Default.
|
|
3755
|
+
* -# "1": Faster preparation time, less optimal graph.
|
|
3756
|
+
* -# "2": Longer preparation time, more optimal graph.
|
|
3757
|
+
* -# "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific
|
|
3758
|
+
* details.
|
|
3759
|
+
* "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values.
|
|
3760
|
+
* Defaults to "0" (unknown).
|
|
3761
|
+
* "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators.
|
|
3762
|
+
* Available options:
|
|
3763
|
+
* -# "0": Default (none).
|
|
3764
|
+
* -# "68"
|
|
3765
|
+
* -# "69"
|
|
3766
|
+
* -# "73"
|
|
3767
|
+
* -# "75"
|
|
3658
3768
|
* "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device).
|
|
3659
3769
|
* "enable_htp_fp16_precision": Used for float32 model for HTP backend.
|
|
3660
|
-
*
|
|
3661
|
-
*
|
|
3662
|
-
*
|
|
3663
|
-
* "enable_htp_weight_sharing": Enable QNN weight sharing feature while compiling multiple graphs into one QNN context.
|
|
3664
|
-
* - "0": Default. Disabled.
|
|
3665
|
-
* - "1": Enabled.
|
|
3770
|
+
* Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision.
|
|
3771
|
+
* -# "0": With fp32 precision.
|
|
3772
|
+
* -# "1": Default. With fp16 precision.
|
|
3666
3773
|
* "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another
|
|
3667
|
-
*
|
|
3668
|
-
*
|
|
3669
|
-
*
|
|
3670
|
-
* "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context
|
|
3671
|
-
*
|
|
3672
|
-
*
|
|
3774
|
+
* execution provider (typically CPU EP).
|
|
3775
|
+
* -# "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O.
|
|
3776
|
+
* -# "1": Enabled. This is the default value.
|
|
3777
|
+
* "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context
|
|
3778
|
+
* binary.
|
|
3779
|
+
* -# "0": Default. Disabled.
|
|
3780
|
+
* -# "1": Enabled.
|
|
3673
3781
|
* "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to
|
|
3674
|
-
*
|
|
3675
|
-
*
|
|
3676
|
-
*
|
|
3782
|
+
* be available.
|
|
3783
|
+
* -# "0": Default. Disabled.
|
|
3784
|
+
* -# "1": Enabled.
|
|
3677
3785
|
* "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition
|
|
3678
3786
|
* assigned to QNN EP is dumped to a separate file.
|
|
3679
3787
|
* "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the
|
|
3680
3788
|
* program's current working directory. Ignored if "dump_json_qnn_graph" is not set.
|
|
3681
3789
|
*
|
|
3682
|
-
* SNPE supported keys:
|
|
3683
|
-
* "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16",
|
|
3684
|
-
* "DSP", "DSP_FIXED8_TF", "AIP_FIXED_TF", "AIP_FIXED8_TF".
|
|
3685
|
-
* Mapping to SNPE Runtime_t definition: CPU, CPU_FLOAT32 => zdl::DlSystem::Runtime_t::CPU;
|
|
3686
|
-
* GPU, GPU_FLOAT32_16_HYBRID => zdl::DlSystem::Runtime_t::GPU;
|
|
3687
|
-
* GPU_FLOAT16 => zdl::DlSystem::Runtime_t::GPU_FLOAT16;
|
|
3688
|
-
* DSP, DSP_FIXED8_TF => zdl::DlSystem::Runtime_t::DSP.
|
|
3689
|
-
* AIP_FIXED_TF, AIP_FIXED8_TF => zdl::DlSystem::Runtime_t::AIP_FIXED_TF.
|
|
3690
|
-
* "priority": execution priority, options: "low", "normal".
|
|
3691
|
-
* "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT".
|
|
3692
|
-
* "ITENSOR" -- default, ITensor which is float only.
|
|
3693
|
-
* "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model
|
|
3694
|
-
* "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default.
|
|
3695
|
-
* If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail.
|
|
3696
|
-
*
|
|
3697
3790
|
* XNNPACK supported keys:
|
|
3698
3791
|
* "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider.
|
|
3699
3792
|
* default value is 0, which means to use the session thread-pool size.
|
|
@@ -4293,8 +4386,8 @@ struct OrtApi {
|
|
|
4293
4386
|
* specific type that is described by the returned ::OrtTypeInfo.
|
|
4294
4387
|
*
|
|
4295
4388
|
* \param[in] optional_type_info
|
|
4296
|
-
* \param[out] out A
|
|
4297
|
-
*
|
|
4389
|
+
* \param[out] out A copy of ::OrtTypeInfo for what the optional value could be.
|
|
4390
|
+
* The user must free this value with ReleaseTypeInfo.
|
|
4298
4391
|
*
|
|
4299
4392
|
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4300
4393
|
*
|
|
@@ -4786,6 +4879,378 @@ struct OrtApi {
|
|
|
4786
4879
|
*/
|
|
4787
4880
|
ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys,
|
|
4788
4881
|
_In_reads_(kv_len) const char* const* values, _In_ size_t kv_len);
|
|
4882
|
+
|
|
4883
|
+
/** \brief Release an OrtValueInfo instance if it was not added to an OrtGraph.
|
|
4884
|
+
* \since Version 1.22.
|
|
4885
|
+
*/
|
|
4886
|
+
ORT_CLASS_RELEASE(ValueInfo);
|
|
4887
|
+
|
|
4888
|
+
/** \brief Release an OrtNode if it was not added to an OrtGraph.
|
|
4889
|
+
* \since Version 1.22.
|
|
4890
|
+
*/
|
|
4891
|
+
ORT_CLASS_RELEASE(Node);
|
|
4892
|
+
|
|
4893
|
+
/** \brief Release an OrtGraph.
|
|
4894
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4895
|
+
* \since Version 1.22.
|
|
4896
|
+
*/
|
|
4897
|
+
ORT_CLASS_RELEASE(Graph);
|
|
4898
|
+
|
|
4899
|
+
/** \brief Release an OrtModel.
|
|
4900
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4901
|
+
* \since Version 1.22.
|
|
4902
|
+
*/
|
|
4903
|
+
ORT_CLASS_RELEASE(Model);
|
|
4904
|
+
|
|
4905
|
+
/** \brief Get the value name from an OrtValueInfo instance.
|
|
4906
|
+
* \param[in] value_info The OrtValueInfo instance.
|
|
4907
|
+
* \param[out] name The name of the OrtValueInfo
|
|
4908
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4909
|
+
* \since Version 1.22.
|
|
4910
|
+
*/
|
|
4911
|
+
ORT_API2_STATUS(GetValueInfoName, _In_ const OrtValueInfo* value_info, _Out_ const char** name);
|
|
4912
|
+
|
|
4913
|
+
/** \brief Get the type information from an OrtValueInfo instance.
|
|
4914
|
+
* \param[in] value_info The OrtValueInfo instance.
|
|
4915
|
+
* \param[out] type_info The type info of the OrtValueInfo
|
|
4916
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4917
|
+
* \since Version 1.22.
|
|
4918
|
+
*/
|
|
4919
|
+
ORT_API2_STATUS(GetValueInfoTypeInfo, _In_ const OrtValueInfo* value_info, _Outptr_ const OrtTypeInfo** type_info);
|
|
4920
|
+
|
|
4921
|
+
/** \brief Get the Model Editor API instance
|
|
4922
|
+
*
|
|
4923
|
+
* Get the Model Editor API instance to create a new model or augment an existing model.
|
|
4924
|
+
*
|
|
4925
|
+
* \return Model Editor API struct
|
|
4926
|
+
*
|
|
4927
|
+
* \since Version 1.22.
|
|
4928
|
+
*/
|
|
4929
|
+
const OrtModelEditorApi*(ORT_API_CALL* GetModelEditorApi)();
|
|
4930
|
+
|
|
4931
|
+
/** \brief Create an OrtValue for a Tensor that uses pre-existing memory.
|
|
4932
|
+
*
|
|
4933
|
+
* ORT will take ownership of the memory and free it using the provided deleter when no longer in use.
|
|
4934
|
+
*
|
|
4935
|
+
* \param[in] deleter OrtAllocator instance that will be used to free the memory.
|
|
4936
|
+
* Only the OrtAllocator:Info and OrtAllocator::Release functions are required.
|
|
4937
|
+
* The OrtMemoryInfo returned by OrtAllocator::Info must match the location of p_data.
|
|
4938
|
+
* \param[in] p_data Pointer to the memory that will be used by the Tensor. ORT will take ownership of the memory.
|
|
4939
|
+
* \param[in] p_data_len Length of the memory in bytes.
|
|
4940
|
+
* \param[in] shape Dimensions of the Tensor. All values should be > 0.
|
|
4941
|
+
* \param[in] shape_len Number of dimensions in the shape array.
|
|
4942
|
+
* \param[in] type Data type of the Tensor.
|
|
4943
|
+
* \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue
|
|
4944
|
+
*
|
|
4945
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4946
|
+
*
|
|
4947
|
+
* \since Version 1.22.
|
|
4948
|
+
*/
|
|
4949
|
+
ORT_API2_STATUS(CreateTensorWithDataAndDeleterAsOrtValue, _In_ OrtAllocator* deleter,
|
|
4950
|
+
_In_ void* p_data, size_t p_data_len,
|
|
4951
|
+
_In_ const int64_t* shape, size_t shape_len,
|
|
4952
|
+
ONNXTensorElementDataType type,
|
|
4953
|
+
_Outptr_ OrtValue** out);
|
|
4954
|
+
|
|
4955
|
+
/** \brief sets load cancellation flag to abort session loading process.
|
|
4956
|
+
*
|
|
4957
|
+
* \param[in] options instance that was passed to the session at creation time.
|
|
4958
|
+
* \param[in] cancel setting this to true after model loading process was initiated will
|
|
4959
|
+
* attempt to cancel the loading process. If cancellation is successful, CreateSession()
|
|
4960
|
+
* CreateSessionFromArray() or any other session creation API that take session options as an
|
|
4961
|
+
* argument will return an OrtStatus indicating that session loading was canceled at user request,
|
|
4962
|
+
* error code ORT_MODEL_LOAD_CANCELED.
|
|
4963
|
+
* The APIs above would not return any valid Session instance. This is the best case effort and the result
|
|
4964
|
+
* is not guaranteed. The session may have already been created and initialized
|
|
4965
|
+
* before the cancellation request was issued.
|
|
4966
|
+
*
|
|
4967
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
4968
|
+
*
|
|
4969
|
+
* \since Version 1.22.
|
|
4970
|
+
*/
|
|
4971
|
+
ORT_API2_STATUS(SessionOptionsSetLoadCancellationFlag, _Inout_ OrtSessionOptions* options,
|
|
4972
|
+
_In_ bool cancel);
|
|
4973
|
+
|
|
4974
|
+
/** \brief Get the Compile API instance.
|
|
4975
|
+
*
|
|
4976
|
+
* Get the Compile API instance to compile ONNX models. Execution providers that support compilation fuse a subgraph
|
|
4977
|
+
* into an EPContext node that wraps a provider-specific binary representation of the subgraph.
|
|
4978
|
+
* For more details about the EPContext design, refer to:
|
|
4979
|
+
* \htmlonly
|
|
4980
|
+
* <a href="https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html">EPContext design document.</a>
|
|
4981
|
+
* \endhtmlonly
|
|
4982
|
+
*
|
|
4983
|
+
* \return Compile API struct instance.
|
|
4984
|
+
*
|
|
4985
|
+
* \since Version 1.22.
|
|
4986
|
+
*/
|
|
4987
|
+
const OrtCompileApi*(ORT_API_CALL* GetCompileApi)();
|
|
4988
|
+
|
|
4989
|
+
//
|
|
4990
|
+
// OrtKeyValuePairs
|
|
4991
|
+
//
|
|
4992
|
+
|
|
4993
|
+
/** \brief Create an OrtKeyValuePairs instance.
|
|
4994
|
+
*
|
|
4995
|
+
* \param[out] out A pointer to a newly created OrtKeyValuePairs instance.
|
|
4996
|
+
*
|
|
4997
|
+
* \note Must be released by calling ReleaseKeyValuePairs.
|
|
4998
|
+
*
|
|
4999
|
+
* \since Version 1.22.
|
|
5000
|
+
*/
|
|
5001
|
+
void(ORT_API_CALL* CreateKeyValuePairs)(_Outptr_ OrtKeyValuePairs** out);
|
|
5002
|
+
|
|
5003
|
+
/** \brief Add a key-value pair to the OrtKeyValuePairs instance.
|
|
5004
|
+
*
|
|
5005
|
+
* \param[in] kvps OrtKeyValuePairs instance.
|
|
5006
|
+
* \param[in] key Key to be added.
|
|
5007
|
+
* \param[in] value Value to be added.
|
|
5008
|
+
*
|
|
5009
|
+
* \note The `key` and `value` are copied internally.
|
|
5010
|
+
*
|
|
5011
|
+
* \since Version 1.22.
|
|
5012
|
+
*/
|
|
5013
|
+
|
|
5014
|
+
void(ORT_API_CALL* AddKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key, _In_ const char* value);
|
|
5015
|
+
|
|
5016
|
+
/** \brief Get the value associated with a key in the OrtKeyValuePairs instance.
|
|
5017
|
+
*
|
|
5018
|
+
* \param[in] kvps OrtKeyValuePairs instance.
|
|
5019
|
+
* \param[in] key Key to be searched.
|
|
5020
|
+
*
|
|
5021
|
+
* \return The value associated with the key, or nullptr if the key does not exist.
|
|
5022
|
+
*
|
|
5023
|
+
* \since Version 1.22.
|
|
5024
|
+
*/
|
|
5025
|
+
const char*(ORT_API_CALL* GetKeyValue)(_In_ const OrtKeyValuePairs* kvps, _In_ const char* key);
|
|
5026
|
+
|
|
5027
|
+
/** \brief Get all the key-value pairs from the OrtKeyValuePairs instance.
|
|
5028
|
+
*
|
|
5029
|
+
* \param[in] kvps OrtKeyValuePairs instance.
|
|
5030
|
+
* \param[out] keys Array of keys from `kvps`.
|
|
5031
|
+
* \param[out] values Array of values from `kvps`.
|
|
5032
|
+
* \param[out] num_entries Number of entries in `keys` and `values`.
|
|
5033
|
+
*
|
|
5034
|
+
* \since Version 1.22.
|
|
5035
|
+
*/
|
|
5036
|
+
void(ORT_API_CALL* GetKeyValuePairs)(_In_ const OrtKeyValuePairs* kvps,
|
|
5037
|
+
_Outptr_ const char* const** keys, _Outptr_ const char* const** values,
|
|
5038
|
+
_Out_ size_t* num_entries);
|
|
5039
|
+
|
|
5040
|
+
/** \brief Remove a key-value pair from the OrtKeyValuePairs instance.
|
|
5041
|
+
*
|
|
5042
|
+
* \param[in] kvps OrtKeyValuePairs instance.
|
|
5043
|
+
* \param[in] key Key to be removed. No error if not found.
|
|
5044
|
+
*
|
|
5045
|
+
* \since Version 1.22.
|
|
5046
|
+
*/
|
|
5047
|
+
void(ORT_API_CALL* RemoveKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key);
|
|
5048
|
+
|
|
5049
|
+
/** \brief Release an OrtKeyValuePairs instance.
|
|
5050
|
+
*
|
|
5051
|
+
* \param[in] input OrtKeyValuePairs instance to be released.
|
|
5052
|
+
*
|
|
5053
|
+
* \since Version 1.22.
|
|
5054
|
+
*/
|
|
5055
|
+
ORT_CLASS_RELEASE(KeyValuePairs);
|
|
5056
|
+
|
|
5057
|
+
/** \brief Register an execution provider library with ORT.
|
|
5058
|
+
*
|
|
5059
|
+
* The library must export 'CreateEpFactories' and 'ReleaseEpFactory' functions.
|
|
5060
|
+
* See OrtEpApi for more details.
|
|
5061
|
+
*
|
|
5062
|
+
* \param[in] env The OrtEnv instance to register the library in.
|
|
5063
|
+
* \param[in] registration_name The name to register the execution provider library under.
|
|
5064
|
+
* \param[in] path The path to the execution provider library.
|
|
5065
|
+
*
|
|
5066
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5067
|
+
*
|
|
5068
|
+
* \since Version 1.22.
|
|
5069
|
+
*/
|
|
5070
|
+
ORT_API2_STATUS(RegisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name,
|
|
5071
|
+
_In_ const ORTCHAR_T* path);
|
|
5072
|
+
|
|
5073
|
+
/** \brief Unregister an execution provider library with ORT.
|
|
5074
|
+
*
|
|
5075
|
+
* ORT will call ReleaseEpFactory for all factories created by the library, and unload the library.
|
|
5076
|
+
*
|
|
5077
|
+
* You <b>MUST</b> ensure there are no Session instances using execution providers created by the library
|
|
5078
|
+
* before calling this function.
|
|
5079
|
+
*
|
|
5080
|
+
* \param[in] env The OrtEnv instance to unregister the library from.
|
|
5081
|
+
* \param[in] registration_name The name the execution provider library was registered under.
|
|
5082
|
+
*
|
|
5083
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5084
|
+
*
|
|
5085
|
+
* \since Version 1.22.
|
|
5086
|
+
*/
|
|
5087
|
+
ORT_API2_STATUS(UnregisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name);
|
|
5088
|
+
|
|
5089
|
+
/** \brief Get the list of available OrtEpDevice instances.
|
|
5090
|
+
*
|
|
5091
|
+
* Each OrtEpDevice instance contains details of the execution provider and the device it will use.
|
|
5092
|
+
*
|
|
5093
|
+
* \param[in] env The OrtEnv instance to query.
|
|
5094
|
+
* \param[out] ep_devices The OrtEpDevice instances that the execution provider will use.
|
|
5095
|
+
* \param[out] num_ep_devices The number of OrtEpDevice instances returned.
|
|
5096
|
+
*
|
|
5097
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5098
|
+
*
|
|
5099
|
+
* \since Version 1.22.
|
|
5100
|
+
*/
|
|
5101
|
+
ORT_API2_STATUS(GetEpDevices, _In_ const OrtEnv* env,
|
|
5102
|
+
_Outptr_ const OrtEpDevice* const** ep_devices, _Out_ size_t* num_ep_devices);
|
|
5103
|
+
|
|
5104
|
+
/** \brief Append the execution provider that is responsible for the selected OrtEpDevice instances
|
|
5105
|
+
* to the session options.
|
|
5106
|
+
*
|
|
5107
|
+
* \param[in] session_options Session options to add execution provider to.
|
|
5108
|
+
* \param[in] env Environment that execution providers were registered with.
|
|
5109
|
+
* \param[in] ep_devices One or more OrtEpDevice instances to create an execution provider for.
|
|
5110
|
+
* Obtain from GetEpDevices. All OrtEpDevice instances must be from the same execution
|
|
5111
|
+
* provider. It is only necessary to provide multiple OrtEpDevices if you want to use the
|
|
5112
|
+
* same execution provider for multiple devices.
|
|
5113
|
+
* e.g. the EP is capable of running on GPU and NPU.
|
|
5114
|
+
* \param[in] num_ep_devices Number of OrtEpDevice instances.
|
|
5115
|
+
* \param[in] ep_option_keys Optional keys to configure the execution provider.
|
|
5116
|
+
* \param[in] ep_option_vals Optional values to configure the execution provider.
|
|
5117
|
+
* \param[in] num_ep_options Number of execution provide options to add.
|
|
5118
|
+
*
|
|
5119
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5120
|
+
*
|
|
5121
|
+
* \since Version 1.22.
|
|
5122
|
+
*/
|
|
5123
|
+
ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_V2, _In_ OrtSessionOptions* session_options,
|
|
5124
|
+
_In_ OrtEnv* env,
|
|
5125
|
+
_In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, _In_ size_t num_ep_devices,
|
|
5126
|
+
_In_reads_(num_op_options) const char* const* ep_option_keys,
|
|
5127
|
+
_In_reads_(num_op_options) const char* const* ep_option_vals,
|
|
5128
|
+
size_t num_ep_options);
|
|
5129
|
+
|
|
5130
|
+
/** \brief Set the execution provider selection policy for the session.
|
|
5131
|
+
*
|
|
5132
|
+
* Allows users to specify a device selection policy for automatic execution provider (EP) selection.
|
|
5133
|
+
* If custom selection is required please use SessionOptionsSetEpSelectionPolicyDelegate instead.
|
|
5134
|
+
*
|
|
5135
|
+
* \param[in] session_options The OrtSessionOptions instance.
|
|
5136
|
+
* \param[in] policy The device selection policy to use (see OrtExecutionProviderDevicePolicy).
|
|
5137
|
+
*
|
|
5138
|
+
* \since Version 1.22
|
|
5139
|
+
*/
|
|
5140
|
+
ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicy, _In_ OrtSessionOptions* session_options,
|
|
5141
|
+
_In_ OrtExecutionProviderDevicePolicy policy);
|
|
5142
|
+
|
|
5143
|
+
/** \brief Set the execution provider selection policy delegate for the session.
|
|
5144
|
+
*
|
|
5145
|
+
* Allows users to provide a custom device selection policy for automatic execution provider (EP) selection.
|
|
5146
|
+
*
|
|
5147
|
+
* \param[in] session_options The OrtSessionOptions instance.
|
|
5148
|
+
* \param[in] delegate Delegate callback for custom selection.
|
|
5149
|
+
* \param[in] delegate_state Optional state that will be passed to the delegate callback. nullptr if not required.
|
|
5150
|
+
*
|
|
5151
|
+
* \since Version 1.22
|
|
5152
|
+
*/
|
|
5153
|
+
ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicyDelegate, _In_ OrtSessionOptions* session_options,
|
|
5154
|
+
_In_ EpSelectionDelegate delegate,
|
|
5155
|
+
_In_opt_ void* delegate_state);
|
|
5156
|
+
|
|
5157
|
+
/** \brief Get the hardware device type.
|
|
5158
|
+
*
|
|
5159
|
+
* \param[in] device The OrtHardwareDevice instance to query.
|
|
5160
|
+
* \return The hardware device type.
|
|
5161
|
+
*
|
|
5162
|
+
* \since Version 1.22.
|
|
5163
|
+
*/
|
|
5164
|
+
OrtHardwareDeviceType(ORT_API_CALL* HardwareDevice_Type)(_In_ const OrtHardwareDevice* device);
|
|
5165
|
+
|
|
5166
|
+
/** \brief Get the hardware device's vendor identifier.
|
|
5167
|
+
*
|
|
5168
|
+
* \param[in] device The OrtHardwareDevice instance to query.
|
|
5169
|
+
* \return The hardware device vendor identifier.
|
|
5170
|
+
*
|
|
5171
|
+
* \since Version 1.22.
|
|
5172
|
+
*/
|
|
5173
|
+
uint32_t(ORT_API_CALL* HardwareDevice_VendorId)(_In_ const OrtHardwareDevice* device);
|
|
5174
|
+
|
|
5175
|
+
/** \brief Get the hardware device's vendor name.
|
|
5176
|
+
*
|
|
5177
|
+
* \param[in] device The OrtHardwareDevice instance to query.
|
|
5178
|
+
* \return The hardware device's vendor name.
|
|
5179
|
+
*
|
|
5180
|
+
* \since Version 1.22.
|
|
5181
|
+
*/
|
|
5182
|
+
const char*(ORT_API_CALL* HardwareDevice_Vendor)(_In_ const OrtHardwareDevice* device);
|
|
5183
|
+
|
|
5184
|
+
/** \brief Get the hardware device's unique identifier.
|
|
5185
|
+
*
|
|
5186
|
+
* \param[in] device The OrtHardwareDevice instance to query.
|
|
5187
|
+
* \return The device id.
|
|
5188
|
+
*
|
|
5189
|
+
* \note This is not a unique identifier. It identifies the hardware type when combined with vendor id.
|
|
5190
|
+
* \since Version 1.22.
|
|
5191
|
+
*/
|
|
5192
|
+
uint32_t(ORT_API_CALL* HardwareDevice_DeviceId)(_In_ const OrtHardwareDevice* device);
|
|
5193
|
+
|
|
5194
|
+
/** \brief Get hardware device metadata.
|
|
5195
|
+
*
|
|
5196
|
+
* \param[in] device The OrtHardwareDevice instance to query.
|
|
5197
|
+
* \return An OrtKeyValuePairs instance containing the metadata for the device.
|
|
5198
|
+
* Note: ORT owns the instance so the user must not call ReleaseKeyValuePairs with it.
|
|
5199
|
+
*
|
|
5200
|
+
* \since Version 1.22.
|
|
5201
|
+
*/
|
|
5202
|
+
const OrtKeyValuePairs*(ORT_API_CALL* HardwareDevice_Metadata)(_In_ const OrtHardwareDevice* device);
|
|
5203
|
+
|
|
5204
|
+
/** \brief Get the execution provider name.
|
|
5205
|
+
*
|
|
5206
|
+
* \param[in] ep_device The OrtEpDevice instance to query.
|
|
5207
|
+
* \return The execution provider name.
|
|
5208
|
+
*
|
|
5209
|
+
* \since Version 1.22.
|
|
5210
|
+
*/
|
|
5211
|
+
const char*(ORT_API_CALL* EpDevice_EpName)(_In_ const OrtEpDevice* ep_device);
|
|
5212
|
+
|
|
5213
|
+
/** \brief Get the execution provider's vendor name.
|
|
5214
|
+
*
|
|
5215
|
+
* \param[in] ep_device The OrtEpDevice instance to query.
|
|
5216
|
+
* \return The execution provider's vendor name.
|
|
5217
|
+
*
|
|
5218
|
+
* \since Version 1.22.
|
|
5219
|
+
*/
|
|
5220
|
+
const char*(ORT_API_CALL* EpDevice_EpVendor)(_In_ const OrtEpDevice* ep_device);
|
|
5221
|
+
|
|
5222
|
+
/** \brief Get the metadata for the OrtEpDevice.
|
|
5223
|
+
*
|
|
5224
|
+
* \param[in] ep_device The OrtEpDevice instance to query.
|
|
5225
|
+
* \return An OrtKeyValuePairs instance containing the metadata for the device.
|
|
5226
|
+
*
|
|
5227
|
+
* \since Version 1.22.
|
|
5228
|
+
*/
|
|
5229
|
+
const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpMetadata)(_In_ const OrtEpDevice* ep_device);
|
|
5230
|
+
|
|
5231
|
+
/** \brief Get the execution provider options for the OrtEpDevice.
|
|
5232
|
+
*
|
|
5233
|
+
* \param[in] ep_device The OrtEpDevice instance to query.
|
|
5234
|
+
* \return An OrtKeyValuePairs instance containing the execution provider options for the device.
|
|
5235
|
+
*
|
|
5236
|
+
* \since Version 1.22.
|
|
5237
|
+
*/
|
|
5238
|
+
const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpOptions)(_In_ const OrtEpDevice* ep_device);
|
|
5239
|
+
|
|
5240
|
+
/** \brief Get the OrtHardwareDevice instance for the OrtEpDevice.
|
|
5241
|
+
*
|
|
5242
|
+
* \param[in] ep_device The OrtEpDevice instance to query.
|
|
5243
|
+
* \return The OrtHardwareDevice instance for the device.
|
|
5244
|
+
*
|
|
5245
|
+
* \since Version 1.22.
|
|
5246
|
+
*/
|
|
5247
|
+
const OrtHardwareDevice*(ORT_API_CALL* EpDevice_Device)(_In_ const OrtEpDevice* ep_device);
|
|
5248
|
+
|
|
5249
|
+
/** \brief Get the OrtEpApi instance for implementing an execution provider.
|
|
5250
|
+
*
|
|
5251
|
+
* \since Version 1.22.
|
|
5252
|
+
*/
|
|
5253
|
+
const OrtEpApi*(ORT_API_CALL* GetEpApi)();
|
|
4789
5254
|
};
|
|
4790
5255
|
|
|
4791
5256
|
/*
|
|
@@ -4900,6 +5365,784 @@ struct OrtCustomOp {
|
|
|
4900
5365
|
void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index);
|
|
4901
5366
|
};
|
|
4902
5367
|
|
|
5368
|
+
/**
|
|
5369
|
+
* ORT Model Editor API
|
|
5370
|
+
*/
|
|
5371
|
+
|
|
5372
|
+
/**
|
|
5373
|
+
* \brief The OrtModelEditorApi struct provides functions to create or edit an ONNX model.
|
|
5374
|
+
*
|
|
5375
|
+
* See onnxruntime/test/shared_lib/test_model_editor_api.cc for example usage.
|
|
5376
|
+
*
|
|
5377
|
+
* \since Version 1.22.
|
|
5378
|
+
*/
|
|
5379
|
+
struct OrtModelEditorApi {
|
|
5380
|
+
// Model building/editing requires a full build. We return nullptr from GetModelEditorApi if this is a minimal
|
|
5381
|
+
// build, so it doesn't matter if there are no function pointers in this struct as a user will never get an
|
|
5382
|
+
// OrtModelEditorApi instance. We do however need a dummy field to avoid empty struct warning.
|
|
5383
|
+
#if defined(ORT_MINIMAL_BUILD)
|
|
5384
|
+
const bool not_defined_in_this_build;
|
|
5385
|
+
#else
|
|
5386
|
+
/** \brief Create an OrtTypeInfo instance for a Tensor.
|
|
5387
|
+
*
|
|
5388
|
+
* Create an OrtTypeInfo instance for a Tensor to use as graph inputs/outputs with the Model Editor API.
|
|
5389
|
+
*
|
|
5390
|
+
* User can release `tensor_info` after creating the OrtTypeInfo.
|
|
5391
|
+
*
|
|
5392
|
+
* \param[in] tensor_info Tensor type and shape information.
|
|
5393
|
+
* \param[out] type_info TypeInfo instance for the tensor.
|
|
5394
|
+
*
|
|
5395
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5396
|
+
*
|
|
5397
|
+
* \since Version 1.22.
|
|
5398
|
+
*/
|
|
5399
|
+
ORT_API2_STATUS(CreateTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info,
|
|
5400
|
+
_Outptr_ OrtTypeInfo** type_info);
|
|
5401
|
+
|
|
5402
|
+
/** \brief Create an OrtTypeInfo instance for a SparseTensor.
|
|
5403
|
+
*
|
|
5404
|
+
* Create an OrtTypeInfo instance for a SparseTensor to use as graph inputs/outputs with the Model Editor API.
|
|
5405
|
+
*
|
|
5406
|
+
* User can release `tensor_info` after creating the OrtTypeInfo.
|
|
5407
|
+
*
|
|
5408
|
+
* \param[in] tensor_info SparseTensor type and shape information.
|
|
5409
|
+
* \param[out] type_info TypeInfo instance for the tensor.
|
|
5410
|
+
*
|
|
5411
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5412
|
+
*
|
|
5413
|
+
* \since Version 1.22.
|
|
5414
|
+
*/
|
|
5415
|
+
ORT_API2_STATUS(CreateSparseTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info,
|
|
5416
|
+
_Outptr_ OrtTypeInfo** type_info);
|
|
5417
|
+
|
|
5418
|
+
/** \brief Create an OrtTypeInfo instance for a Map.
|
|
5419
|
+
*
|
|
5420
|
+
* Create an OrtTypeInfo instance for a Map to use as graph inputs/outputs with the Model Editor API.
|
|
5421
|
+
*
|
|
5422
|
+
* User can release `map_value_type` after creating the OrtTypeInfo.
|
|
5423
|
+
*
|
|
5424
|
+
* \param[in] map_key_type Key type for the map.
|
|
5425
|
+
* \param[in] map_value_type Value type for the map.
|
|
5426
|
+
* \param[out] type_info TypeInfo instance for the map.
|
|
5427
|
+
*
|
|
5428
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5429
|
+
*
|
|
5430
|
+
* \since Version 1.22.
|
|
5431
|
+
*/
|
|
5432
|
+
ORT_API2_STATUS(CreateMapTypeInfo, ONNXTensorElementDataType map_key_type, _In_ const OrtTypeInfo* map_value_type,
|
|
5433
|
+
_Outptr_ OrtTypeInfo** type_info);
|
|
5434
|
+
|
|
5435
|
+
/** \brief Create an OrtTypeInfo instance for a Sequence.
|
|
5436
|
+
*
|
|
5437
|
+
* Create an OrtTypeInfo instance for a Sequence to use as graph inputs/outputs with the Model Editor API.
|
|
5438
|
+
*
|
|
5439
|
+
* User can release `sequence_type` after creating the OrtTypeInfo.
|
|
5440
|
+
*
|
|
5441
|
+
* \param[in] sequence_type Sequence type and shape information.
|
|
5442
|
+
* \param[out] type_info TypeInfo instance for the sequence.
|
|
5443
|
+
*
|
|
5444
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5445
|
+
*
|
|
5446
|
+
* \since Version 1.22.
|
|
5447
|
+
*/
|
|
5448
|
+
ORT_API2_STATUS(CreateSequenceTypeInfo, _In_ const OrtTypeInfo* sequence_type, _Outptr_ OrtTypeInfo** type_info);
|
|
5449
|
+
|
|
5450
|
+
/** \brief Create an OrtTypeInfo instance for an Optional.
|
|
5451
|
+
*
|
|
5452
|
+
* Create an OrtTypeInfo instance for an Optional to use as graph inputs/outputs with the Model Editor API.
|
|
5453
|
+
*
|
|
5454
|
+
* User can release `contained_type` after creating the OrtTypeInfo.
|
|
5455
|
+
*
|
|
5456
|
+
* \param[in] contained_type Tensor type and shape information.
|
|
5457
|
+
* \param[out] type_info TypeInfo instance for the tensor.
|
|
5458
|
+
*
|
|
5459
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5460
|
+
*
|
|
5461
|
+
* \since Version 1.22.
|
|
5462
|
+
*/
|
|
5463
|
+
ORT_API2_STATUS(CreateOptionalTypeInfo, _In_ const OrtTypeInfo* contained_type, _Outptr_ OrtTypeInfo** type_info);
|
|
5464
|
+
|
|
5465
|
+
/** \brief Create an OrtValueInfo for use as an OrtGraph input or output.
|
|
5466
|
+
*
|
|
5467
|
+
* \param[in] name The name of the input or output.
|
|
5468
|
+
* \param[in] type_info The type information for the input or output. The provided value is copied.
|
|
5469
|
+
* \param[out] value_info The OrtValueInfo instance.
|
|
5470
|
+
*
|
|
5471
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5472
|
+
*
|
|
5473
|
+
* \since Version 1.22.
|
|
5474
|
+
*/
|
|
5475
|
+
ORT_API2_STATUS(CreateValueInfo, _In_ const char* name, _In_ const OrtTypeInfo* type_info,
|
|
5476
|
+
_Outptr_ OrtValueInfo** value_info);
|
|
5477
|
+
|
|
5478
|
+
/** \brief Create an OrtNode to add to an OrtGraph.
|
|
5479
|
+
*
|
|
5480
|
+
* Create an OrtNode.
|
|
5481
|
+
*
|
|
5482
|
+
* Create attributes with CreateOpAttr. OrtOpAttr instances are copied.
|
|
5483
|
+
*
|
|
5484
|
+
* \param[in] operator_name The name of the operator.
|
|
5485
|
+
* \param[in] domain_name The domain of the operator. Use an empty string for ONNX operators.
|
|
5486
|
+
* \param[in] node_name The name of the node.
|
|
5487
|
+
* \param[in] input_names The names of the inputs.
|
|
5488
|
+
* \param[in] input_names_len The number of input names.
|
|
5489
|
+
* \param[in] output_names The names of the outputs.
|
|
5490
|
+
* \param[in] output_names_len The number of output names.
|
|
5491
|
+
* \param[in] attributes The optional attributes of the node.
|
|
5492
|
+
* \param[in] attribs_len The number of attributes. May be zero.
|
|
5493
|
+
* \param[out] node The OrtNode instance.
|
|
5494
|
+
*
|
|
5495
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5496
|
+
*
|
|
5497
|
+
* \since Version 1.22.
|
|
5498
|
+
*/
|
|
5499
|
+
ORT_API2_STATUS(CreateNode, _In_ const char* operator_name, _In_ const char* domain_name, _In_ const char* node_name,
|
|
5500
|
+
_In_reads_(input_names_len) const char* const* input_names, size_t input_names_len,
|
|
5501
|
+
_In_reads_(output_names_len) const char* const* output_names, size_t output_names_len,
|
|
5502
|
+
_In_reads_(attribs_len) _In_opt_ OrtOpAttr** attributes, _In_ size_t attribs_len,
|
|
5503
|
+
_Outptr_ OrtNode** node);
|
|
5504
|
+
|
|
5505
|
+
/** \brief Create an OrtGraph
|
|
5506
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5507
|
+
* \since Version 1.22.
|
|
5508
|
+
*/
|
|
5509
|
+
ORT_API2_STATUS(CreateGraph, _Outptr_ OrtGraph** graph);
|
|
5510
|
+
|
|
5511
|
+
/** \brief Set the inputs for the OrtGraph.
|
|
5512
|
+
*
|
|
5513
|
+
* Set the graph inputs. This will replace any existing inputs with the new values.
|
|
5514
|
+
* The OrtGraph takes ownership of the OrtValueInfo instances and you should NOT call ReleaseOrtValueInfo.
|
|
5515
|
+
*
|
|
5516
|
+
* \param[in] graph The OrtGraph instance to update.
|
|
5517
|
+
* \param[in] inputs The input OrtValueInfo instances.
|
|
5518
|
+
* \param[in] inputs_len The number of input OrtValueInfo instances.
|
|
5519
|
+
*
|
|
5520
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5521
|
+
*
|
|
5522
|
+
* \since Version 1.22.
|
|
5523
|
+
*/
|
|
5524
|
+
ORT_API2_STATUS(SetGraphInputs, _Inout_ OrtGraph* graph,
|
|
5525
|
+
_In_reads_(inputs_len) _In_ OrtValueInfo** inputs, _In_ size_t inputs_len);
|
|
5526
|
+
|
|
5527
|
+
/** \brief Set the outputs for the OrtGraph.
|
|
5528
|
+
*
|
|
5529
|
+
* Set the graph outputs. This will replace any existing outputs with the new values.
|
|
5530
|
+
* The OrtGraph takes ownership of the OrtValueInfo instances provided and you should NOT call ReleaseOrtValueInfo.
|
|
5531
|
+
*
|
|
5532
|
+
* \param[in] graph The OrtGraph instance to update.
|
|
5533
|
+
* \param[in] outputs The output OrtValueInfo instances.
|
|
5534
|
+
* \param[in] outputs_len The number of output OrtValueInfo instances.
|
|
5535
|
+
*
|
|
5536
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5537
|
+
*
|
|
5538
|
+
* \since Version 1.22.
|
|
5539
|
+
*/
|
|
5540
|
+
ORT_API2_STATUS(SetGraphOutputs, _Inout_ OrtGraph* graph,
|
|
5541
|
+
_In_reads_(outputs_len) _In_ OrtValueInfo** outputs, _In_ size_t outputs_len);
|
|
5542
|
+
|
|
5543
|
+
/** \brief Add an initializer to the OrtGraph
|
|
5544
|
+
*
|
|
5545
|
+
* ORT will take ownership of the OrtValue and you should NOT call ReleaseOrtValue.
|
|
5546
|
+
*
|
|
5547
|
+
* Two options:
|
|
5548
|
+
*
|
|
5549
|
+
* Allocated memory:
|
|
5550
|
+
* Use CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data.
|
|
5551
|
+
* Set `data_is_external` to false.
|
|
5552
|
+
*
|
|
5553
|
+
* Pre-existing memory:
|
|
5554
|
+
* Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue
|
|
5555
|
+
* with a tensor that contains a pointer to the existing data.
|
|
5556
|
+
* Set `data_is_external` to true.
|
|
5557
|
+
*
|
|
5558
|
+
* The pointer must remain valid for the duration of the inference session.
|
|
5559
|
+
* If using CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session
|
|
5560
|
+
* is released.
|
|
5561
|
+
* If using CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as
|
|
5562
|
+
* soon as the OrtValue is no longer in use.
|
|
5563
|
+
*
|
|
5564
|
+
* NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more.
|
|
5565
|
+
* For smaller tensors use CreateTensorAsOrtValue.
|
|
5566
|
+
*
|
|
5567
|
+
* ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is
|
|
5568
|
+
* typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of
|
|
5569
|
+
* memory, so this limit acts as a simple catch-all rule to avoid issues.
|
|
5570
|
+
* e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`.
|
|
5571
|
+
*
|
|
5572
|
+
* \param[in] graph The OrtGraph instance to update.
|
|
5573
|
+
* \param[in] name The value name for the initializer.
|
|
5574
|
+
* \param[in] tensor The OrtValue instance containing the tensor data.
|
|
5575
|
+
* \param[in] data_is_external Set to true if the data is external and should not be copied.
|
|
5576
|
+
*
|
|
5577
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5578
|
+
*
|
|
5579
|
+
* \since Version 1.22.
|
|
5580
|
+
*/
|
|
5581
|
+
ORT_API2_STATUS(AddInitializerToGraph, _Inout_ OrtGraph* graph, _In_ const char* name, _In_ OrtValue* tensor,
|
|
5582
|
+
bool data_is_external);
|
|
5583
|
+
|
|
5584
|
+
/** \brief Add an OrtNode to an OrtGraph
|
|
5585
|
+
*
|
|
5586
|
+
* Add the node to the graph. The OrtGraph will take ownership of OrtNode and you should NOT call ReleaseOrtNode.
|
|
5587
|
+
*
|
|
5588
|
+
* \param[in] graph The OrtGraph instance to update.
|
|
5589
|
+
* \param[in] node The OrtNode instance to add to the graph.
|
|
5590
|
+
*
|
|
5591
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5592
|
+
*
|
|
5593
|
+
* \since Version 1.22.
|
|
5594
|
+
*/
|
|
5595
|
+
ORT_API2_STATUS(AddNodeToGraph, _Inout_ OrtGraph* graph, _In_ OrtNode* node);
|
|
5596
|
+
|
|
5597
|
+
/** \brief Create an OrtModel.
|
|
5598
|
+
*
|
|
5599
|
+
* Create an OrtModel.
|
|
5600
|
+
*
|
|
5601
|
+
* This can be used to build a new model, or to augment an existing model.
|
|
5602
|
+
*
|
|
5603
|
+
* \param[in] domain_names The domain names for the model.
|
|
5604
|
+
* If augmenting an existing model add additional domains if needed.
|
|
5605
|
+
* \param[in] opset_versions The opset versions for the model.
|
|
5606
|
+
* If augmenting an existing model add additional opset versions if needed.
|
|
5607
|
+
* \param[in] opset_entries_len The number of domain_names and opset_versions entries.
|
|
5608
|
+
* Domain and opset entries should be 1:1
|
|
5609
|
+
* \param[out] model The OrtModel instance.
|
|
5610
|
+
*
|
|
5611
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5612
|
+
*
|
|
5613
|
+
* \since Version 1.22.
|
|
5614
|
+
*/
|
|
5615
|
+
ORT_API2_STATUS(CreateModel,
|
|
5616
|
+
_In_reads_(opset_entries_len) const char* const* domain_names,
|
|
5617
|
+
_In_reads_(opset_entries_len) const int* opset_versions,
|
|
5618
|
+
size_t opset_entries_len,
|
|
5619
|
+
_Outptr_ OrtModel** model);
|
|
5620
|
+
|
|
5621
|
+
/** \brief Add an OrtGraph to an OrtModel.
|
|
5622
|
+
*
|
|
5623
|
+
* Add the graph to a model. This should be called once when creating a new model.
|
|
5624
|
+
*
|
|
5625
|
+
* The OrtModel takes ownership of the OrtGraph and you should NOT call ReleaseOrtGraph.
|
|
5626
|
+
*
|
|
5627
|
+
* \param[in] model The OrtModel instance to update.
|
|
5628
|
+
* \param[in] graph The OrtGraph instance to add to the model.
|
|
5629
|
+
*
|
|
5630
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5631
|
+
*
|
|
5632
|
+
* \since Version 1.22.
|
|
5633
|
+
*/
|
|
5634
|
+
ORT_API2_STATUS(AddGraphToModel, _Inout_ OrtModel* model, _In_ OrtGraph* graph);
|
|
5635
|
+
|
|
5636
|
+
/** \brief Create an OrtSession using the OrtModel.
|
|
5637
|
+
*
|
|
5638
|
+
* Create an inference session using the OrtModel instance.
|
|
5639
|
+
* The OrtModel should have been populated with an OrtGraph containing nodes and initializers, and SetGraphInputs
|
|
5640
|
+
* and SetGraphOutputs must have been called.
|
|
5641
|
+
* This will validate the model, run optimizers, and prepare the session for inferencing.
|
|
5642
|
+
*
|
|
5643
|
+
* ReleaseOrtModel must be called to free the OrtModel after session creation.
|
|
5644
|
+
*
|
|
5645
|
+
* \param[in] env The OrtEnv instance.
|
|
5646
|
+
* \param[in] model The OrtModel instance.
|
|
5647
|
+
* \param[in] options The OrtSessionOptions instance.
|
|
5648
|
+
* \param[out] out The OrtSession instance.
|
|
5649
|
+
*
|
|
5650
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5651
|
+
*
|
|
5652
|
+
* \since Version 1.22.
|
|
5653
|
+
*/
|
|
5654
|
+
ORT_API2_STATUS(CreateSessionFromModel, _In_ const OrtEnv* env, _In_ const OrtModel* model,
|
|
5655
|
+
_In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
|
|
5656
|
+
|
|
5657
|
+
/** \brief Create an OrtSession to augment an existing model.
|
|
5658
|
+
*
|
|
5659
|
+
* Create an OrtSession with an existing model that will be augmented with additional nodes and initializers.
|
|
5660
|
+
* Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the
|
|
5661
|
+
* model is finalized.
|
|
5662
|
+
*
|
|
5663
|
+
* To add nodes and initializers to the existing model, first create an OrtModel using CreateModel.
|
|
5664
|
+
* Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph.
|
|
5665
|
+
* Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made
|
|
5666
|
+
* by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes.
|
|
5667
|
+
*
|
|
5668
|
+
* Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the
|
|
5669
|
+
* session for inferencing by calling FinalizeModelEditorSession.
|
|
5670
|
+
*
|
|
5671
|
+
* \param{in} env The OrtEnv instance.
|
|
5672
|
+
* \param{in} model_path The path to the existing ONNX model to augment.
|
|
5673
|
+
* \param{in} options The OrtSessionOptions instance.
|
|
5674
|
+
* \param{out} out The created OrtSession instance.
|
|
5675
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5676
|
+
*
|
|
5677
|
+
* \since Version 1.22.
|
|
5678
|
+
*/
|
|
5679
|
+
ORT_API2_STATUS(CreateModelEditorSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,
|
|
5680
|
+
_In_ const OrtSessionOptions* options,
|
|
5681
|
+
_Outptr_ OrtSession** out);
|
|
5682
|
+
|
|
5683
|
+
/** \brief Create an OrtSession to augment an existing model.
|
|
5684
|
+
*
|
|
5685
|
+
* Create an OrtSession with an existing model that will be augmented with additional nodes and initializers.
|
|
5686
|
+
* Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the
|
|
5687
|
+
* model is finalized.
|
|
5688
|
+
*
|
|
5689
|
+
* To add nodes and initializers to the existing model, first create an OrtModel using CreateModel.
|
|
5690
|
+
* Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph.
|
|
5691
|
+
* Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made
|
|
5692
|
+
* by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes.
|
|
5693
|
+
*
|
|
5694
|
+
* Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the
|
|
5695
|
+
* session for inferencing by calling FinalizeModelEditorSession.
|
|
5696
|
+
*
|
|
5697
|
+
* \param{in} env The OrtEnv instance.
|
|
5698
|
+
* \param{in} model_data The model data for the existing model to augment.
|
|
5699
|
+
* \param{in} model_data_length The length of the model data.
|
|
5700
|
+
* \param{in} options The OrtSessionOptions instance.
|
|
5701
|
+
* \param{out} out The created OrtSession instance.
|
|
5702
|
+
*
|
|
5703
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5704
|
+
*
|
|
5705
|
+
* \since Version 1.22.
|
|
5706
|
+
*/
|
|
5707
|
+
ORT_API2_STATUS(CreateModelEditorSessionFromArray, _In_ const OrtEnv* env,
|
|
5708
|
+
_In_ const void* model_data, size_t model_data_length,
|
|
5709
|
+
_In_ const OrtSessionOptions* options,
|
|
5710
|
+
_Outptr_ OrtSession** out);
|
|
5711
|
+
|
|
5712
|
+
/** \brief Query the session for the opset version of a domain.
|
|
5713
|
+
*
|
|
5714
|
+
* When using the Model Editor API to augment a model, any new nodes must conform to the opset version of the
|
|
5715
|
+
* original model. To do that the user must be able to discover that opset version.
|
|
5716
|
+
* Returns an error if the domain is not used in the model.
|
|
5717
|
+
*
|
|
5718
|
+
* \param[in] session OrtSession to query
|
|
5719
|
+
* \param[in] domain Domain to query. The ONNX domain is an empty string.
|
|
5720
|
+
* \param[out] opset The opset version of the domain.
|
|
5721
|
+
*
|
|
5722
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5723
|
+
*
|
|
5724
|
+
* \since Version 1.22.
|
|
5725
|
+
*/
|
|
5726
|
+
ORT_API2_STATUS(SessionGetOpsetForDomain, _In_ const OrtSession* session, _In_ const char* domain, _Out_ int* opset);
|
|
5727
|
+
|
|
5728
|
+
/** \brief Apply changes to augment the ONNX model in a session created using CreateModelEditorSession[FromArray]
|
|
5729
|
+
*
|
|
5730
|
+
* Adds new nodes and updates graph inputs/outputs using `model` to augment the original ONNX model in the session.
|
|
5731
|
+
* All changes will be validated.
|
|
5732
|
+
* Call FinalizeModelEditorSession to prepare the session for inferencing.
|
|
5733
|
+
*
|
|
5734
|
+
* Existing input/outputs will only be updated if the OrtGraph inputs/outputs are set in the OrtModel.
|
|
5735
|
+
* i.e. you don't need to call SetGraphInputs/SetGraphOutputs if they are unchanged.
|
|
5736
|
+
*
|
|
5737
|
+
* ReleaseOrtModel must be called to free the OrtModel after it is applied to the session.
|
|
5738
|
+
*
|
|
5739
|
+
* \param[in] session OrtSession to update. Session must have been created using CreateModelEditorSession[FromArray].
|
|
5740
|
+
* \param[in] model OrtModel containing new nodes, new initializers, and updated graph input and/or output info.
|
|
5741
|
+
*
|
|
5742
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5743
|
+
*
|
|
5744
|
+
* \since Version 1.22.
|
|
5745
|
+
*/
|
|
5746
|
+
ORT_API2_STATUS(ApplyModelToModelEditorSession, _Inout_ OrtSession* session, _In_ OrtModel* model);
|
|
5747
|
+
|
|
5748
|
+
/** \brief Finalize the Model Editor session that was created using CreateModelEditorSession[FromArray].
|
|
5749
|
+
*
|
|
5750
|
+
* Finalize the Model Editor session that augmented an ONNX model by adding new nodes.
|
|
5751
|
+
* This will run optimizers and prepare the session for inferencing.
|
|
5752
|
+
*
|
|
5753
|
+
* \param[in] session OrtSession to finalize. Session must have been created using CreateModelEditorSession[FromArray].
|
|
5754
|
+
* \param[in] options OrtSessionOptions to use for the session.
|
|
5755
|
+
* \param[in] prepacked_weights_container Optional OrtPrepackedWeightsContainer to use for the session.
|
|
5756
|
+
Set to nullptr if not used.
|
|
5757
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5758
|
+
*
|
|
5759
|
+
* \since Version 1.22.
|
|
5760
|
+
*/
|
|
5761
|
+
ORT_API2_STATUS(FinalizeModelEditorSession, _Inout_ OrtSession* session, _In_ const OrtSessionOptions* options,
|
|
5762
|
+
_In_opt_ OrtPrepackedWeightsContainer* prepacked_weights_container);
|
|
5763
|
+
#endif // !defined(ORT_MINIMAL_BUILD)
|
|
5764
|
+
};
|
|
5765
|
+
|
|
5766
|
+
/**
|
|
5767
|
+
* ORT Compile API
|
|
5768
|
+
*/
|
|
5769
|
+
|
|
5770
|
+
/**
|
|
5771
|
+
* \brief The OrtCompileApi struct provides functions to compile ONNX models.
|
|
5772
|
+
*
|
|
5773
|
+
* Execution providers that support compilation fuse a subgraph into an EPContext node that wraps a provider-specific
|
|
5774
|
+
* binary representation of the subgraph.
|
|
5775
|
+
* For more details about the EPContext design, refer to:
|
|
5776
|
+
* \htmlonly
|
|
5777
|
+
* <a href="https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html">EPContext design document.</a>
|
|
5778
|
+
* \endhtmlonly
|
|
5779
|
+
*
|
|
5780
|
+
* Example (error handling not shown):
|
|
5781
|
+
* OrtStatus* status = NULL;
|
|
5782
|
+
* OrtCompileApi* compile_api = ort_api->GetCompileApi();
|
|
5783
|
+
* OrtModelCompilationOptions* compile_options = NULL;
|
|
5784
|
+
*
|
|
5785
|
+
* status = compile_api->CreateModelCompilationOptionsFromSessionOptions(env, session_options, &compile_options);
|
|
5786
|
+
* status = compile_api->ModelCompilationOptions_SetInputModelPath(compile_options, ORT_TSTR("model.onnx"));
|
|
5787
|
+
* status = compile_api->ModelCompilationOptions_SetOutputModelPath(compile_options, ORT_TSTR("model.compiled.onnx"));
|
|
5788
|
+
* status = compile_api->CompileModel(env, compile_options);
|
|
5789
|
+
* compile_api->ReleaseModelCompilationOptions(compile_options);
|
|
5790
|
+
*
|
|
5791
|
+
* \since Version 1.22.
|
|
5792
|
+
*/
|
|
5793
|
+
struct OrtCompileApi {
|
|
5794
|
+
/// @}
|
|
5795
|
+
/// \name OrtModelCompilationOptions
|
|
5796
|
+
/// @{
|
|
5797
|
+
ORT_CLASS_RELEASE(ModelCompilationOptions);
|
|
5798
|
+
|
|
5799
|
+
/** \brief Creates an OrtModelCompilationOptions object from an existing OrtSessionOptions object.
|
|
5800
|
+
*
|
|
5801
|
+
* An OrtModelCompilationOptions object contains the settings used to generate a compiled ONNX model.
|
|
5802
|
+
* The OrtSessionOptions object has the execution providers with which the model will be compiled.
|
|
5803
|
+
*
|
|
5804
|
+
* ReleaseOrtModelCompilationsOptions must be called to free the OrtModelCompilationOptions after calling
|
|
5805
|
+
* CompileModel.
|
|
5806
|
+
*
|
|
5807
|
+
* \param[in] env OrtEnv object.
|
|
5808
|
+
* \param[in] session_options The OrtSessionOptions instance from which to create the OrtModelCompilationOptions.
|
|
5809
|
+
* \param[out] out The created OrtModelCompilationOptions instance.
|
|
5810
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5811
|
+
*
|
|
5812
|
+
* \since Version 1.22.
|
|
5813
|
+
*/
|
|
5814
|
+
ORT_API2_STATUS(CreateModelCompilationOptionsFromSessionOptions, _In_ const OrtEnv* env,
|
|
5815
|
+
_In_ const OrtSessionOptions* session_options, _Outptr_ OrtModelCompilationOptions** out);
|
|
5816
|
+
|
|
5817
|
+
/** \brief Sets the file path to the input ONNX model to compile.
|
|
5818
|
+
*
|
|
5819
|
+
* The input model's location (e.g., file path or memory buffer) must be set with either
|
|
5820
|
+
* ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer.
|
|
5821
|
+
*
|
|
5822
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5823
|
+
* \param[in] input_model_path Null terminated string of the path (wchar on Windows, char otherwise).
|
|
5824
|
+
*
|
|
5825
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5826
|
+
*
|
|
5827
|
+
* \since Version 1.22.
|
|
5828
|
+
*/
|
|
5829
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetInputModelPath, _In_ OrtModelCompilationOptions* model_compile_options,
|
|
5830
|
+
_In_ const ORTCHAR_T* input_model_path);
|
|
5831
|
+
|
|
5832
|
+
/** \brief Sets the buffer that stores the bytes of the loaded ONNX model to compile.
|
|
5833
|
+
*
|
|
5834
|
+
* The input model's location (e.g., file path or memory buffer) must be set with either
|
|
5835
|
+
* ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer.
|
|
5836
|
+
*
|
|
5837
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5838
|
+
* \param[in] input_model_data Buffer containing the loaded ONNX model bytes.
|
|
5839
|
+
* \param[in] input_model_data_size The number of bytes in the `input_model_data` buffer.
|
|
5840
|
+
*
|
|
5841
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5842
|
+
*
|
|
5843
|
+
* \since Version 1.22.
|
|
5844
|
+
*/
|
|
5845
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetInputModelFromBuffer,
|
|
5846
|
+
_In_ OrtModelCompilationOptions* model_compile_options,
|
|
5847
|
+
_In_ const void* input_model_data,
|
|
5848
|
+
size_t input_model_data_size);
|
|
5849
|
+
|
|
5850
|
+
/** \brief Sets the file path for the output ONNX model generated by CompileModel.
|
|
5851
|
+
*
|
|
5852
|
+
* The output model's location (e.g., file path or memory buffer) can be set with either
|
|
5853
|
+
* ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer.
|
|
5854
|
+
*
|
|
5855
|
+
* If the output model's location is not set, ONNX Runtime will generate an output file with a path based on
|
|
5856
|
+
* the input model's file path. Examples:
|
|
5857
|
+
* /Path/my_model.onnx -> /Path/my_model_ctx.onnx
|
|
5858
|
+
* /Path/my_model -> /Path/my_model_ctx.onnx
|
|
5859
|
+
*
|
|
5860
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5861
|
+
* \param[in] output_model_path Null terminated string of the path (wchar on Windows, char otherwise).
|
|
5862
|
+
*
|
|
5863
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5864
|
+
*
|
|
5865
|
+
* \since Version 1.22.
|
|
5866
|
+
*/
|
|
5867
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelPath, _In_ OrtModelCompilationOptions* model_compile_options,
|
|
5868
|
+
_In_ const ORTCHAR_T* output_model_path);
|
|
5869
|
+
|
|
5870
|
+
/** \brief Optionally sets the file that should store external initializers for the compiled ONNX model.
|
|
5871
|
+
* If not set, initializers are stored within the model.
|
|
5872
|
+
*
|
|
5873
|
+
* Only initializers for nodes that were not compiled are stored in the external initializers file.
|
|
5874
|
+
* Compiled nodes contain their initializer data within the `ep_cache_context` attribute of EPContext nodes.
|
|
5875
|
+
* Refer to ModelCompilationOptions_SetEpContextEmbedMode.
|
|
5876
|
+
*
|
|
5877
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5878
|
+
* \param[in] external_initializers_file_path Null terminated string of the path to the file.
|
|
5879
|
+
* \param[in] external_initializers_size_threshold Initializers larger than this threshold are stored in the file.
|
|
5880
|
+
*
|
|
5881
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5882
|
+
*
|
|
5883
|
+
* \since Version 1.22.
|
|
5884
|
+
*/
|
|
5885
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelExternalInitializersFile,
|
|
5886
|
+
_In_ OrtModelCompilationOptions* model_compile_options,
|
|
5887
|
+
_In_ const ORTCHAR_T* external_initializers_file_path,
|
|
5888
|
+
size_t external_initializers_size_threshold);
|
|
5889
|
+
|
|
5890
|
+
/** \brief Configures model compilation to store the output compiled ONNX model in a buffer.
|
|
5891
|
+
*
|
|
5892
|
+
* The caller passes an OrtAllocator that ONNX Runtime uses to allocate memory for the buffer.
|
|
5893
|
+
*
|
|
5894
|
+
* The output model's location (e.g., file path or memory buffer) can be set with either
|
|
5895
|
+
* ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer.
|
|
5896
|
+
*
|
|
5897
|
+
* If the output model's location is not set, ONNX Runtime will generate an output file with a path based on
|
|
5898
|
+
* the input model's file path. Examples:
|
|
5899
|
+
* /Path/my_model.onnx -> /Path/my_model_ctx.onnx
|
|
5900
|
+
* /Path/my_model -> /Path/my_model_ctx.onnx
|
|
5901
|
+
*
|
|
5902
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5903
|
+
* \param[in] allocator The allocator used to allocate the buffer for the compiled model.
|
|
5904
|
+
* \param[out] output_model_buffer_ptr Pointer to the buffer that stores the compiled model.
|
|
5905
|
+
* \param[out] output_model_buffer_size_ptr Pointer set to the size of output model in bytes.
|
|
5906
|
+
*
|
|
5907
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5908
|
+
*
|
|
5909
|
+
* \since Version 1.22.
|
|
5910
|
+
*/
|
|
5911
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelBuffer,
|
|
5912
|
+
_In_ OrtModelCompilationOptions* model_compile_options,
|
|
5913
|
+
_Inout_ OrtAllocator* allocator,
|
|
5914
|
+
_Outptr_ void** output_model_buffer_ptr,
|
|
5915
|
+
_Out_ size_t* output_model_buffer_size_ptr);
|
|
5916
|
+
|
|
5917
|
+
/** \brief Enables or disables the embedding of EPContext binary data into the `ep_cache_context` attribute
|
|
5918
|
+
* of EPContext nodes. Defaults to false.
|
|
5919
|
+
*
|
|
5920
|
+
* If enabled, the `ep_cache_context` attribute of EPContext nodes will store the context binary data, which may
|
|
5921
|
+
* include weights for compiled subgraphs.
|
|
5922
|
+
*
|
|
5923
|
+
* If disabled, the `ep_cache_context` attribute of EPContext nodes will contain the path to the file containing the
|
|
5924
|
+
* context binary data. The path is set by the execution provider creating the EPContext node.
|
|
5925
|
+
*
|
|
5926
|
+
* More details relate to EPContext design refers to:
|
|
5927
|
+
* \htmlonly
|
|
5928
|
+
* <a href="https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html">EPContext design document.</a>
|
|
5929
|
+
* \endhtmlonly
|
|
5930
|
+
*
|
|
5931
|
+
* \param[in] model_compile_options The OrtModelCompilationOptions instance.
|
|
5932
|
+
* \param[in] embed_ep_context_in_model True to embed EPContext binary data into the EPContext node
|
|
5933
|
+
* `ep_cache_context` attributes.
|
|
5934
|
+
*
|
|
5935
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5936
|
+
*
|
|
5937
|
+
* \since Version 1.22.
|
|
5938
|
+
*/
|
|
5939
|
+
ORT_API2_STATUS(ModelCompilationOptions_SetEpContextEmbedMode, _In_ OrtModelCompilationOptions* model_compile_options,
|
|
5940
|
+
bool embed_ep_context_in_model);
|
|
5941
|
+
|
|
5942
|
+
/** \brief Compiles an input ONNX model with the given compilation options.
|
|
5943
|
+
*
|
|
5944
|
+
* \param[in] env OrtEnv object.
|
|
5945
|
+
* \param[in] model_options The compilation options that defines compilation options for a model.
|
|
5946
|
+
*
|
|
5947
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
5948
|
+
*
|
|
5949
|
+
* \since Version 1.22.
|
|
5950
|
+
*/
|
|
5951
|
+
ORT_API2_STATUS(CompileModel, _In_ const OrtEnv* env, _In_ const OrtModelCompilationOptions* model_options);
|
|
5952
|
+
};
|
|
5953
|
+
|
|
5954
|
+
ORT_RUNTIME_CLASS(Ep);
|
|
5955
|
+
ORT_RUNTIME_CLASS(EpFactory);
|
|
5956
|
+
|
|
5957
|
+
struct OrtEpApi {
|
|
5958
|
+
/** \brief Create an OrtEpDevice for the EP and an OrtHardwareDevice.
|
|
5959
|
+
* \param[in] ep_factory Execution provider factory that is creating the instance.
|
|
5960
|
+
* \param[in] hardware_device Hardware device that the EP can utilize.
|
|
5961
|
+
* \param[in] ep_metadata Optional OrtKeyValuePairs instance for execution provider metadata that may be used
|
|
5962
|
+
* during execution provider selection and passed to CreateEp.
|
|
5963
|
+
* ep_device will copy this instance and the user should call ReleaseKeyValuePairs.
|
|
5964
|
+
* \param[in] ep_options Optional OrtKeyValuePairs instance for execution provider options that will be added
|
|
5965
|
+
* to the Session configuration options if the execution provider is selected.
|
|
5966
|
+
* ep_device will copy this instance and the user should call ReleaseKeyValuePairs.
|
|
5967
|
+
* \param ep_device OrtExecutionDevice that is created.
|
|
5968
|
+
*
|
|
5969
|
+
* \since Version 1.22.
|
|
5970
|
+
*/
|
|
5971
|
+
ORT_API2_STATUS(CreateEpDevice, _In_ OrtEpFactory* ep_factory,
|
|
5972
|
+
_In_ const OrtHardwareDevice* hardware_device,
|
|
5973
|
+
_In_opt_ const OrtKeyValuePairs* ep_metadata,
|
|
5974
|
+
_In_opt_ const OrtKeyValuePairs* ep_options,
|
|
5975
|
+
_Out_ OrtEpDevice** ep_device);
|
|
5976
|
+
|
|
5977
|
+
ORT_CLASS_RELEASE(EpDevice);
|
|
5978
|
+
};
|
|
5979
|
+
|
|
5980
|
+
/**
|
|
5981
|
+
* \brief The OrtEp struct provides functions to implement for an execution provider.
|
|
5982
|
+
* \since Version 1.22.
|
|
5983
|
+
*/
|
|
5984
|
+
struct OrtEp {
|
|
5985
|
+
/** \brief The ONNX Runtime version the execution provider was compiled with.
|
|
5986
|
+
*
|
|
5987
|
+
* Implementation should set to ORT_API_VERSION.
|
|
5988
|
+
* ORT will use this to ensure it does not call functions that were not available when the library was compiled.
|
|
5989
|
+
*
|
|
5990
|
+
* \since Version 1.22.
|
|
5991
|
+
*/
|
|
5992
|
+
uint32_t ort_version_supported;
|
|
5993
|
+
|
|
5994
|
+
/** \brief Get the execution provider name.
|
|
5995
|
+
*
|
|
5996
|
+
* \param[in] this_ptr The OrtEp instance.
|
|
5997
|
+
* \return The execution provider name.
|
|
5998
|
+
*
|
|
5999
|
+
* \note Returned string is owned by ORT and valid until UnregisterExecutionProviderLibrary is called.
|
|
6000
|
+
*
|
|
6001
|
+
* \since Version 1.22.
|
|
6002
|
+
*/
|
|
6003
|
+
const char*(ORT_API_CALL* GetName)(const OrtEp* this_ptr);
|
|
6004
|
+
|
|
6005
|
+
// OrtStatus* GetCapability(OrtEp* ep, const OrtGraph* graph,
|
|
6006
|
+
// size_t* num_supported_subgraphs,
|
|
6007
|
+
// OrtIndexedSubgraph** supported_subgraphs, OrtAllocator* allocator);
|
|
6008
|
+
|
|
6009
|
+
// OrtStatus* Compile(OrtEp* ep, const OrtGraph** graphs, OrtNode** fused_graph_nodes,
|
|
6010
|
+
// size_t count, OrtNodeComputeInfo* node_compute_infos);
|
|
6011
|
+
|
|
6012
|
+
// TODO: Implement OrtEpApi and the complete OrtEp interface as the next step.
|
|
6013
|
+
};
|
|
6014
|
+
|
|
6015
|
+
/** \brief The function signature that ORT will call to create OrtEpFactory instances.
|
|
6016
|
+
*
|
|
6017
|
+
* This must be available in a function called 'CreateEpFactories' in the execution provider library.
|
|
6018
|
+
*
|
|
6019
|
+
* \param[in] registered_name The name the execution library is registered with by RegisterExecutionProviderLibrary
|
|
6020
|
+
* \param[in] ort_api_base The OrtApiBase instance that is used by the factory to get the OrtApi instance for the
|
|
6021
|
+
* version of ORT that the library was compiled against.
|
|
6022
|
+
* \param[in,out] factories The implementation should create and add OrtEpFactory instances to this
|
|
6023
|
+
* pre-allocated array.
|
|
6024
|
+
* i.e. usage is `factories[0] = new MyEpFactory();`
|
|
6025
|
+
* \param[in] max_factories The maximum number of OrtEpFactory instances that can be added to `factories`.
|
|
6026
|
+
* Current default is to allow 4 factories. This can be increased in the future if needed.
|
|
6027
|
+
* \param[out] num_factories The number of OrtEpFactory instances created by the factory and added to `factories`.
|
|
6028
|
+
*
|
|
6029
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
6030
|
+
*
|
|
6031
|
+
* \since Version 1.22.
|
|
6032
|
+
*/
|
|
6033
|
+
typedef OrtStatus* (*CreateEpApiFactoriesFn)(_In_ const char* registered_name, _In_ const OrtApiBase* ort_api_base,
|
|
6034
|
+
_Inout_ OrtEpFactory** factories, _In_ size_t max_factories,
|
|
6035
|
+
_Out_ size_t* num_factories);
|
|
6036
|
+
|
|
6037
|
+
/** \brief The function signature that ORT will call to release an OrtEpFactory instance.
|
|
6038
|
+
*
|
|
6039
|
+
* This must be available in a function called 'ReleaseEpFactory' in the execution provider library.
|
|
6040
|
+
*
|
|
6041
|
+
* \param[in] factory The OrtEpFactory instance to release.
|
|
6042
|
+
*
|
|
6043
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
6044
|
+
*
|
|
6045
|
+
* \since Version 1.22.
|
|
6046
|
+
*/
|
|
6047
|
+
typedef OrtStatus* (*ReleaseEpApiFactoryFn)(_In_ OrtEpFactory* factory);
|
|
6048
|
+
|
|
6049
|
+
/**
|
|
6050
|
+
* \brief The OrtEpFactory provides functions to create and manage execution providers.
|
|
6051
|
+
* \since Version 1.22.
|
|
6052
|
+
*/
|
|
6053
|
+
struct OrtEpFactory {
|
|
6054
|
+
/** \brief The ONNX Runtime version the execution provider was compiled with.
|
|
6055
|
+
*
|
|
6056
|
+
* Implementation should set to ORT_API_VERSION.
|
|
6057
|
+
* ORT will use this to ensure it does not call functions that were not available when the library was compiled.
|
|
6058
|
+
*
|
|
6059
|
+
* \since Version 1.22.
|
|
6060
|
+
*/
|
|
6061
|
+
uint32_t ort_version_supported;
|
|
6062
|
+
|
|
6063
|
+
/** \brief Get the name the of the execution provider that the factory creates.
|
|
6064
|
+
*
|
|
6065
|
+
* \param[in] this_ptr The OrtEpFactory instance.
|
|
6066
|
+
* \return The name of the execution provider the factory creates.
|
|
6067
|
+
*
|
|
6068
|
+
* \since Version 1.22.
|
|
6069
|
+
*/
|
|
6070
|
+
const char*(ORT_API_CALL* GetName)(const OrtEpFactory* this_ptr);
|
|
6071
|
+
|
|
6072
|
+
/** \brief Get the name of vendor who owns the execution provider that the factory creates.
|
|
6073
|
+
*
|
|
6074
|
+
* \param[in] this_ptr The OrtEpFactory instance.
|
|
6075
|
+
* \return vendor The vendor name of the execution provider the factory creates.
|
|
6076
|
+
*
|
|
6077
|
+
* \since Version 1.22.
|
|
6078
|
+
*/
|
|
6079
|
+
const char*(ORT_API_CALL* GetVendor)(const OrtEpFactory* this_ptr); // return EP vendor
|
|
6080
|
+
|
|
6081
|
+
/** \brief Get information from the execution provider if it supports the OrtHardwareDevice.
|
|
6082
|
+
*
|
|
6083
|
+
* \param[in] this_ptr The OrtEpFactory instance.
|
|
6084
|
+
* Non-const as the factory is passed through to the CreateEp call via the OrtEpDevice.
|
|
6085
|
+
* \param[in] devices The OrtHardwareDevice instances that are available.
|
|
6086
|
+
* \param[in] num_devices The number of OrtHardwareDevice instances.
|
|
6087
|
+
* \param[out] ep_devices OrtEpDevice instances for each OrtHardwareDevice that the EP can use.
|
|
6088
|
+
* The implementation should call OrtEpApi::CreateEpDevice to create, and add the OrtEpDevice
|
|
6089
|
+
* instances to this pre-allocated array. ORT will take ownership of the values returned.
|
|
6090
|
+
* i.e. usage is `ep_devices[0] = <ptr to OrtEpDevice created with OrtEpApi::CreateEpDevice>;`
|
|
6091
|
+
* \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices.
|
|
6092
|
+
* Current default is 8. This can be increased if needed.
|
|
6093
|
+
* \param[out] num_ep_devices The number of EP devices added to ep_devices.
|
|
6094
|
+
* \return true if the factory can create an execution provider that uses `device`.
|
|
6095
|
+
*
|
|
6096
|
+
* \note ORT will take ownership or ep_metadata and/or ep_options if they are not null.
|
|
6097
|
+
*
|
|
6098
|
+
* \since Version 1.22.
|
|
6099
|
+
*/
|
|
6100
|
+
OrtStatus*(ORT_API_CALL* GetSupportedDevices)(_In_ OrtEpFactory* this_ptr,
|
|
6101
|
+
_In_reads_(num_devices) const OrtHardwareDevice* const* devices,
|
|
6102
|
+
_In_ size_t num_devices,
|
|
6103
|
+
_Inout_ OrtEpDevice** ep_devices,
|
|
6104
|
+
_In_ size_t max_ep_devices,
|
|
6105
|
+
_Out_ size_t* num_ep_devices);
|
|
6106
|
+
|
|
6107
|
+
/** \brief Function to create an OrtEp instance for use in a Session.
|
|
6108
|
+
*
|
|
6109
|
+
* ORT will call ReleaseEp to release the instance when it is no longer needed.
|
|
6110
|
+
*
|
|
6111
|
+
* \param[in] this_ptr The OrtEpFactory instance.
|
|
6112
|
+
* \param[in] devices The OrtHardwareDevice instances that the execution provider was selected to use.
|
|
6113
|
+
* \param[in] ep_metadata_pairs Execution provider metadata that was provided to OrtEpApi::CreateEpDevice, for each
|
|
6114
|
+
* device.
|
|
6115
|
+
* \param[in] num_devices The number of devices the execution provider was selected for.
|
|
6116
|
+
* \param[in] session_options The OrtSessionOptions instance that contains the configuration options for the
|
|
6117
|
+
* session. This will include ep_options from GetSupportedDevices as well as any
|
|
6118
|
+
* user provided overrides.
|
|
6119
|
+
* Execution provider options will have been added with a prefix of 'ep.<ep name>.'.
|
|
6120
|
+
* The OrtSessionOptions instance will NOT be valid after this call and should not be
|
|
6121
|
+
* stored for later use.
|
|
6122
|
+
* \param[in] logger The OrtLogger instance for the session that the execution provider should use for logging.
|
|
6123
|
+
* \param[out] ep The OrtEp instance created by the factory.
|
|
6124
|
+
*
|
|
6125
|
+
* \snippet{doc} snippets.dox OrtStatus Return Value
|
|
6126
|
+
*
|
|
6127
|
+
* \since Version <coming soon>. This is a placeholder.
|
|
6128
|
+
*/
|
|
6129
|
+
OrtStatus*(ORT_API_CALL* CreateEp)(_In_ OrtEpFactory* this_ptr,
|
|
6130
|
+
_In_reads_(num_devices) const OrtHardwareDevice* const* devices,
|
|
6131
|
+
_In_reads_(num_devices) const OrtKeyValuePairs* const* ep_metadata_pairs,
|
|
6132
|
+
_In_ size_t num_devices,
|
|
6133
|
+
_In_ const OrtSessionOptions* session_options,
|
|
6134
|
+
_In_ const OrtLogger* logger, _Outptr_ OrtEp** ep);
|
|
6135
|
+
|
|
6136
|
+
/** \brief Release the OrtEp instance.
|
|
6137
|
+
*
|
|
6138
|
+
* \param[in] this_ptr The OrtEpFactory instance.
|
|
6139
|
+
* \param[in] ep The OrtEp instance to release.
|
|
6140
|
+
*
|
|
6141
|
+
* \since Version <coming soon>. This is a placeholder.
|
|
6142
|
+
*/
|
|
6143
|
+
void(ORT_API_CALL* ReleaseEp)(OrtEpFactory* this_ptr, struct OrtEp* ep);
|
|
6144
|
+
};
|
|
6145
|
+
|
|
4903
6146
|
/*
|
|
4904
6147
|
* This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality
|
|
4905
6148
|
* This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists
|