@flyteorg/flyteidl 1.16.0 → 2.0.0-test1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -9
- package/LICENSE +0 -202
- package/NOTICE +0 -4
- package/README.md +0 -14
- package/gen/pb-js/flyteidl.d.ts +0 -28307
- package/gen/pb-js/flyteidl.js +0 -66636
- package/protos/buf.lock +0 -18
- package/protos/buf.yaml +0 -12
- package/protos/docs/admin/admin.rst +0 -4623
- package/protos/docs/admin/index.rst +0 -13
- package/protos/docs/contributing.md +0 -68
- package/protos/docs/core/core.rst +0 -3952
- package/protos/docs/core/index.rst +0 -15
- package/protos/docs/datacatalog/datacatalog.rst +0 -1313
- package/protos/docs/datacatalog/index.rst +0 -16
- package/protos/docs/event/event.rst +0 -726
- package/protos/docs/event/index.rst +0 -27
- package/protos/docs/plugins/index.rst +0 -14
- package/protos/docs/plugins/plugins.rst +0 -780
- package/protos/docs/restructuredtext.tmpl +0 -129
- package/protos/docs/service/index.rst +0 -13
- package/protos/docs/service/service.rst +0 -543
- package/protos/docs/withoutscalar_restructuredtext.tmpl +0 -105
- package/protos/docs_index.rst +0 -19
- package/protos/flyteidl/admin/agent.proto +0 -309
- package/protos/flyteidl/admin/cluster_assignment.proto +0 -11
- package/protos/flyteidl/admin/common.proto +0 -337
- package/protos/flyteidl/admin/description_entity.proto +0 -95
- package/protos/flyteidl/admin/event.proto +0 -70
- package/protos/flyteidl/admin/execution.proto +0 -436
- package/protos/flyteidl/admin/launch_plan.proto +0 -231
- package/protos/flyteidl/admin/matchable_resource.proto +0 -194
- package/protos/flyteidl/admin/node_execution.proto +0 -248
- package/protos/flyteidl/admin/notification.proto +0 -27
- package/protos/flyteidl/admin/project.proto +0 -132
- package/protos/flyteidl/admin/project_attributes.proto +0 -69
- package/protos/flyteidl/admin/project_domain_attributes.proto +0 -80
- package/protos/flyteidl/admin/schedule.proto +0 -43
- package/protos/flyteidl/admin/signal.proto +0 -86
- package/protos/flyteidl/admin/task.proto +0 -71
- package/protos/flyteidl/admin/task_execution.proto +0 -171
- package/protos/flyteidl/admin/version.proto +0 -27
- package/protos/flyteidl/admin/workflow.proto +0 -92
- package/protos/flyteidl/admin/workflow_attributes.proto +0 -89
- package/protos/flyteidl/cacheservice/cacheservice.proto +0 -143
- package/protos/flyteidl/core/artifact_id.proto +0 -112
- package/protos/flyteidl/core/catalog.proto +0 -63
- package/protos/flyteidl/core/compiler.proto +0 -64
- package/protos/flyteidl/core/condition.proto +0 -63
- package/protos/flyteidl/core/dynamic_job.proto +0 -32
- package/protos/flyteidl/core/errors.proto +0 -42
- package/protos/flyteidl/core/execution.proto +0 -156
- package/protos/flyteidl/core/execution_envs.proto +0 -45
- package/protos/flyteidl/core/identifier.proto +0 -80
- package/protos/flyteidl/core/interface.proto +0 -64
- package/protos/flyteidl/core/literals.proto +0 -205
- package/protos/flyteidl/core/metrics.proto +0 -50
- package/protos/flyteidl/core/security.proto +0 -135
- package/protos/flyteidl/core/tasks.proto +0 -392
- package/protos/flyteidl/core/types.proto +0 -208
- package/protos/flyteidl/core/workflow.proto +0 -361
- package/protos/flyteidl/core/workflow_closure.proto +0 -18
- package/protos/flyteidl/datacatalog/datacatalog.proto +0 -420
- package/protos/flyteidl/event/cloudevents.proto +0 -81
- package/protos/flyteidl/event/event.proto +0 -347
- package/protos/flyteidl/plugins/array_job.proto +0 -30
- package/protos/flyteidl/plugins/common.proto +0 -27
- package/protos/flyteidl/plugins/dask.proto +0 -41
- package/protos/flyteidl/plugins/kubeflow/common.proto +0 -28
- package/protos/flyteidl/plugins/kubeflow/mpi.proto +0 -47
- package/protos/flyteidl/plugins/kubeflow/pytorch.proto +0 -53
- package/protos/flyteidl/plugins/kubeflow/tensorflow.proto +0 -46
- package/protos/flyteidl/plugins/mpi.proto +0 -20
- package/protos/flyteidl/plugins/presto.proto +0 -14
- package/protos/flyteidl/plugins/pytorch.proto +0 -25
- package/protos/flyteidl/plugins/qubole.proto +0 -26
- package/protos/flyteidl/plugins/ray.proto +0 -59
- package/protos/flyteidl/plugins/spark.proto +0 -42
- package/protos/flyteidl/plugins/tensorflow.proto +0 -18
- package/protos/flyteidl/plugins/waitable.proto +0 -15
- package/protos/flyteidl/service/admin.proto +0 -668
- package/protos/flyteidl/service/agent.proto +0 -79
- package/protos/flyteidl/service/auth.proto +0 -94
- package/protos/flyteidl/service/dataproxy.proto +0 -205
- package/protos/flyteidl/service/external_plugin_service.proto +0 -79
- package/protos/flyteidl/service/identity.proto +0 -51
- package/protos/flyteidl/service/signal.proto +0 -55
|
@@ -1,347 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.event;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/core/literals.proto";
|
|
8
|
-
import "flyteidl/core/compiler.proto";
|
|
9
|
-
import "flyteidl/core/execution.proto";
|
|
10
|
-
import "flyteidl/core/identifier.proto";
|
|
11
|
-
import "flyteidl/core/catalog.proto";
|
|
12
|
-
import "google/protobuf/timestamp.proto";
|
|
13
|
-
import "google/protobuf/struct.proto";
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
message WorkflowExecutionEvent {
|
|
17
|
-
// Workflow execution id
|
|
18
|
-
core.WorkflowExecutionIdentifier execution_id = 1;
|
|
19
|
-
|
|
20
|
-
// the id of the originator (Propeller) of the event
|
|
21
|
-
string producer_id = 2;
|
|
22
|
-
|
|
23
|
-
core.WorkflowExecution.Phase phase = 3;
|
|
24
|
-
|
|
25
|
-
// This timestamp represents when the original event occurred, it is generated
|
|
26
|
-
// by the executor of the workflow.
|
|
27
|
-
google.protobuf.Timestamp occurred_at = 4;
|
|
28
|
-
|
|
29
|
-
oneof output_result {
|
|
30
|
-
// URL to the output of the execution, it encodes all the information
|
|
31
|
-
// including Cloud source provider. ie., s3://...
|
|
32
|
-
string output_uri = 5;
|
|
33
|
-
|
|
34
|
-
// Error information for the execution
|
|
35
|
-
core.ExecutionError error = 6;
|
|
36
|
-
|
|
37
|
-
// Raw output data produced by this workflow execution.
|
|
38
|
-
core.LiteralMap output_data = 7;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
message NodeExecutionEvent {
|
|
43
|
-
// Unique identifier for this node execution
|
|
44
|
-
core.NodeExecutionIdentifier id = 1;
|
|
45
|
-
|
|
46
|
-
// the id of the originator (Propeller) of the event
|
|
47
|
-
string producer_id = 2;
|
|
48
|
-
|
|
49
|
-
core.NodeExecution.Phase phase = 3;
|
|
50
|
-
|
|
51
|
-
// This timestamp represents when the original event occurred, it is generated
|
|
52
|
-
// by the executor of the node.
|
|
53
|
-
google.protobuf.Timestamp occurred_at = 4;
|
|
54
|
-
|
|
55
|
-
oneof input_value {
|
|
56
|
-
string input_uri = 5;
|
|
57
|
-
|
|
58
|
-
// Raw input data consumed by this node execution.
|
|
59
|
-
core.LiteralMap input_data = 20;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
oneof output_result {
|
|
63
|
-
// URL to the output of the execution, it encodes all the information
|
|
64
|
-
// including Cloud source provider. ie., s3://...
|
|
65
|
-
string output_uri = 6;
|
|
66
|
-
|
|
67
|
-
// Error information for the execution
|
|
68
|
-
core.ExecutionError error = 7;
|
|
69
|
-
|
|
70
|
-
// Raw output data produced by this node execution.
|
|
71
|
-
core.LiteralMap output_data = 15;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Additional metadata to do with this event's node target based
|
|
75
|
-
// on the node type
|
|
76
|
-
oneof target_metadata {
|
|
77
|
-
WorkflowNodeMetadata workflow_node_metadata = 8;
|
|
78
|
-
TaskNodeMetadata task_node_metadata = 14;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
// [To be deprecated] Specifies which task (if any) launched this node.
|
|
82
|
-
ParentTaskExecutionMetadata parent_task_metadata = 9;
|
|
83
|
-
|
|
84
|
-
// Specifies the parent node of the current node execution. Node executions at level zero will not have a parent node.
|
|
85
|
-
ParentNodeExecutionMetadata parent_node_metadata = 10;
|
|
86
|
-
|
|
87
|
-
// Retry group to indicate grouping of nodes by retries
|
|
88
|
-
string retry_group = 11;
|
|
89
|
-
|
|
90
|
-
// Identifier of the node in the original workflow/graph
|
|
91
|
-
// This maps to value of WorkflowTemplate.nodes[X].id
|
|
92
|
-
string spec_node_id = 12;
|
|
93
|
-
|
|
94
|
-
// Friendly readable name for the node
|
|
95
|
-
string node_name = 13;
|
|
96
|
-
|
|
97
|
-
int32 event_version = 16;
|
|
98
|
-
|
|
99
|
-
// Whether this node launched a subworkflow.
|
|
100
|
-
bool is_parent = 17;
|
|
101
|
-
|
|
102
|
-
// Whether this node yielded a dynamic workflow.
|
|
103
|
-
bool is_dynamic = 18;
|
|
104
|
-
|
|
105
|
-
// String location uniquely identifying where the deck HTML file is
|
|
106
|
-
// NativeUrl specifies the url in the format of the configured storage provider (e.g. s3://my-bucket/randomstring/suffix.tar)
|
|
107
|
-
string deck_uri = 19;
|
|
108
|
-
|
|
109
|
-
// This timestamp represents the instant when the event was reported by the executing framework. For example,
|
|
110
|
-
// when first processing a node the `occurred_at` timestamp should be the instant propeller makes progress, so when
|
|
111
|
-
// literal inputs are initially copied. The event however will not be sent until after the copy completes.
|
|
112
|
-
// Extracting both of these timestamps facilitates a more accurate portrayal of the evaluation time-series.
|
|
113
|
-
google.protobuf.Timestamp reported_at = 21;
|
|
114
|
-
|
|
115
|
-
// Indicates if this node is an ArrayNode.
|
|
116
|
-
bool is_array = 22;
|
|
117
|
-
|
|
118
|
-
// So that Admin doesn't have to rebuild the node execution graph to find the target entity, propeller will fill this
|
|
119
|
-
// in optionally - currently this is only filled in for subworkflows. This is the ID of the subworkflow corresponding
|
|
120
|
-
// to this node execution. It is difficult to find because Admin only sees one node at a time. A subworkflow could be
|
|
121
|
-
// nested multiple layers deep, and you'd need to access the correct workflow template to know the target subworkflow.
|
|
122
|
-
core.Identifier target_entity = 23;
|
|
123
|
-
|
|
124
|
-
// Tasks and subworkflows (but not launch plans) that are run within a dynamic task are effectively independent of
|
|
125
|
-
// the tasks that are registered in Admin's db. Confusingly, they are often identical, but sometimes they are not
|
|
126
|
-
// even registered at all. Similar to the target_entity field, at the time Admin receives this event, it has no idea
|
|
127
|
-
// if the relevant execution entity is was registered, or dynamic. This field indicates that the target_entity ID,
|
|
128
|
-
// as well as task IDs in any corresponding Task Executions, should not be used to looked up the task in Admin's db.
|
|
129
|
-
bool is_in_dynamic_chain = 24;
|
|
130
|
-
|
|
131
|
-
// Whether this node launched an eager task.
|
|
132
|
-
bool is_eager = 25;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// For Workflow Nodes we need to send information about the workflow that's launched
|
|
136
|
-
message WorkflowNodeMetadata {
|
|
137
|
-
core.WorkflowExecutionIdentifier execution_id = 1;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
message TaskNodeMetadata {
|
|
141
|
-
// Captures the status of caching for this execution.
|
|
142
|
-
core.CatalogCacheStatus cache_status = 1;
|
|
143
|
-
// This structure carries the catalog artifact information
|
|
144
|
-
core.CatalogMetadata catalog_key = 2;
|
|
145
|
-
// Captures the status of cache reservations for this execution.
|
|
146
|
-
core.CatalogReservation.Status reservation_status = 3;
|
|
147
|
-
// The latest checkpoint location
|
|
148
|
-
string checkpoint_uri = 4;
|
|
149
|
-
|
|
150
|
-
// In the case this task launched a dynamic workflow we capture its structure here.
|
|
151
|
-
DynamicWorkflowNodeMetadata dynamic_workflow = 16;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
// For dynamic workflow nodes we send information about the dynamic workflow definition that gets generated.
|
|
155
|
-
message DynamicWorkflowNodeMetadata {
|
|
156
|
-
// id represents the unique identifier of the workflow.
|
|
157
|
-
core.Identifier id = 1;
|
|
158
|
-
|
|
159
|
-
// Represents the compiled representation of the embedded dynamic workflow.
|
|
160
|
-
core.CompiledWorkflowClosure compiled_workflow = 2;
|
|
161
|
-
|
|
162
|
-
// dynamic_job_spec_uri is the location of the DynamicJobSpec proto message for this DynamicWorkflow. This is
|
|
163
|
-
// required to correctly recover partially completed executions where the workflow has already been compiled.
|
|
164
|
-
string dynamic_job_spec_uri = 3;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
message ParentTaskExecutionMetadata {
|
|
168
|
-
core.TaskExecutionIdentifier id = 1;
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
message ParentNodeExecutionMetadata {
|
|
172
|
-
// Unique identifier of the parent node id within the execution
|
|
173
|
-
// This is value of core.NodeExecutionIdentifier.node_id of the parent node
|
|
174
|
-
string node_id = 1;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
message EventReason {
|
|
178
|
-
// An explanation for this event
|
|
179
|
-
string reason = 1;
|
|
180
|
-
|
|
181
|
-
// The time this reason occurred
|
|
182
|
-
google.protobuf.Timestamp occurred_at = 2;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// Plugin specific execution event information. For tasks like Python, Hive, Spark, DynamicJob.
|
|
186
|
-
message TaskExecutionEvent {
|
|
187
|
-
// ID of the task. In combination with the retryAttempt this will indicate
|
|
188
|
-
// the task execution uniquely for a given parent node execution.
|
|
189
|
-
core.Identifier task_id = 1;
|
|
190
|
-
|
|
191
|
-
// A task execution is always kicked off by a node execution, the event consumer
|
|
192
|
-
// will use the parent_id to relate the task to it's parent node execution
|
|
193
|
-
core.NodeExecutionIdentifier parent_node_execution_id = 2;
|
|
194
|
-
|
|
195
|
-
// retry attempt number for this task, ie., 2 for the second attempt
|
|
196
|
-
uint32 retry_attempt = 3;
|
|
197
|
-
|
|
198
|
-
// Phase associated with the event
|
|
199
|
-
core.TaskExecution.Phase phase = 4;
|
|
200
|
-
|
|
201
|
-
// id of the process that sent this event, mainly for trace debugging
|
|
202
|
-
string producer_id = 5;
|
|
203
|
-
|
|
204
|
-
// log information for the task execution
|
|
205
|
-
repeated core.TaskLog logs = 6;
|
|
206
|
-
|
|
207
|
-
// This timestamp represents when the original event occurred, it is generated
|
|
208
|
-
// by the executor of the task.
|
|
209
|
-
google.protobuf.Timestamp occurred_at = 7;
|
|
210
|
-
|
|
211
|
-
oneof input_value {
|
|
212
|
-
// URI of the input file, it encodes all the information
|
|
213
|
-
// including Cloud source provider. ie., s3://...
|
|
214
|
-
string input_uri = 8;
|
|
215
|
-
|
|
216
|
-
// Raw input data consumed by this task execution.
|
|
217
|
-
core.LiteralMap input_data = 19;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
oneof output_result {
|
|
221
|
-
// URI to the output of the execution, it will be in a format that encodes all the information
|
|
222
|
-
// including Cloud source provider. ie., s3://...
|
|
223
|
-
string output_uri = 9;
|
|
224
|
-
|
|
225
|
-
// Error information for the execution
|
|
226
|
-
core.ExecutionError error = 10;
|
|
227
|
-
|
|
228
|
-
// Raw output data produced by this task execution.
|
|
229
|
-
core.LiteralMap output_data = 17;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
// Custom data that the task plugin sends back. This is extensible to allow various plugins in the system.
|
|
233
|
-
google.protobuf.Struct custom_info = 11;
|
|
234
|
-
|
|
235
|
-
// Some phases, like RUNNING, can send multiple events with changed metadata (new logs, additional custom_info, etc)
|
|
236
|
-
// that should be recorded regardless of the lack of phase change.
|
|
237
|
-
// The version field should be incremented when metadata changes across the duration of an individual phase.
|
|
238
|
-
uint32 phase_version = 12;
|
|
239
|
-
|
|
240
|
-
// An optional explanation for the phase transition.
|
|
241
|
-
// Deprecated: Use reasons instead.
|
|
242
|
-
string reason = 13 [deprecated = true];
|
|
243
|
-
|
|
244
|
-
// An optional list of explanations for the phase transition.
|
|
245
|
-
repeated EventReason reasons = 21;
|
|
246
|
-
|
|
247
|
-
// A predefined yet extensible Task type identifier. If the task definition is already registered in flyte admin
|
|
248
|
-
// this type will be identical, but not all task executions necessarily use pre-registered definitions and this
|
|
249
|
-
// type is useful to render the task in the UI, filter task executions, etc.
|
|
250
|
-
string task_type = 14;
|
|
251
|
-
|
|
252
|
-
// Metadata around how a task was executed.
|
|
253
|
-
TaskExecutionMetadata metadata = 16;
|
|
254
|
-
|
|
255
|
-
// The event version is used to indicate versioned changes in how data is reported using this
|
|
256
|
-
// proto message. For example, event_verison > 0 means that maps tasks report logs using the
|
|
257
|
-
// TaskExecutionMetadata ExternalResourceInfo fields for each subtask rather than the TaskLog
|
|
258
|
-
// in this message.
|
|
259
|
-
int32 event_version = 18;
|
|
260
|
-
|
|
261
|
-
// This timestamp represents the instant when the event was reported by the executing framework. For example, a k8s
|
|
262
|
-
// pod task may be marked completed at (ie. `occurred_at`) the instant the container running user code completes,
|
|
263
|
-
// but this event will not be reported until the pod is marked as completed. Extracting both of these timestamps
|
|
264
|
-
// facilitates a more accurate portrayal of the evaluation time-series.
|
|
265
|
-
google.protobuf.Timestamp reported_at = 20;
|
|
266
|
-
|
|
267
|
-
// Contains metadata required to identify logs related to this task execution
|
|
268
|
-
core.LogContext log_context = 22;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
// This message contains metadata about external resources produced or used by a specific task execution.
|
|
272
|
-
message ExternalResourceInfo {
|
|
273
|
-
|
|
274
|
-
// Identifier for an external resource created by this task execution, for example Qubole query ID or presto query ids.
|
|
275
|
-
string external_id = 1;
|
|
276
|
-
|
|
277
|
-
// A unique index for the external resource with respect to all external resources for this task. Although the
|
|
278
|
-
// identifier may change between task reporting events or retries, this will remain the same to enable aggregating
|
|
279
|
-
// information from multiple reports.
|
|
280
|
-
uint32 index = 2;
|
|
281
|
-
|
|
282
|
-
// Retry attempt number for this external resource, ie., 2 for the second attempt
|
|
283
|
-
uint32 retry_attempt = 3;
|
|
284
|
-
|
|
285
|
-
// Phase associated with the external resource
|
|
286
|
-
core.TaskExecution.Phase phase = 4;
|
|
287
|
-
|
|
288
|
-
// Captures the status of caching for this external resource execution.
|
|
289
|
-
core.CatalogCacheStatus cache_status = 5;
|
|
290
|
-
|
|
291
|
-
// log information for the external resource execution
|
|
292
|
-
repeated core.TaskLog logs = 6;
|
|
293
|
-
|
|
294
|
-
// Additional metadata to do with this event's node target based on the node type. We are
|
|
295
|
-
// explicitly not including the task_node_metadata here because it is not clear if it is needed.
|
|
296
|
-
// If we decide to include in the future, we should deprecate the cache_status field.
|
|
297
|
-
oneof target_metadata {
|
|
298
|
-
WorkflowNodeMetadata workflow_node_metadata = 7;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
// Extensible field for custom, plugin-specific info
|
|
302
|
-
google.protobuf.Struct custom_info = 8;
|
|
303
|
-
|
|
304
|
-
// Contains metadata required to identify logs related to this task execution
|
|
305
|
-
core.LogContext log_context = 9;
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
// This message holds task execution metadata specific to resource allocation used to manage concurrent
|
|
310
|
-
// executions for a project namespace.
|
|
311
|
-
message ResourcePoolInfo {
|
|
312
|
-
// Unique resource ID used to identify this execution when allocating a token.
|
|
313
|
-
string allocation_token = 1;
|
|
314
|
-
|
|
315
|
-
// Namespace under which this task execution requested an allocation token.
|
|
316
|
-
string namespace = 2;
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
// Holds metadata around how a task was executed.
|
|
320
|
-
// As a task transitions across event phases during execution some attributes, such its generated name, generated external resources,
|
|
321
|
-
// and more may grow in size but not change necessarily based on the phase transition that sparked the event update.
|
|
322
|
-
// Metadata is a container for these attributes across the task execution lifecycle.
|
|
323
|
-
message TaskExecutionMetadata {
|
|
324
|
-
|
|
325
|
-
// Unique, generated name for this task execution used by the backend.
|
|
326
|
-
string generated_name = 1;
|
|
327
|
-
|
|
328
|
-
// Additional data on external resources on other back-ends or platforms (e.g. Hive, Qubole, etc) launched by this task execution.
|
|
329
|
-
repeated ExternalResourceInfo external_resources = 2;
|
|
330
|
-
|
|
331
|
-
// Includes additional data on concurrent resource management used during execution..
|
|
332
|
-
// This is a repeated field because a plugin can request multiple resource allocations during execution.
|
|
333
|
-
repeated ResourcePoolInfo resource_pool_info = 3;
|
|
334
|
-
|
|
335
|
-
// The identifier of the plugin used to execute this task.
|
|
336
|
-
string plugin_identifier = 4;
|
|
337
|
-
|
|
338
|
-
// Includes the broad category of machine used for this specific task execution.
|
|
339
|
-
enum InstanceClass {
|
|
340
|
-
// The default instance class configured for the flyte application platform.
|
|
341
|
-
DEFAULT = 0;
|
|
342
|
-
|
|
343
|
-
// The instance class configured for interruptible tasks.
|
|
344
|
-
INTERRUPTIBLE = 1;
|
|
345
|
-
}
|
|
346
|
-
InstanceClass instance_class = 16;
|
|
347
|
-
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
// Describes a job that can process independent pieces of data concurrently. Multiple copies of the runnable component
|
|
8
|
-
// will be executed concurrently.
|
|
9
|
-
message ArrayJob {
|
|
10
|
-
// Defines the maximum number of instances to bring up concurrently at any given point. Note that this is an
|
|
11
|
-
// optimistic restriction and that, due to network partitioning or other failures, the actual number of currently
|
|
12
|
-
// running instances might be more. This has to be a positive number if assigned. Default value is size.
|
|
13
|
-
int64 parallelism = 1;
|
|
14
|
-
|
|
15
|
-
// Defines the number of instances to launch at most. This number should match the size of the input if the job
|
|
16
|
-
// requires processing of all input data. This has to be a positive number.
|
|
17
|
-
// In the case this is not defined, the back-end will determine the size at run-time by reading the inputs.
|
|
18
|
-
int64 size = 2;
|
|
19
|
-
|
|
20
|
-
oneof success_criteria {
|
|
21
|
-
// An absolute number of the minimum number of successful completions of subtasks. As soon as this criteria is met,
|
|
22
|
-
// the array job will be marked as successful and outputs will be computed. This has to be a non-negative number if
|
|
23
|
-
// assigned. Default value is size (if specified).
|
|
24
|
-
int64 min_successes = 3;
|
|
25
|
-
|
|
26
|
-
// If the array job size is not known beforehand, the min_success_ratio can instead be used to determine when an array
|
|
27
|
-
// job can be marked successful.
|
|
28
|
-
float min_success_ratio = 4;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/core/tasks.proto";
|
|
8
|
-
|
|
9
|
-
enum RestartPolicy {
|
|
10
|
-
RESTART_POLICY_NEVER = 0;
|
|
11
|
-
RESTART_POLICY_ON_FAILURE = 1;
|
|
12
|
-
RESTART_POLICY_ALWAYS = 2;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
message CommonReplicaSpec {
|
|
16
|
-
// Number of replicas
|
|
17
|
-
int32 replicas = 1;
|
|
18
|
-
|
|
19
|
-
// Image used for the replica group
|
|
20
|
-
string image = 2;
|
|
21
|
-
|
|
22
|
-
// Resources required for the replica group
|
|
23
|
-
core.Resources resources = 3;
|
|
24
|
-
|
|
25
|
-
// RestartPolicy determines whether pods will be restarted when they exit
|
|
26
|
-
RestartPolicy restart_policy = 4;
|
|
27
|
-
}
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
import "flyteidl/core/tasks.proto";
|
|
4
|
-
|
|
5
|
-
package flyteidl.plugins;
|
|
6
|
-
|
|
7
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
// Custom Proto for Dask Plugin.
|
|
11
|
-
message DaskJob {
|
|
12
|
-
// Spec for the scheduler pod.
|
|
13
|
-
DaskScheduler scheduler = 1;
|
|
14
|
-
|
|
15
|
-
// Spec of the default worker group.
|
|
16
|
-
DaskWorkerGroup workers = 2;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// Specification for the scheduler pod.
|
|
20
|
-
message DaskScheduler {
|
|
21
|
-
// Optional image to use. If unset, will use the default image.
|
|
22
|
-
string image = 1;
|
|
23
|
-
|
|
24
|
-
// Resources assigned to the scheduler pod.
|
|
25
|
-
core.Resources resources = 2;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
message DaskWorkerGroup {
|
|
29
|
-
// Number of workers in the group.
|
|
30
|
-
uint32 number_of_workers = 1;
|
|
31
|
-
|
|
32
|
-
// Optional image to use for the pods of the worker group. If unset, will use the default image.
|
|
33
|
-
string image = 2;
|
|
34
|
-
|
|
35
|
-
// Resources assigned to the all pods of the worker group.
|
|
36
|
-
// As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices
|
|
37
|
-
// it is advised to only set limits. If requests are not explicitly set, the plugin will make
|
|
38
|
-
// sure to set requests==limits.
|
|
39
|
-
// The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit.
|
|
40
|
-
core.Resources resources = 3;
|
|
41
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.kubeflow;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins/kubeflow";
|
|
6
|
-
|
|
7
|
-
import public "flyteidl/plugins/common.proto";
|
|
8
|
-
|
|
9
|
-
enum CleanPodPolicy {
|
|
10
|
-
CLEANPOD_POLICY_NONE = 0;
|
|
11
|
-
CLEANPOD_POLICY_RUNNING = 1;
|
|
12
|
-
CLEANPOD_POLICY_ALL = 2;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
message RunPolicy {
|
|
16
|
-
// Defines the policy to kill pods after the job completes. Default to None.
|
|
17
|
-
CleanPodPolicy clean_pod_policy = 1;
|
|
18
|
-
|
|
19
|
-
// TTL to clean up jobs. Default to infinite.
|
|
20
|
-
int32 ttl_seconds_after_finished = 2;
|
|
21
|
-
|
|
22
|
-
// Specifies the duration in seconds relative to the startTime that the job may be active
|
|
23
|
-
// before the system tries to terminate it; value must be positive integer.
|
|
24
|
-
int32 active_deadline_seconds = 3;
|
|
25
|
-
|
|
26
|
-
// Number of retries before marking this job failed.
|
|
27
|
-
int32 backoff_limit = 4;
|
|
28
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.kubeflow;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins/kubeflow";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/core/tasks.proto";
|
|
8
|
-
import "flyteidl/plugins/kubeflow/common.proto";
|
|
9
|
-
|
|
10
|
-
// Proto for plugin that enables distributed training using https://github.com/kubeflow/mpi-operator
|
|
11
|
-
message DistributedMPITrainingTask {
|
|
12
|
-
// Worker replicas spec
|
|
13
|
-
DistributedMPITrainingReplicaSpec worker_replicas = 1;
|
|
14
|
-
|
|
15
|
-
// Master replicas spec
|
|
16
|
-
DistributedMPITrainingReplicaSpec launcher_replicas = 2;
|
|
17
|
-
|
|
18
|
-
// RunPolicy encapsulates various runtime policies of the distributed training
|
|
19
|
-
// job, for example how to clean up resources and how long the job can stay
|
|
20
|
-
// active.
|
|
21
|
-
RunPolicy run_policy = 3;
|
|
22
|
-
|
|
23
|
-
// Number of slots per worker
|
|
24
|
-
int32 slots = 4;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
// Replica specification for distributed MPI training
|
|
28
|
-
message DistributedMPITrainingReplicaSpec {
|
|
29
|
-
// 1~4 deprecated. Use common instead.
|
|
30
|
-
// Number of replicas
|
|
31
|
-
int32 replicas = 1 [deprecated = true];
|
|
32
|
-
|
|
33
|
-
// Image used for the replica group
|
|
34
|
-
string image = 2 [deprecated = true];
|
|
35
|
-
|
|
36
|
-
// Resources required for the replica group
|
|
37
|
-
core.Resources resources = 3 [deprecated = true];
|
|
38
|
-
|
|
39
|
-
// Restart policy determines whether pods will be restarted when they exit
|
|
40
|
-
RestartPolicy restart_policy = 4 [deprecated = true];
|
|
41
|
-
|
|
42
|
-
// MPI sometimes requires different command set for different replica groups
|
|
43
|
-
repeated string command = 5;
|
|
44
|
-
|
|
45
|
-
// The common replica spec
|
|
46
|
-
CommonReplicaSpec common = 6;
|
|
47
|
-
}
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.kubeflow;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins/kubeflow";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/core/tasks.proto";
|
|
8
|
-
import "flyteidl/plugins/kubeflow/common.proto";
|
|
9
|
-
|
|
10
|
-
// Custom proto for torch elastic config for distributed training using
|
|
11
|
-
// https://github.com/kubeflow/training-operator/blob/master/pkg/apis/kubeflow.org/v1/pytorch_types.go
|
|
12
|
-
message ElasticConfig {
|
|
13
|
-
string rdzv_backend = 1;
|
|
14
|
-
int32 min_replicas = 2;
|
|
15
|
-
int32 max_replicas = 3;
|
|
16
|
-
int32 nproc_per_node = 4;
|
|
17
|
-
int32 max_restarts = 5;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
// Proto for plugin that enables distributed training using https://github.com/kubeflow/pytorch-operator
|
|
21
|
-
message DistributedPyTorchTrainingTask {
|
|
22
|
-
// Worker replicas spec
|
|
23
|
-
DistributedPyTorchTrainingReplicaSpec worker_replicas = 1;
|
|
24
|
-
|
|
25
|
-
// Master replicas spec, master replicas can only have 1 replica
|
|
26
|
-
DistributedPyTorchTrainingReplicaSpec master_replicas = 2;
|
|
27
|
-
|
|
28
|
-
// RunPolicy encapsulates various runtime policies of the distributed training
|
|
29
|
-
// job, for example how to clean up resources and how long the job can stay
|
|
30
|
-
// active.
|
|
31
|
-
RunPolicy run_policy = 3;
|
|
32
|
-
|
|
33
|
-
// config for an elastic pytorch job
|
|
34
|
-
ElasticConfig elastic_config = 4;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
message DistributedPyTorchTrainingReplicaSpec {
|
|
38
|
-
// 1~4 deprecated. Use common instead.
|
|
39
|
-
// Number of replicas
|
|
40
|
-
int32 replicas = 1 [deprecated = true];
|
|
41
|
-
|
|
42
|
-
// Image used for the replica group
|
|
43
|
-
string image = 2 [deprecated = true];
|
|
44
|
-
|
|
45
|
-
// Resources required for the replica group
|
|
46
|
-
core.Resources resources = 3 [deprecated = true];
|
|
47
|
-
|
|
48
|
-
// Restart policy determines whether pods will be restarted when they exit
|
|
49
|
-
RestartPolicy restart_policy = 4 [deprecated = true];
|
|
50
|
-
|
|
51
|
-
// The common replica spec
|
|
52
|
-
CommonReplicaSpec common = 5;
|
|
53
|
-
}
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins.kubeflow;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins/kubeflow";
|
|
6
|
-
|
|
7
|
-
import "flyteidl/core/tasks.proto";
|
|
8
|
-
import "flyteidl/plugins/kubeflow/common.proto";
|
|
9
|
-
|
|
10
|
-
// Proto for plugin that enables distributed training using https://github.com/kubeflow/tf-operator
|
|
11
|
-
message DistributedTensorflowTrainingTask {
|
|
12
|
-
// Worker replicas spec
|
|
13
|
-
DistributedTensorflowTrainingReplicaSpec worker_replicas = 1;
|
|
14
|
-
|
|
15
|
-
// Parameter server replicas spec
|
|
16
|
-
DistributedTensorflowTrainingReplicaSpec ps_replicas = 2;
|
|
17
|
-
|
|
18
|
-
// Chief replicas spec
|
|
19
|
-
DistributedTensorflowTrainingReplicaSpec chief_replicas = 3;
|
|
20
|
-
|
|
21
|
-
// RunPolicy encapsulates various runtime policies of the distributed training
|
|
22
|
-
// job, for example how to clean up resources and how long the job can stay
|
|
23
|
-
// active.
|
|
24
|
-
RunPolicy run_policy = 4;
|
|
25
|
-
|
|
26
|
-
// Evaluator replicas spec
|
|
27
|
-
DistributedTensorflowTrainingReplicaSpec evaluator_replicas = 5;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
message DistributedTensorflowTrainingReplicaSpec {
|
|
31
|
-
// 1~4 deprecated. Use common instead.
|
|
32
|
-
// Number of replicas
|
|
33
|
-
int32 replicas = 1 [deprecated = true];
|
|
34
|
-
|
|
35
|
-
// Image used for the replica group
|
|
36
|
-
string image = 2 [deprecated = true];
|
|
37
|
-
|
|
38
|
-
// Resources required for the replica group
|
|
39
|
-
core.Resources resources = 3 [deprecated = true];
|
|
40
|
-
|
|
41
|
-
// Restart policy determines whether pods will be restarted when they exit
|
|
42
|
-
RestartPolicy restart_policy = 4 [deprecated = true];
|
|
43
|
-
|
|
44
|
-
// The common replica spec
|
|
45
|
-
CommonReplicaSpec common = 5;
|
|
46
|
-
}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
// MPI operator proposal https://github.com/kubeflow/community/blob/master/proposals/mpi-operator-proposal.md
|
|
8
|
-
// Custom proto for plugin that enables distributed training using https://github.com/kubeflow/mpi-operator
|
|
9
|
-
message DistributedMPITrainingTask {
|
|
10
|
-
// number of worker spawned in the cluster for this job
|
|
11
|
-
int32 num_workers = 1;
|
|
12
|
-
|
|
13
|
-
// number of launcher replicas spawned in the cluster for this job
|
|
14
|
-
// The launcher pod invokes mpirun and communicates with worker pods through MPI.
|
|
15
|
-
int32 num_launcher_replicas = 2;
|
|
16
|
-
|
|
17
|
-
// number of slots per worker used in hostfile.
|
|
18
|
-
// The available slots (GPUs) in each pod.
|
|
19
|
-
int32 slots = 3;
|
|
20
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
syntax = "proto3";
|
|
2
|
-
|
|
3
|
-
package flyteidl.plugins;
|
|
4
|
-
|
|
5
|
-
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
|
-
|
|
7
|
-
// This message works with the 'presto' task type in the SDK and is the object that will be in the 'custom' field
|
|
8
|
-
// of a Presto task's TaskTemplate
|
|
9
|
-
message PrestoQuery {
|
|
10
|
-
string routing_group = 1;
|
|
11
|
-
string catalog = 2;
|
|
12
|
-
string schema = 3;
|
|
13
|
-
string statement = 4;
|
|
14
|
-
}
|