@flyteorg/flyteidl 1.5.21 → 1.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gen/pb-js/flyteidl.d.ts +6480 -4250
- package/gen/pb-js/flyteidl.js +15117 -9702
- package/package.json +1 -1
- package/protos/buf.lock +7 -0
- package/protos/buf.yaml +1 -0
- package/protos/docs/admin/admin.rst +4 -4
- package/protos/docs/datacatalog/datacatalog.rst +1 -1
- package/protos/docs/service/index.rst +1 -1
- package/protos/flyteidl/admin/agent.proto +95 -6
- package/protos/flyteidl/admin/cluster_assignment.proto +1 -1
- package/protos/flyteidl/admin/common.proto +9 -1
- package/protos/flyteidl/admin/description_entity.proto +1 -1
- package/protos/flyteidl/admin/event.proto +1 -1
- package/protos/flyteidl/admin/execution.proto +10 -1
- package/protos/flyteidl/admin/launch_plan.proto +12 -4
- package/protos/flyteidl/admin/matchable_resource.proto +9 -3
- package/protos/flyteidl/admin/node_execution.proto +13 -1
- package/protos/flyteidl/admin/notification.proto +1 -1
- package/protos/flyteidl/admin/project.proto +7 -1
- package/protos/flyteidl/admin/project_attributes.proto +10 -1
- package/protos/flyteidl/admin/project_domain_attributes.proto +11 -2
- package/protos/flyteidl/admin/schedule.proto +1 -1
- package/protos/flyteidl/admin/signal.proto +2 -2
- package/protos/flyteidl/admin/task.proto +1 -1
- package/protos/flyteidl/admin/task_execution.proto +1 -1
- package/protos/flyteidl/admin/version.proto +1 -1
- package/protos/flyteidl/admin/workflow.proto +2 -2
- package/protos/flyteidl/admin/workflow_attributes.proto +10 -1
- package/protos/flyteidl/core/artifact_id.proto +90 -0
- package/protos/flyteidl/core/catalog.proto +3 -1
- package/protos/flyteidl/core/compiler.proto +12 -1
- package/protos/flyteidl/core/condition.proto +1 -1
- package/protos/flyteidl/core/dynamic_job.proto +1 -1
- package/protos/flyteidl/core/errors.proto +1 -1
- package/protos/flyteidl/core/execution.proto +1 -1
- package/protos/flyteidl/core/identifier.proto +7 -1
- package/protos/flyteidl/core/interface.proto +14 -1
- package/protos/flyteidl/core/literals.proto +4 -1
- package/protos/flyteidl/core/metrics.proto +15 -1
- package/protos/flyteidl/core/security.proto +1 -1
- package/protos/flyteidl/core/tasks.proto +33 -1
- package/protos/flyteidl/core/types.proto +7 -1
- package/protos/flyteidl/core/workflow.proto +28 -1
- package/protos/flyteidl/core/workflow_closure.proto +1 -1
- package/protos/flyteidl/datacatalog/datacatalog.proto +10 -2
- package/protos/flyteidl/event/cloudevents.proto +73 -0
- package/protos/flyteidl/event/event.proto +5 -1
- package/protos/flyteidl/plugins/array_job.proto +1 -1
- package/protos/flyteidl/plugins/dask.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/common.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/mpi.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/pytorch.proto +1 -1
- package/protos/flyteidl/plugins/kubeflow/tensorflow.proto +4 -1
- package/protos/flyteidl/plugins/mpi.proto +1 -1
- package/protos/flyteidl/plugins/presto.proto +1 -1
- package/protos/flyteidl/plugins/pytorch.proto +1 -1
- package/protos/flyteidl/plugins/qubole.proto +1 -1
- package/protos/flyteidl/plugins/ray.proto +7 -1
- package/protos/flyteidl/plugins/spark.proto +1 -1
- package/protos/flyteidl/plugins/tensorflow.proto +6 -2
- package/protos/flyteidl/plugins/waitable.proto +1 -1
- package/protos/flyteidl/service/admin.proto +398 -197
- package/protos/flyteidl/service/agent.proto +32 -2
- package/protos/flyteidl/service/auth.proto +8 -8
- package/protos/flyteidl/service/dataproxy.proto +12 -12
- package/protos/flyteidl/service/external_plugin_service.proto +1 -2
- package/protos/flyteidl/service/identity.proto +5 -5
- package/protos/flyteidl/service/signal.proto +30 -23
- package/protos/flyteidl/plugins/sagemaker/hyperparameter_tuning_job.proto +0 -83
- package/protos/flyteidl/plugins/sagemaker/parameter_ranges.proto +0 -61
- package/protos/flyteidl/plugins/sagemaker/training_job.proto +0 -119
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
syntax = "proto3";
|
|
2
2
|
|
|
3
3
|
package flyteidl.admin;
|
|
4
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
|
|
4
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin";
|
|
5
5
|
|
|
6
6
|
import "flyteidl/admin/matchable_resource.proto";
|
|
7
7
|
|
|
@@ -18,6 +18,9 @@ message WorkflowAttributes {
|
|
|
18
18
|
string workflow = 3;
|
|
19
19
|
|
|
20
20
|
MatchingAttributes matching_attributes = 4;
|
|
21
|
+
|
|
22
|
+
// Optional, org key applied to the attributes.
|
|
23
|
+
string org = 5;
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
// Sets custom attributes for a project, domain and workflow combination.
|
|
@@ -48,6 +51,9 @@ message WorkflowAttributesGetRequest {
|
|
|
48
51
|
// Which type of matchable attributes to return.
|
|
49
52
|
// +required
|
|
50
53
|
MatchableResource resource_type = 4;
|
|
54
|
+
|
|
55
|
+
// Optional, org key applied to the attributes.
|
|
56
|
+
string org = 5;
|
|
51
57
|
}
|
|
52
58
|
|
|
53
59
|
// Response to get an individual workflow attribute override.
|
|
@@ -73,6 +79,9 @@ message WorkflowAttributesDeleteRequest {
|
|
|
73
79
|
// Which type of matchable attributes to delete.
|
|
74
80
|
// +required
|
|
75
81
|
MatchableResource resource_type = 4;
|
|
82
|
+
|
|
83
|
+
// Optional, org key applied to the attributes.
|
|
84
|
+
string org = 5;
|
|
76
85
|
}
|
|
77
86
|
|
|
78
87
|
// Purposefully empty, may be populated in the future.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
syntax = "proto3";
|
|
2
|
+
|
|
3
|
+
package flyteidl.core;
|
|
4
|
+
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
|
+
|
|
7
|
+
import "google/protobuf/timestamp.proto";
|
|
8
|
+
import "flyteidl/core/identifier.proto";
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
message ArtifactKey {
|
|
12
|
+
// Project and domain and suffix needs to be unique across a given artifact store.
|
|
13
|
+
string project = 1;
|
|
14
|
+
string domain = 2;
|
|
15
|
+
string name = 3;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// Only valid for triggers
|
|
19
|
+
message ArtifactBindingData {
|
|
20
|
+
uint32 index = 1;
|
|
21
|
+
|
|
22
|
+
// These two fields are only relevant in the partition value case
|
|
23
|
+
oneof partition_data {
|
|
24
|
+
string partition_key = 2;
|
|
25
|
+
bool bind_to_time_partition = 3;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// This is only relevant in the time partition case
|
|
29
|
+
string transform = 4;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
message InputBindingData {
|
|
33
|
+
string var = 1;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
message LabelValue {
|
|
37
|
+
oneof value {
|
|
38
|
+
// The string static value is for use in the Partitions object
|
|
39
|
+
string static_value = 1;
|
|
40
|
+
|
|
41
|
+
// The time value is for use in the TimePartition case
|
|
42
|
+
google.protobuf.Timestamp time_value = 2;
|
|
43
|
+
ArtifactBindingData triggered_binding = 3;
|
|
44
|
+
InputBindingData input_binding = 4;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
message Partitions {
|
|
49
|
+
map<string, LabelValue> value = 1;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
message TimePartition {
|
|
53
|
+
LabelValue value = 1;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
message ArtifactID {
|
|
57
|
+
ArtifactKey artifact_key = 1;
|
|
58
|
+
|
|
59
|
+
string version = 2;
|
|
60
|
+
|
|
61
|
+
// Think of a partition as a tag on an Artifact, except it's a key-value pair.
|
|
62
|
+
// Different partitions naturally have different versions (execution ids).
|
|
63
|
+
Partitions partitions = 3;
|
|
64
|
+
|
|
65
|
+
// There is no such thing as an empty time partition - if it's not set, then there is no time partition.
|
|
66
|
+
TimePartition time_partition = 4;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
message ArtifactTag {
|
|
70
|
+
ArtifactKey artifact_key = 1;
|
|
71
|
+
|
|
72
|
+
LabelValue value = 2;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Uniqueness constraints for Artifacts
|
|
76
|
+
// - project, domain, name, version, partitions
|
|
77
|
+
// Option 2 (tags are standalone, point to an individual artifact id):
|
|
78
|
+
// - project, domain, name, alias (points to one partition if partitioned)
|
|
79
|
+
// - project, domain, name, partition key, partition value
|
|
80
|
+
message ArtifactQuery {
|
|
81
|
+
oneof identifier {
|
|
82
|
+
ArtifactID artifact_id = 1;
|
|
83
|
+
ArtifactTag artifact_tag = 2;
|
|
84
|
+
string uri = 3;
|
|
85
|
+
|
|
86
|
+
// This is used in the trigger case, where a user specifies a value for an input that is one of the triggering
|
|
87
|
+
// artifacts, or a partition value derived from a triggering artifact.
|
|
88
|
+
ArtifactBindingData binding = 4;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/identifier.proto";
|
|
8
8
|
|
|
@@ -22,6 +22,8 @@ enum CatalogCacheStatus {
|
|
|
22
22
|
CACHE_PUT_FAILURE = 5;
|
|
23
23
|
// Used to indicate the cache lookup was skipped
|
|
24
24
|
CACHE_SKIPPED = 6;
|
|
25
|
+
// Used to indicate that the cache was evicted
|
|
26
|
+
CACHE_EVICTED = 7;
|
|
25
27
|
};
|
|
26
28
|
|
|
27
29
|
message CatalogArtifactTag {
|
|
@@ -2,8 +2,10 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
|
+
import "flyteidl/core/identifier.proto";
|
|
8
|
+
import "flyteidl/core/interface.proto";
|
|
7
9
|
import "flyteidl/core/workflow.proto";
|
|
8
10
|
import "flyteidl/core/tasks.proto";
|
|
9
11
|
|
|
@@ -29,6 +31,12 @@ message CompiledWorkflow {
|
|
|
29
31
|
ConnectionSet connections = 2;
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
// Output of the compilation step. This object represents one LaunchPlan. We store more metadata at this layer
|
|
35
|
+
message CompiledLaunchPlan {
|
|
36
|
+
// Completely contained LaunchPlan Template
|
|
37
|
+
LaunchPlanTemplate template = 1;
|
|
38
|
+
}
|
|
39
|
+
|
|
32
40
|
// Output of the Compilation step. This object represent one Task. We store more metadata at this layer
|
|
33
41
|
message CompiledTask {
|
|
34
42
|
// Completely contained TaskTemplate
|
|
@@ -50,4 +58,7 @@ message CompiledWorkflowClosure {
|
|
|
50
58
|
// Guaranteed that there will only exist one and only one task with a given id, i.e., every task has a unique id
|
|
51
59
|
//+required (at least 1)
|
|
52
60
|
repeated CompiledTask tasks = 3;
|
|
61
|
+
// A collection of launch plans that are compiled. Guaranteed that there will only exist one and only one launch plan
|
|
62
|
+
// with a given id, i.e., every launch plan has a unique id.
|
|
63
|
+
repeated CompiledLaunchPlan launch_plans = 4;
|
|
53
64
|
}
|
|
@@ -6,7 +6,7 @@ import "flyteidl/core/literals.proto";
|
|
|
6
6
|
|
|
7
7
|
package flyteidl.core;
|
|
8
8
|
|
|
9
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
9
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
10
10
|
|
|
11
11
|
// Describes a set of tasks to execute and how the final outputs are produced.
|
|
12
12
|
message DynamicJobSpec {
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
// Indicates a resource type within Flyte.
|
|
8
8
|
enum ResourceType {
|
|
@@ -33,6 +33,9 @@ message Identifier {
|
|
|
33
33
|
|
|
34
34
|
// Specific version of the resource.
|
|
35
35
|
string version = 5;
|
|
36
|
+
|
|
37
|
+
// Optional, org key applied to the resource.
|
|
38
|
+
string org = 6;
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
// Encapsulation of fields that uniquely identifies a Flyte workflow execution
|
|
@@ -46,6 +49,9 @@ message WorkflowExecutionIdentifier {
|
|
|
46
49
|
|
|
47
50
|
// User or system provided value for the resource.
|
|
48
51
|
string name = 4;
|
|
52
|
+
|
|
53
|
+
// Optional, org key applied to the resource.
|
|
54
|
+
string org = 5;
|
|
49
55
|
}
|
|
50
56
|
|
|
51
57
|
// Encapsulation of fields that identify a Flyte node execution entity.
|
|
@@ -2,10 +2,11 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/types.proto";
|
|
8
8
|
import "flyteidl/core/literals.proto";
|
|
9
|
+
import "flyteidl/core/artifact_id.proto";
|
|
9
10
|
|
|
10
11
|
// Defines a strongly typed variable.
|
|
11
12
|
message Variable {
|
|
@@ -14,6 +15,12 @@ message Variable {
|
|
|
14
15
|
|
|
15
16
|
//+optional string describing input variable
|
|
16
17
|
string description = 2;
|
|
18
|
+
|
|
19
|
+
//+optional This object allows the user to specify how Artifacts are created.
|
|
20
|
+
// name, tag, partitions can be specified. The other fields (version and project/domain) are ignored.
|
|
21
|
+
core.ArtifactID artifact_partial_id = 3;
|
|
22
|
+
|
|
23
|
+
core.ArtifactTag artifact_tag = 4;
|
|
17
24
|
}
|
|
18
25
|
|
|
19
26
|
// A map of Variables
|
|
@@ -41,6 +48,12 @@ message Parameter {
|
|
|
41
48
|
|
|
42
49
|
//+optional, is this value required to be filled.
|
|
43
50
|
bool required = 3;
|
|
51
|
+
|
|
52
|
+
// This is an execution time search basically that should result in exactly one Artifact with a Type that
|
|
53
|
+
// matches the type of the variable.
|
|
54
|
+
core.ArtifactQuery artifact_query = 4;
|
|
55
|
+
|
|
56
|
+
core.ArtifactID artifact_id = 5;
|
|
44
57
|
}
|
|
45
58
|
}
|
|
46
59
|
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "google/protobuf/timestamp.proto";
|
|
8
8
|
import "google/protobuf/duration.proto";
|
|
@@ -108,6 +108,9 @@ message Literal {
|
|
|
108
108
|
// This is used for caching purposes. For more details refer to RFC 1893
|
|
109
109
|
// (https://github.com/flyteorg/flyte/blob/master/rfc/system/1893-caching-of-offloaded-objects.md)
|
|
110
110
|
string hash = 4;
|
|
111
|
+
|
|
112
|
+
// Additional metadata for literals.
|
|
113
|
+
map<string, string> metadata = 5;
|
|
111
114
|
}
|
|
112
115
|
|
|
113
116
|
// A collection of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
|
|
@@ -2,10 +2,11 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/identifier.proto";
|
|
8
8
|
import "google/protobuf/timestamp.proto";
|
|
9
|
+
import "google/protobuf/struct.proto";
|
|
9
10
|
|
|
10
11
|
// Span represents a duration trace of Flyte execution. The id field denotes a Flyte execution entity or an operation
|
|
11
12
|
// which uniquely identifies the Span. The spans attribute allows this Span to be further broken down into more
|
|
@@ -34,3 +35,16 @@ message Span {
|
|
|
34
35
|
// spans defines a collection of Spans that breakdown this execution.
|
|
35
36
|
repeated Span spans = 7;
|
|
36
37
|
}
|
|
38
|
+
|
|
39
|
+
// ExecutionMetrics is a collection of metrics that are collected during the execution of a Flyte task.
|
|
40
|
+
message ExecutionMetricResult {
|
|
41
|
+
// The metric this data represents. e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG.
|
|
42
|
+
string metric = 1;
|
|
43
|
+
|
|
44
|
+
// The result data in prometheus range query result format
|
|
45
|
+
// https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats.
|
|
46
|
+
// This may include multiple time series, differentiated by their metric labels.
|
|
47
|
+
// Start time is greater of (execution attempt start, 48h ago)
|
|
48
|
+
// End time is lesser of (execution attempt end, now)
|
|
49
|
+
google.protobuf.Struct data = 2;
|
|
50
|
+
}
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
// Secret encapsulates information about the secret a task needs to proceed. An environment variable
|
|
8
8
|
// FLYTE_SECRETS_ENV_PREFIX will be passed to indicate the prefix of the environment variables that will be present if
|
|
@@ -9,7 +9,7 @@ import "google/protobuf/struct.proto";
|
|
|
9
9
|
|
|
10
10
|
package flyteidl.core;
|
|
11
11
|
|
|
12
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
12
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
13
13
|
|
|
14
14
|
// A customizable interface to convey resources requested for a container. This can be interpreted differently for different
|
|
15
15
|
// container engines.
|
|
@@ -43,6 +43,31 @@ message Resources {
|
|
|
43
43
|
repeated ResourceEntry limits = 2;
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
// Metadata associated with the GPU accelerator to allocate to a task. Contains
|
|
47
|
+
// information about device type, and for multi-instance GPUs, the partition size to
|
|
48
|
+
// use.
|
|
49
|
+
message GPUAccelerator {
|
|
50
|
+
// This can be any arbitrary string, and should be informed by the labels or taints
|
|
51
|
+
// associated with the nodes in question. Default cloud provider labels typically
|
|
52
|
+
// use the following values: `nvidia-tesla-t4`, `nvidia-tesla-a100`, etc.
|
|
53
|
+
string device = 1;
|
|
54
|
+
oneof partition_size_value {
|
|
55
|
+
bool unpartitioned = 2;
|
|
56
|
+
// Like `device`, this can be any arbitrary string, and should be informed by
|
|
57
|
+
// the labels or taints associated with the nodes in question. Default cloud
|
|
58
|
+
// provider labels typically use the following values: `1g.5gb`, `2g.10gb`, etc.
|
|
59
|
+
string partition_size = 3;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Encapsulates all non-standard resources, not captured by v1.ResourceRequirements, to
|
|
64
|
+
// allocate to a task.
|
|
65
|
+
message ExtendedResources {
|
|
66
|
+
// GPU accelerator to select for task. Contains information about device type, and
|
|
67
|
+
// for multi-instance GPUs, the partition size to use.
|
|
68
|
+
GPUAccelerator gpu_accelerator = 1;
|
|
69
|
+
}
|
|
70
|
+
|
|
46
71
|
// Runtime information. This is loosely defined to allow for extensibility.
|
|
47
72
|
message RuntimeMetadata {
|
|
48
73
|
enum RuntimeType {
|
|
@@ -106,6 +131,9 @@ message TaskMetadata {
|
|
|
106
131
|
// task creates a k8s Pod. If this value is set, the specified PodTemplate will be used instead of, but applied
|
|
107
132
|
// identically as, the default PodTemplate configured in FlytePropeller.
|
|
108
133
|
string pod_template_name = 12;
|
|
134
|
+
|
|
135
|
+
// cache_ignore_input_vars is the input variables that should not be included when calculating hash for cache.
|
|
136
|
+
repeated string cache_ignore_input_vars = 13;
|
|
109
137
|
}
|
|
110
138
|
|
|
111
139
|
// A Task structure that uniquely identifies a task in the system
|
|
@@ -144,6 +172,10 @@ message TaskTemplate {
|
|
|
144
172
|
// security_context encapsulates security attributes requested to run this task.
|
|
145
173
|
SecurityContext security_context = 8;
|
|
146
174
|
|
|
175
|
+
// Encapsulates all non-standard resources, not captured by
|
|
176
|
+
// v1.ResourceRequirements, to allocate to a task.
|
|
177
|
+
ExtendedResources extended_resources = 9;
|
|
178
|
+
|
|
147
179
|
// Metadata about the custom defined for this task. This is extensible to allow various plugins in the system
|
|
148
180
|
// to use as required.
|
|
149
181
|
// reserve the field numbers 1 through 15 for very frequently occurring message elements
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "google/protobuf/struct.proto";
|
|
8
8
|
|
|
@@ -114,6 +114,12 @@ message UnionType {
|
|
|
114
114
|
message TypeStructure {
|
|
115
115
|
// Must exactly match for types to be castable
|
|
116
116
|
string tag = 1;
|
|
117
|
+
// dataclass_type only exists for dataclasses.
|
|
118
|
+
// This is used to resolve the type of the fields of dataclass
|
|
119
|
+
// The key is the field name, and the value is the literal type of the field
|
|
120
|
+
// e.g. For dataclass Foo, with fields a, and a is a string
|
|
121
|
+
// Foo.a will be resolved as a literal type of string from dataclass_type
|
|
122
|
+
map<string, LiteralType> dataclass_type = 2;
|
|
117
123
|
}
|
|
118
124
|
|
|
119
125
|
// TypeAnnotation encapsulates registration time information about a type. This can be used for various control-plane operations. TypeAnnotation will not be available at runtime when a task runs.
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/condition.proto";
|
|
8
8
|
import "flyteidl/core/execution.proto";
|
|
@@ -149,6 +149,20 @@ message NodeMetadata {
|
|
|
149
149
|
bool interruptible = 6;
|
|
150
150
|
};
|
|
151
151
|
|
|
152
|
+
// Identify whether a node should have it's outputs cached.
|
|
153
|
+
oneof cacheable_value {
|
|
154
|
+
bool cacheable = 7;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// The version of the cache to use.
|
|
158
|
+
oneof cache_version_value {
|
|
159
|
+
string cache_version = 8;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Identify whether caching operations involving this node should be serialized.
|
|
163
|
+
oneof cache_serializable_value {
|
|
164
|
+
bool cache_serializable = 9;
|
|
165
|
+
}
|
|
152
166
|
}
|
|
153
167
|
|
|
154
168
|
// Links a variable to an alias.
|
|
@@ -279,4 +293,17 @@ message WorkflowTemplate {
|
|
|
279
293
|
message TaskNodeOverrides {
|
|
280
294
|
// A customizable interface to convey resources requested for a task container.
|
|
281
295
|
Resources resources = 1;
|
|
296
|
+
|
|
297
|
+
// Overrides for all non-standard resources, not captured by
|
|
298
|
+
// v1.ResourceRequirements, to allocate to a task.
|
|
299
|
+
ExtendedResources extended_resources = 2;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// A structure that uniquely identifies a launch plan in the system.
|
|
303
|
+
message LaunchPlanTemplate {
|
|
304
|
+
// A globally unique identifier for the launch plan.
|
|
305
|
+
Identifier id = 1;
|
|
306
|
+
|
|
307
|
+
// The input and output interface for the launch plan
|
|
308
|
+
TypedInterface interface = 2;
|
|
282
309
|
}
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.core;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/workflow.proto";
|
|
8
8
|
import "flyteidl/core/tasks.proto";
|
|
@@ -6,7 +6,7 @@ import "flyteidl/core/literals.proto";
|
|
|
6
6
|
import "google/protobuf/duration.proto";
|
|
7
7
|
import "google/protobuf/timestamp.proto";
|
|
8
8
|
|
|
9
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog";
|
|
9
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog";
|
|
10
10
|
|
|
11
11
|
/*
|
|
12
12
|
* Data Catalog service definition
|
|
@@ -189,6 +189,9 @@ message UpdateArtifactRequest {
|
|
|
189
189
|
// List of data to overwrite stored artifact data with. Must contain ALL data for updated Artifact as any missing
|
|
190
190
|
// ArtifactData entries will be removed from the underlying blob storage and database.
|
|
191
191
|
repeated ArtifactData data = 4;
|
|
192
|
+
|
|
193
|
+
// Update execution metadata(including execution domain, name, node, project data) when overwriting cache
|
|
194
|
+
Metadata metadata = 5;
|
|
192
195
|
}
|
|
193
196
|
|
|
194
197
|
/*
|
|
@@ -210,7 +213,7 @@ message ReservationID {
|
|
|
210
213
|
string tag_name = 2;
|
|
211
214
|
}
|
|
212
215
|
|
|
213
|
-
// Try to acquire or extend an artifact reservation. If an active reservation exists,
|
|
216
|
+
// Try to acquire or extend an artifact reservation. If an active reservation exists, retrieve that instance.
|
|
214
217
|
message GetOrExtendReservationRequest {
|
|
215
218
|
// The unique ID for the reservation
|
|
216
219
|
ReservationID reservation_id = 1;
|
|
@@ -286,6 +289,9 @@ message DatasetID {
|
|
|
286
289
|
string domain = 3; // The domain (eg. environment)
|
|
287
290
|
string version = 4; // Version of the data schema
|
|
288
291
|
string UUID = 5; // UUID for the dataset (if set the above fields are optional)
|
|
292
|
+
|
|
293
|
+
// Optional, org key applied to the resource.
|
|
294
|
+
string org = 6;
|
|
289
295
|
}
|
|
290
296
|
|
|
291
297
|
/*
|
|
@@ -383,6 +389,8 @@ message DatasetPropertyFilter {
|
|
|
383
389
|
string name = 2;
|
|
384
390
|
string domain = 3;
|
|
385
391
|
string version = 4;
|
|
392
|
+
// Optional, org key applied to the dataset.
|
|
393
|
+
string org = 5;
|
|
386
394
|
}
|
|
387
395
|
}
|
|
388
396
|
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
syntax = "proto3";
|
|
2
|
+
|
|
3
|
+
package flyteidl.event;
|
|
4
|
+
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event";
|
|
6
|
+
|
|
7
|
+
import "flyteidl/event/event.proto";
|
|
8
|
+
import "flyteidl/core/literals.proto";
|
|
9
|
+
import "flyteidl/core/interface.proto";
|
|
10
|
+
import "flyteidl/core/artifact_id.proto";
|
|
11
|
+
import "flyteidl/core/identifier.proto";
|
|
12
|
+
import "google/protobuf/timestamp.proto";
|
|
13
|
+
|
|
14
|
+
// This is the cloud event parallel to the raw WorkflowExecutionEvent message. It's filled in with additional
|
|
15
|
+
// information that downstream consumers may find useful.
|
|
16
|
+
message CloudEventWorkflowExecution {
|
|
17
|
+
event.WorkflowExecutionEvent raw_event = 1;
|
|
18
|
+
|
|
19
|
+
core.TypedInterface output_interface = 2;
|
|
20
|
+
|
|
21
|
+
// The following are ExecutionMetadata fields
|
|
22
|
+
// We can't have the ExecutionMetadata object directly because of import cycle
|
|
23
|
+
repeated core.ArtifactID artifact_ids = 3;
|
|
24
|
+
core.WorkflowExecutionIdentifier reference_execution = 4;
|
|
25
|
+
string principal = 5;
|
|
26
|
+
|
|
27
|
+
// The ID of the LP that generated the execution that generated the Artifact.
|
|
28
|
+
// Here for provenance information.
|
|
29
|
+
// Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
|
|
30
|
+
core.Identifier launch_plan_id = 6;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
message CloudEventNodeExecution {
|
|
34
|
+
event.NodeExecutionEvent raw_event = 1;
|
|
35
|
+
|
|
36
|
+
// The relevant task execution if applicable
|
|
37
|
+
core.TaskExecutionIdentifier task_exec_id = 2;
|
|
38
|
+
|
|
39
|
+
// The typed interface for the task that produced the event.
|
|
40
|
+
core.TypedInterface output_interface = 3;
|
|
41
|
+
|
|
42
|
+
// The following are ExecutionMetadata fields
|
|
43
|
+
// We can't have the ExecutionMetadata object directly because of import cycle
|
|
44
|
+
repeated core.ArtifactID artifact_ids = 4;
|
|
45
|
+
string principal = 5;
|
|
46
|
+
|
|
47
|
+
// The ID of the LP that generated the execution that generated the Artifact.
|
|
48
|
+
// Here for provenance information.
|
|
49
|
+
// Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
|
|
50
|
+
core.Identifier launch_plan_id = 6;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
message CloudEventTaskExecution {
|
|
54
|
+
event.TaskExecutionEvent raw_event = 1;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// This event is to be sent by Admin after it creates an execution.
|
|
58
|
+
message CloudEventExecutionStart {
|
|
59
|
+
// The execution created.
|
|
60
|
+
core.WorkflowExecutionIdentifier execution_id = 1;
|
|
61
|
+
// The launch plan used.
|
|
62
|
+
core.Identifier launch_plan_id = 2;
|
|
63
|
+
|
|
64
|
+
core.Identifier workflow_id = 3;
|
|
65
|
+
|
|
66
|
+
// Artifact inputs to the workflow execution for which we have the full Artifact ID. These are likely the result of artifact queries that are run.
|
|
67
|
+
repeated core.ArtifactID artifact_ids = 4;
|
|
68
|
+
|
|
69
|
+
// Artifact inputs to the workflow execution for which we only have the tracking bit that's installed into the Literal's metadata by the Artifact service.
|
|
70
|
+
repeated string artifact_trackers = 5;
|
|
71
|
+
|
|
72
|
+
string principal = 6;
|
|
73
|
+
}
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.event;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event";
|
|
6
6
|
|
|
7
7
|
import "flyteidl/core/literals.proto";
|
|
8
8
|
import "flyteidl/core/compiler.proto";
|
|
@@ -12,6 +12,7 @@ import "flyteidl/core/catalog.proto";
|
|
|
12
12
|
import "google/protobuf/timestamp.proto";
|
|
13
13
|
import "google/protobuf/struct.proto";
|
|
14
14
|
|
|
15
|
+
|
|
15
16
|
message WorkflowExecutionEvent {
|
|
16
17
|
// Workflow execution id
|
|
17
18
|
core.WorkflowExecutionIdentifier execution_id = 1;
|
|
@@ -110,6 +111,9 @@ message NodeExecutionEvent {
|
|
|
110
111
|
// literal inputs are initially copied. The event however will not be sent until after the copy completes.
|
|
111
112
|
// Extracting both of these timestamps facilitates a more accurate portrayal of the evaluation time-series.
|
|
112
113
|
google.protobuf.Timestamp reported_at = 21;
|
|
114
|
+
|
|
115
|
+
// Indicates if this node is an ArrayNode.
|
|
116
|
+
bool is_array = 22;
|
|
113
117
|
}
|
|
114
118
|
|
|
115
119
|
// For Workflow Nodes we need to send information about the workflow that's launched
|
|
@@ -2,7 +2,7 @@ syntax = "proto3";
|
|
|
2
2
|
|
|
3
3
|
package flyteidl.plugins;
|
|
4
4
|
|
|
5
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
5
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
6
6
|
|
|
7
7
|
// Describes a job that can process independent pieces of data concurrently. Multiple copies of the runnable component
|
|
8
8
|
// will be executed concurrently.
|
|
@@ -4,7 +4,7 @@ import "flyteidl/core/tasks.proto";
|
|
|
4
4
|
|
|
5
5
|
package flyteidl.plugins;
|
|
6
6
|
|
|
7
|
-
option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
7
|
+
option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
// Custom Proto for Dask Plugin.
|