@flyteorg/flyteidl 1.5.21 → 1.10.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/gen/pb-js/flyteidl.d.ts +6480 -4250
  2. package/gen/pb-js/flyteidl.js +15117 -9702
  3. package/package.json +1 -1
  4. package/protos/buf.lock +7 -0
  5. package/protos/buf.yaml +1 -0
  6. package/protos/docs/admin/admin.rst +4 -4
  7. package/protos/docs/datacatalog/datacatalog.rst +1 -1
  8. package/protos/docs/service/index.rst +1 -1
  9. package/protos/flyteidl/admin/agent.proto +95 -6
  10. package/protos/flyteidl/admin/cluster_assignment.proto +1 -1
  11. package/protos/flyteidl/admin/common.proto +9 -1
  12. package/protos/flyteidl/admin/description_entity.proto +1 -1
  13. package/protos/flyteidl/admin/event.proto +1 -1
  14. package/protos/flyteidl/admin/execution.proto +10 -1
  15. package/protos/flyteidl/admin/launch_plan.proto +12 -4
  16. package/protos/flyteidl/admin/matchable_resource.proto +9 -3
  17. package/protos/flyteidl/admin/node_execution.proto +13 -1
  18. package/protos/flyteidl/admin/notification.proto +1 -1
  19. package/protos/flyteidl/admin/project.proto +7 -1
  20. package/protos/flyteidl/admin/project_attributes.proto +10 -1
  21. package/protos/flyteidl/admin/project_domain_attributes.proto +11 -2
  22. package/protos/flyteidl/admin/schedule.proto +1 -1
  23. package/protos/flyteidl/admin/signal.proto +2 -2
  24. package/protos/flyteidl/admin/task.proto +1 -1
  25. package/protos/flyteidl/admin/task_execution.proto +1 -1
  26. package/protos/flyteidl/admin/version.proto +1 -1
  27. package/protos/flyteidl/admin/workflow.proto +2 -2
  28. package/protos/flyteidl/admin/workflow_attributes.proto +10 -1
  29. package/protos/flyteidl/core/artifact_id.proto +90 -0
  30. package/protos/flyteidl/core/catalog.proto +3 -1
  31. package/protos/flyteidl/core/compiler.proto +12 -1
  32. package/protos/flyteidl/core/condition.proto +1 -1
  33. package/protos/flyteidl/core/dynamic_job.proto +1 -1
  34. package/protos/flyteidl/core/errors.proto +1 -1
  35. package/protos/flyteidl/core/execution.proto +1 -1
  36. package/protos/flyteidl/core/identifier.proto +7 -1
  37. package/protos/flyteidl/core/interface.proto +14 -1
  38. package/protos/flyteidl/core/literals.proto +4 -1
  39. package/protos/flyteidl/core/metrics.proto +15 -1
  40. package/protos/flyteidl/core/security.proto +1 -1
  41. package/protos/flyteidl/core/tasks.proto +33 -1
  42. package/protos/flyteidl/core/types.proto +7 -1
  43. package/protos/flyteidl/core/workflow.proto +28 -1
  44. package/protos/flyteidl/core/workflow_closure.proto +1 -1
  45. package/protos/flyteidl/datacatalog/datacatalog.proto +10 -2
  46. package/protos/flyteidl/event/cloudevents.proto +73 -0
  47. package/protos/flyteidl/event/event.proto +5 -1
  48. package/protos/flyteidl/plugins/array_job.proto +1 -1
  49. package/protos/flyteidl/plugins/dask.proto +1 -1
  50. package/protos/flyteidl/plugins/kubeflow/common.proto +1 -1
  51. package/protos/flyteidl/plugins/kubeflow/mpi.proto +1 -1
  52. package/protos/flyteidl/plugins/kubeflow/pytorch.proto +1 -1
  53. package/protos/flyteidl/plugins/kubeflow/tensorflow.proto +4 -1
  54. package/protos/flyteidl/plugins/mpi.proto +1 -1
  55. package/protos/flyteidl/plugins/presto.proto +1 -1
  56. package/protos/flyteidl/plugins/pytorch.proto +1 -1
  57. package/protos/flyteidl/plugins/qubole.proto +1 -1
  58. package/protos/flyteidl/plugins/ray.proto +7 -1
  59. package/protos/flyteidl/plugins/spark.proto +1 -1
  60. package/protos/flyteidl/plugins/tensorflow.proto +6 -2
  61. package/protos/flyteidl/plugins/waitable.proto +1 -1
  62. package/protos/flyteidl/service/admin.proto +398 -197
  63. package/protos/flyteidl/service/agent.proto +32 -2
  64. package/protos/flyteidl/service/auth.proto +8 -8
  65. package/protos/flyteidl/service/dataproxy.proto +12 -12
  66. package/protos/flyteidl/service/external_plugin_service.proto +1 -2
  67. package/protos/flyteidl/service/identity.proto +5 -5
  68. package/protos/flyteidl/service/signal.proto +30 -23
  69. package/protos/flyteidl/plugins/sagemaker/hyperparameter_tuning_job.proto +0 -83
  70. package/protos/flyteidl/plugins/sagemaker/parameter_ranges.proto +0 -61
  71. package/protos/flyteidl/plugins/sagemaker/training_job.proto +0 -119
@@ -1,7 +1,7 @@
1
1
  syntax = "proto3";
2
2
 
3
3
  package flyteidl.admin;
4
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
4
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin";
5
5
 
6
6
  import "flyteidl/admin/matchable_resource.proto";
7
7
 
@@ -18,6 +18,9 @@ message WorkflowAttributes {
18
18
  string workflow = 3;
19
19
 
20
20
  MatchingAttributes matching_attributes = 4;
21
+
22
+ // Optional, org key applied to the attributes.
23
+ string org = 5;
21
24
  }
22
25
 
23
26
  // Sets custom attributes for a project, domain and workflow combination.
@@ -48,6 +51,9 @@ message WorkflowAttributesGetRequest {
48
51
  // Which type of matchable attributes to return.
49
52
  // +required
50
53
  MatchableResource resource_type = 4;
54
+
55
+ // Optional, org key applied to the attributes.
56
+ string org = 5;
51
57
  }
52
58
 
53
59
  // Response to get an individual workflow attribute override.
@@ -73,6 +79,9 @@ message WorkflowAttributesDeleteRequest {
73
79
  // Which type of matchable attributes to delete.
74
80
  // +required
75
81
  MatchableResource resource_type = 4;
82
+
83
+ // Optional, org key applied to the attributes.
84
+ string org = 5;
76
85
  }
77
86
 
78
87
  // Purposefully empty, may be populated in the future.
@@ -0,0 +1,90 @@
1
+ syntax = "proto3";
2
+
3
+ package flyteidl.core;
4
+
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
+
7
+ import "google/protobuf/timestamp.proto";
8
+ import "flyteidl/core/identifier.proto";
9
+
10
+
11
+ message ArtifactKey {
12
+ // Project and domain and suffix needs to be unique across a given artifact store.
13
+ string project = 1;
14
+ string domain = 2;
15
+ string name = 3;
16
+ }
17
+
18
+ // Only valid for triggers
19
+ message ArtifactBindingData {
20
+ uint32 index = 1;
21
+
22
+ // These two fields are only relevant in the partition value case
23
+ oneof partition_data {
24
+ string partition_key = 2;
25
+ bool bind_to_time_partition = 3;
26
+ }
27
+
28
+ // This is only relevant in the time partition case
29
+ string transform = 4;
30
+ }
31
+
32
+ message InputBindingData {
33
+ string var = 1;
34
+ }
35
+
36
+ message LabelValue {
37
+ oneof value {
38
+ // The string static value is for use in the Partitions object
39
+ string static_value = 1;
40
+
41
+ // The time value is for use in the TimePartition case
42
+ google.protobuf.Timestamp time_value = 2;
43
+ ArtifactBindingData triggered_binding = 3;
44
+ InputBindingData input_binding = 4;
45
+ }
46
+ }
47
+
48
+ message Partitions {
49
+ map<string, LabelValue> value = 1;
50
+ }
51
+
52
+ message TimePartition {
53
+ LabelValue value = 1;
54
+ }
55
+
56
+ message ArtifactID {
57
+ ArtifactKey artifact_key = 1;
58
+
59
+ string version = 2;
60
+
61
+ // Think of a partition as a tag on an Artifact, except it's a key-value pair.
62
+ // Different partitions naturally have different versions (execution ids).
63
+ Partitions partitions = 3;
64
+
65
+ // There is no such thing as an empty time partition - if it's not set, then there is no time partition.
66
+ TimePartition time_partition = 4;
67
+ }
68
+
69
+ message ArtifactTag {
70
+ ArtifactKey artifact_key = 1;
71
+
72
+ LabelValue value = 2;
73
+ }
74
+
75
+ // Uniqueness constraints for Artifacts
76
+ // - project, domain, name, version, partitions
77
+ // Option 2 (tags are standalone, point to an individual artifact id):
78
+ // - project, domain, name, alias (points to one partition if partitioned)
79
+ // - project, domain, name, partition key, partition value
80
+ message ArtifactQuery {
81
+ oneof identifier {
82
+ ArtifactID artifact_id = 1;
83
+ ArtifactTag artifact_tag = 2;
84
+ string uri = 3;
85
+
86
+ // This is used in the trigger case, where a user specifies a value for an input that is one of the triggering
87
+ // artifacts, or a partition value derived from a triggering artifact.
88
+ ArtifactBindingData binding = 4;
89
+ }
90
+ }
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/identifier.proto";
8
8
 
@@ -22,6 +22,8 @@ enum CatalogCacheStatus {
22
22
  CACHE_PUT_FAILURE = 5;
23
23
  // Used to indicate the cache lookup was skipped
24
24
  CACHE_SKIPPED = 6;
25
+ // Used to indicate that the cache was evicted
26
+ CACHE_EVICTED = 7;
25
27
  };
26
28
 
27
29
  message CatalogArtifactTag {
@@ -2,8 +2,10 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
+ import "flyteidl/core/identifier.proto";
8
+ import "flyteidl/core/interface.proto";
7
9
  import "flyteidl/core/workflow.proto";
8
10
  import "flyteidl/core/tasks.proto";
9
11
 
@@ -29,6 +31,12 @@ message CompiledWorkflow {
29
31
  ConnectionSet connections = 2;
30
32
  }
31
33
 
34
+ // Output of the compilation step. This object represents one LaunchPlan. We store more metadata at this layer
35
+ message CompiledLaunchPlan {
36
+ // Completely contained LaunchPlan Template
37
+ LaunchPlanTemplate template = 1;
38
+ }
39
+
32
40
  // Output of the Compilation step. This object represent one Task. We store more metadata at this layer
33
41
  message CompiledTask {
34
42
  // Completely contained TaskTemplate
@@ -50,4 +58,7 @@ message CompiledWorkflowClosure {
50
58
  // Guaranteed that there will only exist one and only one task with a given id, i.e., every task has a unique id
51
59
  //+required (at least 1)
52
60
  repeated CompiledTask tasks = 3;
61
+ // A collection of launch plans that are compiled. Guaranteed that there will only exist one and only one launch plan
62
+ // with a given id, i.e., every launch plan has a unique id.
63
+ repeated CompiledLaunchPlan launch_plans = 4;
53
64
  }
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/literals.proto";
8
8
 
@@ -6,7 +6,7 @@ import "flyteidl/core/literals.proto";
6
6
 
7
7
  package flyteidl.core;
8
8
 
9
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
9
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
10
10
 
11
11
  // Describes a set of tasks to execute and how the final outputs are produced.
12
12
  message DynamicJobSpec {
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/execution.proto";
8
8
 
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "google/protobuf/duration.proto";
8
8
 
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  // Indicates a resource type within Flyte.
8
8
  enum ResourceType {
@@ -33,6 +33,9 @@ message Identifier {
33
33
 
34
34
  // Specific version of the resource.
35
35
  string version = 5;
36
+
37
+ // Optional, org key applied to the resource.
38
+ string org = 6;
36
39
  }
37
40
 
38
41
  // Encapsulation of fields that uniquely identifies a Flyte workflow execution
@@ -46,6 +49,9 @@ message WorkflowExecutionIdentifier {
46
49
 
47
50
  // User or system provided value for the resource.
48
51
  string name = 4;
52
+
53
+ // Optional, org key applied to the resource.
54
+ string org = 5;
49
55
  }
50
56
 
51
57
  // Encapsulation of fields that identify a Flyte node execution entity.
@@ -2,10 +2,11 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/types.proto";
8
8
  import "flyteidl/core/literals.proto";
9
+ import "flyteidl/core/artifact_id.proto";
9
10
 
10
11
  // Defines a strongly typed variable.
11
12
  message Variable {
@@ -14,6 +15,12 @@ message Variable {
14
15
 
15
16
  //+optional string describing input variable
16
17
  string description = 2;
18
+
19
+ //+optional This object allows the user to specify how Artifacts are created.
20
+ // name, tag, partitions can be specified. The other fields (version and project/domain) are ignored.
21
+ core.ArtifactID artifact_partial_id = 3;
22
+
23
+ core.ArtifactTag artifact_tag = 4;
17
24
  }
18
25
 
19
26
  // A map of Variables
@@ -41,6 +48,12 @@ message Parameter {
41
48
 
42
49
  //+optional, is this value required to be filled.
43
50
  bool required = 3;
51
+
52
+ // This is an execution time search basically that should result in exactly one Artifact with a Type that
53
+ // matches the type of the variable.
54
+ core.ArtifactQuery artifact_query = 4;
55
+
56
+ core.ArtifactID artifact_id = 5;
44
57
  }
45
58
  }
46
59
 
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "google/protobuf/timestamp.proto";
8
8
  import "google/protobuf/duration.proto";
@@ -108,6 +108,9 @@ message Literal {
108
108
  // This is used for caching purposes. For more details refer to RFC 1893
109
109
  // (https://github.com/flyteorg/flyte/blob/master/rfc/system/1893-caching-of-offloaded-objects.md)
110
110
  string hash = 4;
111
+
112
+ // Additional metadata for literals.
113
+ map<string, string> metadata = 5;
111
114
  }
112
115
 
113
116
  // A collection of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
@@ -2,10 +2,11 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/identifier.proto";
8
8
  import "google/protobuf/timestamp.proto";
9
+ import "google/protobuf/struct.proto";
9
10
 
10
11
  // Span represents a duration trace of Flyte execution. The id field denotes a Flyte execution entity or an operation
11
12
  // which uniquely identifies the Span. The spans attribute allows this Span to be further broken down into more
@@ -34,3 +35,16 @@ message Span {
34
35
  // spans defines a collection of Spans that breakdown this execution.
35
36
  repeated Span spans = 7;
36
37
  }
38
+
39
+ // ExecutionMetrics is a collection of metrics that are collected during the execution of a Flyte task.
40
+ message ExecutionMetricResult {
41
+ // The metric this data represents. e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG.
42
+ string metric = 1;
43
+
44
+ // The result data in prometheus range query result format
45
+ // https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats.
46
+ // This may include multiple time series, differentiated by their metric labels.
47
+ // Start time is greater of (execution attempt start, 48h ago)
48
+ // End time is lesser of (execution attempt end, now)
49
+ google.protobuf.Struct data = 2;
50
+ }
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  // Secret encapsulates information about the secret a task needs to proceed. An environment variable
8
8
  // FLYTE_SECRETS_ENV_PREFIX will be passed to indicate the prefix of the environment variables that will be present if
@@ -9,7 +9,7 @@ import "google/protobuf/struct.proto";
9
9
 
10
10
  package flyteidl.core;
11
11
 
12
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
12
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
13
13
 
14
14
  // A customizable interface to convey resources requested for a container. This can be interpreted differently for different
15
15
  // container engines.
@@ -43,6 +43,31 @@ message Resources {
43
43
  repeated ResourceEntry limits = 2;
44
44
  }
45
45
 
46
+ // Metadata associated with the GPU accelerator to allocate to a task. Contains
47
+ // information about device type, and for multi-instance GPUs, the partition size to
48
+ // use.
49
+ message GPUAccelerator {
50
+ // This can be any arbitrary string, and should be informed by the labels or taints
51
+ // associated with the nodes in question. Default cloud provider labels typically
52
+ // use the following values: `nvidia-tesla-t4`, `nvidia-tesla-a100`, etc.
53
+ string device = 1;
54
+ oneof partition_size_value {
55
+ bool unpartitioned = 2;
56
+ // Like `device`, this can be any arbitrary string, and should be informed by
57
+ // the labels or taints associated with the nodes in question. Default cloud
58
+ // provider labels typically use the following values: `1g.5gb`, `2g.10gb`, etc.
59
+ string partition_size = 3;
60
+ }
61
+ }
62
+
63
+ // Encapsulates all non-standard resources, not captured by v1.ResourceRequirements, to
64
+ // allocate to a task.
65
+ message ExtendedResources {
66
+ // GPU accelerator to select for task. Contains information about device type, and
67
+ // for multi-instance GPUs, the partition size to use.
68
+ GPUAccelerator gpu_accelerator = 1;
69
+ }
70
+
46
71
  // Runtime information. This is loosely defined to allow for extensibility.
47
72
  message RuntimeMetadata {
48
73
  enum RuntimeType {
@@ -106,6 +131,9 @@ message TaskMetadata {
106
131
  // task creates a k8s Pod. If this value is set, the specified PodTemplate will be used instead of, but applied
107
132
  // identically as, the default PodTemplate configured in FlytePropeller.
108
133
  string pod_template_name = 12;
134
+
135
+ // cache_ignore_input_vars is the input variables that should not be included when calculating hash for cache.
136
+ repeated string cache_ignore_input_vars = 13;
109
137
  }
110
138
 
111
139
  // A Task structure that uniquely identifies a task in the system
@@ -144,6 +172,10 @@ message TaskTemplate {
144
172
  // security_context encapsulates security attributes requested to run this task.
145
173
  SecurityContext security_context = 8;
146
174
 
175
+ // Encapsulates all non-standard resources, not captured by
176
+ // v1.ResourceRequirements, to allocate to a task.
177
+ ExtendedResources extended_resources = 9;
178
+
147
179
  // Metadata about the custom defined for this task. This is extensible to allow various plugins in the system
148
180
  // to use as required.
149
181
  // reserve the field numbers 1 through 15 for very frequently occurring message elements
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "google/protobuf/struct.proto";
8
8
 
@@ -114,6 +114,12 @@ message UnionType {
114
114
  message TypeStructure {
115
115
  // Must exactly match for types to be castable
116
116
  string tag = 1;
117
+ // dataclass_type only exists for dataclasses.
118
+ // This is used to resolve the type of the fields of dataclass
119
+ // The key is the field name, and the value is the literal type of the field
120
+ // e.g. For dataclass Foo, with fields a, and a is a string
121
+ // Foo.a will be resolved as a literal type of string from dataclass_type
122
+ map<string, LiteralType> dataclass_type = 2;
117
123
  }
118
124
 
119
125
  // TypeAnnotation encapsulates registration time information about a type. This can be used for various control-plane operations. TypeAnnotation will not be available at runtime when a task runs.
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/condition.proto";
8
8
  import "flyteidl/core/execution.proto";
@@ -149,6 +149,20 @@ message NodeMetadata {
149
149
  bool interruptible = 6;
150
150
  };
151
151
 
152
+ // Identify whether a node should have it's outputs cached.
153
+ oneof cacheable_value {
154
+ bool cacheable = 7;
155
+ }
156
+
157
+ // The version of the cache to use.
158
+ oneof cache_version_value {
159
+ string cache_version = 8;
160
+ }
161
+
162
+ // Identify whether caching operations involving this node should be serialized.
163
+ oneof cache_serializable_value {
164
+ bool cache_serializable = 9;
165
+ }
152
166
  }
153
167
 
154
168
  // Links a variable to an alias.
@@ -279,4 +293,17 @@ message WorkflowTemplate {
279
293
  message TaskNodeOverrides {
280
294
  // A customizable interface to convey resources requested for a task container.
281
295
  Resources resources = 1;
296
+
297
+ // Overrides for all non-standard resources, not captured by
298
+ // v1.ResourceRequirements, to allocate to a task.
299
+ ExtendedResources extended_resources = 2;
300
+ }
301
+
302
+ // A structure that uniquely identifies a launch plan in the system.
303
+ message LaunchPlanTemplate {
304
+ // A globally unique identifier for the launch plan.
305
+ Identifier id = 1;
306
+
307
+ // The input and output interface for the launch plan
308
+ TypedInterface interface = 2;
282
309
  }
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.core;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
6
6
 
7
7
  import "flyteidl/core/workflow.proto";
8
8
  import "flyteidl/core/tasks.proto";
@@ -6,7 +6,7 @@ import "flyteidl/core/literals.proto";
6
6
  import "google/protobuf/duration.proto";
7
7
  import "google/protobuf/timestamp.proto";
8
8
 
9
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/datacatalog";
9
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/datacatalog";
10
10
 
11
11
  /*
12
12
  * Data Catalog service definition
@@ -189,6 +189,9 @@ message UpdateArtifactRequest {
189
189
  // List of data to overwrite stored artifact data with. Must contain ALL data for updated Artifact as any missing
190
190
  // ArtifactData entries will be removed from the underlying blob storage and database.
191
191
  repeated ArtifactData data = 4;
192
+
193
+ // Update execution metadata(including execution domain, name, node, project data) when overwriting cache
194
+ Metadata metadata = 5;
192
195
  }
193
196
 
194
197
  /*
@@ -210,7 +213,7 @@ message ReservationID {
210
213
  string tag_name = 2;
211
214
  }
212
215
 
213
- // Try to acquire or extend an artifact reservation. If an active reservation exists, retreive that instance.
216
+ // Try to acquire or extend an artifact reservation. If an active reservation exists, retrieve that instance.
214
217
  message GetOrExtendReservationRequest {
215
218
  // The unique ID for the reservation
216
219
  ReservationID reservation_id = 1;
@@ -286,6 +289,9 @@ message DatasetID {
286
289
  string domain = 3; // The domain (eg. environment)
287
290
  string version = 4; // Version of the data schema
288
291
  string UUID = 5; // UUID for the dataset (if set the above fields are optional)
292
+
293
+ // Optional, org key applied to the resource.
294
+ string org = 6;
289
295
  }
290
296
 
291
297
  /*
@@ -383,6 +389,8 @@ message DatasetPropertyFilter {
383
389
  string name = 2;
384
390
  string domain = 3;
385
391
  string version = 4;
392
+ // Optional, org key applied to the dataset.
393
+ string org = 5;
386
394
  }
387
395
  }
388
396
 
@@ -0,0 +1,73 @@
1
+ syntax = "proto3";
2
+
3
+ package flyteidl.event;
4
+
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event";
6
+
7
+ import "flyteidl/event/event.proto";
8
+ import "flyteidl/core/literals.proto";
9
+ import "flyteidl/core/interface.proto";
10
+ import "flyteidl/core/artifact_id.proto";
11
+ import "flyteidl/core/identifier.proto";
12
+ import "google/protobuf/timestamp.proto";
13
+
14
+ // This is the cloud event parallel to the raw WorkflowExecutionEvent message. It's filled in with additional
15
+ // information that downstream consumers may find useful.
16
+ message CloudEventWorkflowExecution {
17
+ event.WorkflowExecutionEvent raw_event = 1;
18
+
19
+ core.TypedInterface output_interface = 2;
20
+
21
+ // The following are ExecutionMetadata fields
22
+ // We can't have the ExecutionMetadata object directly because of import cycle
23
+ repeated core.ArtifactID artifact_ids = 3;
24
+ core.WorkflowExecutionIdentifier reference_execution = 4;
25
+ string principal = 5;
26
+
27
+ // The ID of the LP that generated the execution that generated the Artifact.
28
+ // Here for provenance information.
29
+ // Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
30
+ core.Identifier launch_plan_id = 6;
31
+ }
32
+
33
+ message CloudEventNodeExecution {
34
+ event.NodeExecutionEvent raw_event = 1;
35
+
36
+ // The relevant task execution if applicable
37
+ core.TaskExecutionIdentifier task_exec_id = 2;
38
+
39
+ // The typed interface for the task that produced the event.
40
+ core.TypedInterface output_interface = 3;
41
+
42
+ // The following are ExecutionMetadata fields
43
+ // We can't have the ExecutionMetadata object directly because of import cycle
44
+ repeated core.ArtifactID artifact_ids = 4;
45
+ string principal = 5;
46
+
47
+ // The ID of the LP that generated the execution that generated the Artifact.
48
+ // Here for provenance information.
49
+ // Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
50
+ core.Identifier launch_plan_id = 6;
51
+ }
52
+
53
+ message CloudEventTaskExecution {
54
+ event.TaskExecutionEvent raw_event = 1;
55
+ }
56
+
57
+ // This event is to be sent by Admin after it creates an execution.
58
+ message CloudEventExecutionStart {
59
+ // The execution created.
60
+ core.WorkflowExecutionIdentifier execution_id = 1;
61
+ // The launch plan used.
62
+ core.Identifier launch_plan_id = 2;
63
+
64
+ core.Identifier workflow_id = 3;
65
+
66
+ // Artifact inputs to the workflow execution for which we have the full Artifact ID. These are likely the result of artifact queries that are run.
67
+ repeated core.ArtifactID artifact_ids = 4;
68
+
69
+ // Artifact inputs to the workflow execution for which we only have the tracking bit that's installed into the Literal's metadata by the Artifact service.
70
+ repeated string artifact_trackers = 5;
71
+
72
+ string principal = 6;
73
+ }
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.event;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event";
6
6
 
7
7
  import "flyteidl/core/literals.proto";
8
8
  import "flyteidl/core/compiler.proto";
@@ -12,6 +12,7 @@ import "flyteidl/core/catalog.proto";
12
12
  import "google/protobuf/timestamp.proto";
13
13
  import "google/protobuf/struct.proto";
14
14
 
15
+
15
16
  message WorkflowExecutionEvent {
16
17
  // Workflow execution id
17
18
  core.WorkflowExecutionIdentifier execution_id = 1;
@@ -110,6 +111,9 @@ message NodeExecutionEvent {
110
111
  // literal inputs are initially copied. The event however will not be sent until after the copy completes.
111
112
  // Extracting both of these timestamps facilitates a more accurate portrayal of the evaluation time-series.
112
113
  google.protobuf.Timestamp reported_at = 21;
114
+
115
+ // Indicates if this node is an ArrayNode.
116
+ bool is_array = 22;
113
117
  }
114
118
 
115
119
  // For Workflow Nodes we need to send information about the workflow that's launched
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.plugins;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
6
6
 
7
7
  // Describes a job that can process independent pieces of data concurrently. Multiple copies of the runnable component
8
8
  // will be executed concurrently.
@@ -4,7 +4,7 @@ import "flyteidl/core/tasks.proto";
4
4
 
5
5
  package flyteidl.plugins;
6
6
 
7
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
7
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
8
8
 
9
9
 
10
10
  // Custom Proto for Dask Plugin.
@@ -2,7 +2,7 @@ syntax = "proto3";
2
2
 
3
3
  package flyteidl.plugins.kubeflow;
4
4
 
5
- option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
5
+ option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
6
6
 
7
7
 
8
8
  enum RestartPolicy {