@flyteorg/flyteidl 0.23.1 → 0.24.2

package/protos/docs/core/core.rst

@@ -1216,6 +1216,7 @@ Specifies either a simple value or a reference to another output.
    "collection", ":ref:`ref_flyteidl.core.BindingDataCollection`", "", "A collection of binding data. This allows nesting of binding data to any number of levels."
    "promise", ":ref:`ref_flyteidl.core.OutputReference`", "", "References an output promised by another node."
    "map", ":ref:`ref_flyteidl.core.BindingDataMap`", "", "A map of bindings. The key is always a string."
+   "union", ":ref:`ref_flyteidl.core.UnionInfo`", "", ""
 
 
 
@@ -1509,6 +1510,7 @@ Scalar
    "error", ":ref:`ref_flyteidl.core.Error`", "", ""
    "generic", ":ref:`ref_google.protobuf.Struct`", "", ""
    "structured_dataset", ":ref:`ref_flyteidl.core.StructuredDataset`", "", ""
+   "union", ":ref:`ref_flyteidl.core.Union`", "", ""
 
 
 
@@ -1581,6 +1583,49 @@ StructuredDatasetMetadata
 
 
 
+.. _ref_flyteidl.core.Union:
+
+Union
+------------------------------------------------------------------
+
+The runtime representation of a tagged union value. See `UnionType` for more details.
+
+
+
+.. csv-table:: Union type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "value", ":ref:`ref_flyteidl.core.Literal`", "", ""
+   "type", ":ref:`ref_flyteidl.core.LiteralType`", "", ""
+
+
+
+
+
+
+
+.. _ref_flyteidl.core.UnionInfo:
+
+UnionInfo
+------------------------------------------------------------------
+
+
+
+
+
+.. csv-table:: UnionInfo type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "targetType", ":ref:`ref_flyteidl.core.LiteralType`", "", ""
+
+
+
+
+
+
+
 .. _ref_flyteidl.core.Void:
 
 Void
@@ -2383,8 +2428,10 @@ Defines a strong type to allow type checking between interfaces.
    "blob", ":ref:`ref_flyteidl.core.BlobType`", "", "A blob might have specialized implementation details depending on associated metadata."
    "enum_type", ":ref:`ref_flyteidl.core.EnumType`", "", "Defines an enum with pre-defined string values."
    "structured_dataset_type", ":ref:`ref_flyteidl.core.StructuredDatasetType`", "", "Generalized schema support"
+   "union_type", ":ref:`ref_flyteidl.core.UnionType`", "", "Defines an union type with pre-defined LiteralTypes."
    "metadata", ":ref:`ref_google.protobuf.Struct`", "", "This field contains type metadata that is descriptive of the type, but is NOT considered in type-checking. This might be used by consumers to identify special behavior or display extended information for the type."
    "annotation", ":ref:`ref_flyteidl.core.TypeAnnotation`", "", "This field contains arbitrary data that might have special semantic meaning for the client but does not effect internal flyte behavior."
+   "structure", ":ref:`ref_flyteidl.core.TypeStructure`", "", "Hints to improve type matching."
 
 
 
@@ -2523,6 +2570,62 @@ TypeAnnotation encapsulates registration time information about a type. This can
 
 
 
+
+
+.. _ref_flyteidl.core.TypeStructure:
+
+TypeStructure
+------------------------------------------------------------------
+
+Hints to improve type matching
+e.g. allows distinguishing output from custom type transformers
+even if the underlying IDL serialization matches.
+
+
+
+.. csv-table:: TypeStructure type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "tag", ":ref:`ref_string`", "", "Must exactly match for types to be castable"
+
+
+
+
+
+
+
+.. _ref_flyteidl.core.UnionType:
+
+UnionType
+------------------------------------------------------------------
+
+Defines a tagged union type, also known as a variant (and formally as the sum type).
+
+A sum type S is defined by a sequence of types (A, B, C, ...), each tagged by a string tag
+A value of type S is constructed from a value of any of the variant types. The specific choice of type is recorded by
+storing the varaint's tag with the literal value and can be examined in runtime.
+
+Type S is typically written as
+S := Apple A | Banana B | Cantaloupe C | ...
+
+Notably, a nullable (optional) type is a sum type between some type X and the singleton type representing a null-value:
+Optional X := X | Null
+
+See also: https://en.wikipedia.org/wiki/Tagged_union
+
+
+
+.. csv-table:: UnionType type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "variants", ":ref:`ref_flyteidl.core.LiteralType`", "repeated", "Predefined set of variants in union."
+
+
+
+
+
  
 
 
@@ -2594,46 +2697,6 @@ Define a set of simple types.
 
 
 
-.. _ref_flyteidl/core/workflow_closure.proto:
-
-flyteidl/core/workflow_closure.proto
-==================================================================
-
-
-
-
-
-.. _ref_flyteidl.core.WorkflowClosure:
-
-WorkflowClosure
-------------------------------------------------------------------
-
-Defines an enclosed package of workflow and tasks it references.
-
-
-
-.. csv-table:: WorkflowClosure type fields
-   :header: "Field", "Type", "Label", "Description"
-   :widths: auto
-
-   "workflow", ":ref:`ref_flyteidl.core.WorkflowTemplate`", "", "required. Workflow template."
-   "tasks", ":ref:`ref_flyteidl.core.TaskTemplate`", "repeated", "optional. A collection of tasks referenced by the workflow. Only needed if the workflow references tasks."
-
-
-
-
-
- 
-
- 
-
- 
-
- 
-
-
-
-
 .. _ref_flyteidl/core/workflow.proto:
 
 flyteidl/core/workflow.proto
@@ -2954,6 +3017,46 @@ Failure Handling Strategy
 
 
 
+.. _ref_flyteidl/core/workflow_closure.proto:
+
+flyteidl/core/workflow_closure.proto
+==================================================================
+
+
+
+
+
+.. _ref_flyteidl.core.WorkflowClosure:
+
+WorkflowClosure
+------------------------------------------------------------------
+
+Defines an enclosed package of workflow and tasks it references.
+
+
+
+.. csv-table:: WorkflowClosure type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "workflow", ":ref:`ref_flyteidl.core.WorkflowTemplate`", "", "required. Workflow template."
+   "tasks", ":ref:`ref_flyteidl.core.TaskTemplate`", "repeated", "optional. A collection of tasks referenced by the workflow. Only needed if the workflow references tasks."
+
+
+
+
+
+ 
+
+ 
+
+ 
+
+ 
+
+
+
+
 .. _ref_google/protobuf/timestamp.proto:
 
 google/protobuf/timestamp.proto

package/protos/flyteidl/admin/cluster_assignment.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package flyteidl.admin;
+option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
+
+
+// Encapsulates specifications for routing an execution onto a specific cluster.
+message ClusterAssignment {
+  Affinity affinity = 1;
+}
+
+// Defines a set of constraints used to select eligible objects based on labels they possess.
+message Affinity {
+  // Multiples selectors are 'and'-ed together to produce the list of matching, eligible objects.
+  repeated Selector selectors = 1;
+}
+
+// A Selector is a specification for identifying a set of objects with corresponding labels.
+message Selector {
+
+  // The label key.
+  string key = 1;
+
+  // One or more values used to match labels.
+  // For equality (or inequality) requirements, values must contain a single element.
+  // For set-based requirements, values may contain one or more elements.
+  repeated string value = 2;
+
+  // Defines how a label with a corresponding key and value is selected or excluded.
+  enum Operator {
+    EQUALS = 0;
+    NOT_EQUALS = 1;
+    IN = 2;
+    NOT_IN = 3;
+    EXISTS = 4; // A label key with any value
+    
+    // K8s supports more operators, we can consider adding them if necessary
+  }
+  Operator operator = 3;
+}
+

package/protos/flyteidl/admin/execution.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package flyteidl.admin;
 option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
 
+import "flyteidl/admin/cluster_assignment.proto";
 import "flyteidl/admin/common.proto";
 import "flyteidl/core/literals.proto";
 import "flyteidl/core/execution.proto";
@@ -285,6 +286,9 @@ message ExecutionSpec {
     // User setting to configure where to store offloaded data (i.e. Blobs, structured datasets, query data, etc.).
     // This should be a prefix like s3://my-bucket/my-data
     RawOutputDataConfig raw_output_data_config = 19;
+
+    // Controls how to select an available cluster on which this execution should run.
+    ClusterAssignment cluster_assignment = 20;
 }
 
 // Request to terminate an in-progress execution.  This action is irreversible.
@@ -356,4 +360,4 @@ message ExecutionStateChangeDetails {
     string principal = 3;
 }
 
-message ExecutionUpdateResponse {}
+message ExecutionUpdateResponse {}

package/protos/flyteidl/admin/matchable_resource.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package flyteidl.admin;
 option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
 
+import "flyteidl/admin/cluster_assignment.proto";
 import "flyteidl/core/execution.proto";
 
 // Defines a resource that can be configured by customizable Project-, ProjectDomain- or WorkflowAttributes
@@ -28,6 +29,9 @@ enum MatchableResource {
 
   // Adds defaults for customizable workflow-execution specifications and overrides.
   WORKFLOW_EXECUTION_CONFIG = 6;
+
+  // Controls how to select an available cluster on which this execution should run.
+  CLUSTER_ASSIGNMENT = 7;
 }
 
 // Defines a set of overridable task resource attributes set during task registration.
@@ -116,6 +120,8 @@ message MatchingAttributes {
     PluginOverrides plugin_overrides = 6;
 
     WorkflowExecutionConfig workflow_execution_config = 7;
+
+    ClusterAssignment cluster_assignment = 8;
   }
 }
 

package/protos/flyteidl/core/literals.proto

@@ -52,6 +52,12 @@ message Schema {
     SchemaType type = 3;
 }
 
+// The runtime representation of a tagged union value. See `UnionType` for more details.
+message Union {
+    Literal value = 1;
+    LiteralType type = 2;
+}
+
 message StructuredDatasetMetadata {
     // Bundle the type information along with the literal.
     // This is here because StructuredDatasets can often be more defined at run time than at compile time.
@@ -81,6 +87,7 @@ message Scalar {
         Error error = 6;
         google.protobuf.Struct generic = 7;
         StructuredDataset structured_dataset = 8;
+        Union union = 9;
     }
 }
 
@@ -123,6 +130,10 @@ message BindingDataMap {
     map<string, BindingData> bindings = 1;
 }
 
+message UnionInfo {
+  LiteralType targetType = 1;
+}
+
 // Specifies either a simple value or a reference to another output.
 message BindingData {
     oneof value {
@@ -139,6 +150,8 @@ message BindingData {
         // A map of bindings. The key is always a string.
         BindingDataMap map = 4;
     }
+
+    UnionInfo union = 5;
 }
 
 // An input/output binding of a variable to either static value or a node output.

package/protos/flyteidl/core/types.proto

@@ -90,6 +90,32 @@ message EnumType {
     repeated string values = 1;
 }
 
+// Defines a tagged union type, also known as a variant (and formally as the sum type).
+//
+// A sum type S is defined by a sequence of types (A, B, C, ...), each tagged by a string tag
+// A value of type S is constructed from a value of any of the variant types. The specific choice of type is recorded by
+// storing the varaint's tag with the literal value and can be examined in runtime.
+//
+// Type S is typically written as
+// S := Apple A | Banana B | Cantaloupe C | ...
+//
+// Notably, a nullable (optional) type is a sum type between some type X and the singleton type representing a null-value:
+// Optional X := X | Null
+//
+// See also: https://en.wikipedia.org/wiki/Tagged_union
+message UnionType {
+    // Predefined set of variants in union.
+    repeated LiteralType variants = 1;
+}
+
+// Hints to improve type matching
+// e.g. allows distinguishing output from custom type transformers
+// even if the underlying IDL serialization matches.
+message TypeStructure {
+    // Must exactly match for types to be castable
+    string tag = 1;
+}
+
 // TypeAnnotation encapsulates registration time information about a type. This can be used for various control-plane operations. TypeAnnotation will not be available at runtime when a task runs.
 message TypeAnnotation {
     // A arbitrary JSON payload to describe a type.
@@ -119,6 +145,9 @@ message LiteralType {
 
         // Generalized schema support
         StructuredDatasetType structured_dataset_type = 8;
+
+        // Defines an union type with pre-defined LiteralTypes.
+        UnionType union_type = 10;
     }
 
     // This field contains type metadata that is descriptive of the type, but is NOT considered in type-checking.  This might be used by
@@ -128,6 +157,9 @@ message LiteralType {
     // This field contains arbitrary data that might have special semantic
     // meaning for the client but does not effect internal flyte behavior.
     TypeAnnotation annotation = 9;
+
+    // Hints to improve type matching.
+    TypeStructure structure = 11;
 }
 
 // A reference to an output produced by a node. The type can be retrieved -and validated- from