@flyteorg/flyteidl 1.16.0 → 2.0.0-test1

package/protos/flyteidl/core/errors.proto

@@ -1,42 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "flyteidl/core/execution.proto";
-import "google/protobuf/timestamp.proto";
-
-// Error message to propagate detailed errors from container executions to the execution
-// engine.
-message ContainerError {
-    // A simplified code for errors, so that we can provide a glossary of all possible errors.
-    string code = 1;
-    // A detailed error message.
-    string message = 2;
-
-    // Defines a generic error type that dictates the behavior of the retry strategy.
-    enum Kind {
-        NON_RECOVERABLE = 0;
-        RECOVERABLE = 1;
-    }
-
-    // An abstract error kind for this error. Defaults to Non_Recoverable if not specified.
-    Kind kind = 3;
-
-    // Defines the origin of the error (system, user, unknown).
-    ExecutionError.ErrorKind origin = 4;
-
-    // Timestamp of the error
-    google.protobuf.Timestamp timestamp = 5;
-
-    // Worker that generated the error
-    string worker = 6;
-}
-
-// Defines the errors.pb file format the container can produce to communicate
-// failure reasons to the execution engine.
-message ErrorDocument {
-    // The error raised during execution.
-    ContainerError error = 1;
-}

package/protos/flyteidl/core/execution.proto

@@ -1,156 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "google/protobuf/duration.proto";
-import "google/protobuf/timestamp.proto";
-
-// Indicates various phases of Workflow Execution
-message WorkflowExecution {
-    enum Phase {
-        UNDEFINED = 0;
-        QUEUED = 1;
-        RUNNING = 2;
-        SUCCEEDING = 3;
-        SUCCEEDED = 4;
-        FAILING = 5;
-        FAILED = 6;
-        ABORTED = 7;
-        TIMED_OUT = 8;
-        ABORTING = 9;
-    }
-}
-
-// Indicates various phases of Node Execution that only include the time spent to run the nodes/workflows
-message NodeExecution {
-    enum Phase {
-        UNDEFINED = 0;
-        QUEUED = 1;
-        RUNNING = 2;
-        SUCCEEDED = 3;
-        FAILING = 4;
-        FAILED = 5;
-        ABORTED = 6;
-        SKIPPED = 7;
-        TIMED_OUT = 8;
-        DYNAMIC_RUNNING = 9;
-        RECOVERED = 10;
-    }
-}
-
-// Phases that task plugins can go through. Not all phases may be applicable to a specific plugin task,
-// but this is the cumulative list that customers may want to know about for their task.
-message TaskExecution{
-    enum Phase {
-        UNDEFINED = 0;
-        QUEUED = 1;
-        RUNNING = 2;
-        SUCCEEDED = 3;
-        ABORTED = 4;
-        FAILED = 5;
-        // To indicate cases where task is initializing, like: ErrImagePull, ContainerCreating, PodInitializing
-        INITIALIZING = 6;
-        // To address cases, where underlying resource is not available: Backoff error, Resource quota exceeded
-        WAITING_FOR_RESOURCES = 7;
-        RETRYABLE_FAILED = 8;
-    }
-}
-
-
-// Represents the error message from the execution.
-message ExecutionError {
-    // Error code indicates a grouping of a type of error.
-    // More Info: <Link>
-    string code = 1;
-    // Detailed description of the error - including stack trace.
-    string message = 2;
-    // Full error contents accessible via a URI
-    string error_uri = 3;
-    // Error type: System or User
-    enum ErrorKind {
-        UNKNOWN = 0;
-        USER = 1;
-        SYSTEM = 2;
-    }
-    ErrorKind kind = 4;
-    // Timestamp of the error
-    google.protobuf.Timestamp timestamp = 5;
-    // Worker that generated the error
-    string worker = 6;
-}
-
-// Log information for the task that is specific to a log sink
-// When our log story is flushed out, we may have more metadata here like log link expiry
-message TaskLog {
-
-    enum MessageFormat {
-        UNKNOWN = 0;
-        CSV = 1;
-        JSON = 2;
-    }
-
-    string uri = 1;
-    string name = 2;
-    MessageFormat message_format = 3;
-    google.protobuf.Duration ttl = 4;
-    bool ShowWhilePending = 5;
-    bool HideOnceFinished = 6;
-}
-
-// Contains metadata required to identify logs produces by a set of pods
-message LogContext {
-    repeated PodLogContext pods = 1;
-    string primary_pod_name = 2;
-}
-
-// Contains metadata required to identify logs produces by a single pod
-message PodLogContext {
-    string namespace = 1;
-
-    string pod_name = 2;
-
-    repeated ContainerContext containers = 3;
-
-    string primary_container_name = 4;
-
-    repeated ContainerContext init_containers = 5;
-}
-
-// Contains metadata required to identify logs produces by a single container
-message ContainerContext {
-    string container_name = 1;
-
-    // Contains metadata required to identify logs produces by a single light-weight process that was run inside a container
-    message ProcessContext {
-        google.protobuf.Timestamp container_start_time = 1;
-        google.protobuf.Timestamp container_end_time = 2;
-    }
-
-    ProcessContext process = 2;
-}
-
-// Represents customized execution run-time attributes.
-message QualityOfServiceSpec {
-    // Indicates how much queueing delay an execution can tolerate.
-    google.protobuf.Duration queueing_budget = 1;
-
-    // Add future, user-configurable options here
-}
-
-// Indicates the priority of an execution.
-message QualityOfService {
-    enum Tier {
-        // Default: no quality of service specified.
-        UNDEFINED = 0;
-        HIGH = 1;
-        MEDIUM = 2;
-        LOW = 3;
-    }
-
-    oneof designation {
-        Tier tier = 1;
-        QualityOfServiceSpec spec = 2;
-    }
-}

package/protos/flyteidl/core/execution_envs.proto

@@ -1,45 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "google/protobuf/struct.proto";
-
-// ExecutionEnvAssignment is a message that is used to assign an execution environment to a set of
-// nodes.
-message ExecutionEnvAssignment {
-  // node_ids is a list of node ids that are being assigned the execution environment.
-  repeated string node_ids = 1;
-
-  // task_type is the type of task that is being assigned. This is used to override which Flyte
-  // plugin will be used during execution.
-  string task_type = 2;
-
-  // execution_env is the environment that is being assigned to the nodes.
-  ExecutionEnv execution_env = 3;
-}
-
-// ExecutionEnv is a message that is used to specify the execution environment.
-message ExecutionEnv {
-  // name is a human-readable identifier for the execution environment. This is combined with the
-  // project, domain, and version to uniquely identify an execution environment.
-  string name = 1;
-
-  // type is the type of the execution environment.
-  string type = 2;
-
-  // environment is a oneof field that can be used to specify the environment in different ways.
-  oneof environment {
-    // extant is a reference to an existing environment.
-    google.protobuf.Struct extant = 3;
-
-    // spec is a specification of the environment.
-    google.protobuf.Struct spec = 4;
-  }
-
-  // version is the version of the execution environment. This may be used differently by each
-  // individual environment type (ex. auto-generated or manually provided), but is intended to
-  // allow variance in environment specifications with the same ID.
-  string version = 5;
-}

package/protos/flyteidl/core/identifier.proto

@@ -1,80 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-// Indicates a resource type within Flyte.
-enum ResourceType {
-  UNSPECIFIED        = 0;
-  TASK               = 1;
-  WORKFLOW           = 2;
-  LAUNCH_PLAN        = 3;
-  // A dataset represents an entity modeled in Flyte DataCatalog. A Dataset is also a versioned entity and can be a compilation of multiple individual objects.
-  // Eventually all Catalog objects should be modeled similar to Flyte Objects. The Dataset entities makes it possible for the UI  and CLI to act on the objects 
-  // in a similar manner to other Flyte objects
-  DATASET            = 4;
-}
-
-// Encapsulation of fields that uniquely identifies a Flyte resource.
-message Identifier {
-    // Identifies the specific type of resource that this identifier corresponds to.
-    core.ResourceType resource_type  = 1;
-
-    // Name of the project the resource belongs to.
-    string project              = 2;
-
-    // Name of the domain the resource belongs to.
-    // A domain can be considered as a subset within a specific project.
-    string domain               = 3;
-
-    // User provided value for the resource.
-    string name                 = 4;
-
-    // Specific version of the resource.
-    string version              = 5;
-
-    // Optional, org key applied to the resource.
-    string org = 6;
-}
-
-// Encapsulation of fields that uniquely identifies a Flyte workflow execution
-message WorkflowExecutionIdentifier {
-    // Name of the project the resource belongs to.
-    string project              = 1;
-
-    // Name of the domain the resource belongs to.
-    // A domain can be considered as a subset within a specific project.
-    string domain               = 2;
-
-    // User or system provided value for the resource.
-    string name                 = 4;
-
-    // Optional, org key applied to the resource.
-    string org = 5;
-}
-
-// Encapsulation of fields that identify a Flyte node execution entity.
-message NodeExecutionIdentifier {
-    string node_id           = 1;
-
-    WorkflowExecutionIdentifier execution_id        = 2;
-}
-
-// Encapsulation of fields that identify a Flyte task execution entity.
-message TaskExecutionIdentifier {
-    core.Identifier task_id           = 1;
-
-    core.NodeExecutionIdentifier node_execution_id = 2;
-
-    uint32 retry_attempt     = 3;
-}
-
-// Encapsulation of fields the uniquely identify a signal.
-message SignalIdentifier {
-    // Unique identifier for a signal.
-    string signal_id = 1;
-
-    // Identifies the Flyte workflow execution this signal belongs to.
-    WorkflowExecutionIdentifier execution_id = 2;
-}

package/protos/flyteidl/core/interface.proto

@@ -1,64 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "flyteidl/core/types.proto";
-import "flyteidl/core/literals.proto";
-import "flyteidl/core/artifact_id.proto";
-
-// Defines a strongly typed variable.
-message Variable {
-    // Variable literal type.
-    LiteralType type = 1;
-
-    //+optional string describing input variable
-    string description = 2;
-
-    //+optional This object allows the user to specify how Artifacts are created.
-    // name, tag, partitions can be specified. The other fields (version and project/domain) are ignored.
-    core.ArtifactID artifact_partial_id = 3;
-
-    core.ArtifactTag artifact_tag = 4;
-}
-
-// A map of Variables
-message VariableMap {
-    // Defines a map of variable names to variables.
-    map<string, Variable> variables = 1;
-}
-
-// Defines strongly typed inputs and outputs.
-message TypedInterface {
-    VariableMap inputs  = 1;
-    VariableMap outputs = 2;
-}
-
-// A parameter is used as input to a launch plan and has
-// the special ability to have a default value or mark itself as required.
-message Parameter {
-    //+required Variable. Defines the type of the variable backing this parameter.
-    Variable var = 1;
-
-    //+optional
-    oneof behavior {
-        // Defines a default value that has to match the variable type defined.
-        Literal default = 2;
-
-        //+optional, is this value required to be filled.
-        bool required = 3;
-
-        // This is an execution time search basically that should result in exactly one Artifact with a Type that
-        // matches the type of the variable.
-        core.ArtifactQuery artifact_query = 4;
-
-        core.ArtifactID artifact_id = 5;
-    }
-}
-
-// A map of Parameters.
-message ParameterMap {
-    // Defines a map of parameter names to parameters.
-    map<string, Parameter> parameters = 1;
-}

package/protos/flyteidl/core/literals.proto

@@ -1,205 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "google/protobuf/timestamp.proto";
-import "google/protobuf/duration.proto";
-import "google/protobuf/struct.proto";
-import "flyteidl/core/types.proto";
-
-// Primitive Types
-message Primitive {
-    // Defines one of simple primitive types. These types will get translated into different programming languages as
-    // described in https://developers.google.com/protocol-buffers/docs/proto#scalar.
-    oneof value {
-        int64 integer = 1;
-        double float_value = 2;
-        string string_value = 3;
-        bool boolean = 4;
-        google.protobuf.Timestamp datetime = 5;
-        google.protobuf.Duration duration = 6;
-    }
-}
-
-// Used to denote a nil/null/None assignment to a scalar value. The underlying LiteralType for Void is intentionally
-// undefined since it can be assigned to a scalar of any LiteralType.
-message Void {
-}
-
-// Refers to an offloaded set of files. It encapsulates the type of the store and a unique uri for where the data is.
-// There are no restrictions on how the uri is formatted since it will depend on how to interact with the store.
-message Blob {
-    BlobMetadata metadata = 1;
-    string uri = 3;
-}
-
-message BlobMetadata {
-    BlobType type = 1;
-}
-
-// A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.
-// It's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data.
-message Binary {
-    bytes value = 1; // Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict.
-    string tag = 2; // The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization.
-}
-
-// A strongly typed schema that defines the interface of data retrieved from the underlying storage medium.
-message Schema {
-    string uri = 1;
-    SchemaType type = 3;
-}
-
-// The runtime representation of a tagged union value. See `UnionType` for more details.
-message Union {
-    Literal value = 1;
-    LiteralType type = 2;
-}
-
-message StructuredDatasetMetadata {
-    // Bundle the type information along with the literal.
-    // This is here because StructuredDatasets can often be more defined at run time than at compile time.
-    // That is, at compile time you might only declare a task to return a pandas dataframe or a StructuredDataset,
-    // without any column information, but at run time, you might have that column information.
-    // flytekit python will copy this type information into the literal, from the type information, if not provided by
-    // the various plugins (encoders).
-    // Since this field is run time generated, it's not used for any type checking.
-    StructuredDatasetType structured_dataset_type = 1;
-}
-
-message StructuredDataset {
-    // String location uniquely identifying where the data is.
-    // Should start with the storage location (e.g. s3://, gs://, bq://, etc.)
-    string uri = 1;
-
-    StructuredDatasetMetadata metadata = 2;
-}
-
-message Scalar {
-    oneof value {
-        Primitive primitive = 1;
-        Blob blob = 2;
-        Binary binary = 3;
-        Schema schema = 4;
-        Void none_type = 5;
-        Error error = 6;
-        google.protobuf.Struct generic = 7;
-        StructuredDataset structured_dataset = 8;
-        Union union = 9;
-    }
-}
-
-// A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives.
-message Literal {
-    reserved 6, 7;
-    oneof value {
-        // A simple value.
-        Scalar scalar = 1;
-
-        // A collection of literals to allow nesting.
-        LiteralCollection collection = 2;
-
-        // A map of strings to literals.
-        LiteralMap map = 3;
-
-        // Offloaded literal metadata
-        // When you deserialize the offloaded metadata, it would be of Literal and its type would be defined by LiteralType stored in offloaded_metadata.
-        LiteralOffloadedMetadata offloaded_metadata = 8;
-    }
-
-    // A hash representing this literal.
-    // This is used for caching purposes. For more details refer to RFC 1893
-    // (https://github.com/flyteorg/flyte/blob/master/rfc/system/1893-caching-of-offloaded-objects.md)
-    string hash = 4;
-
-    // Additional metadata for literals.
-    map<string, string> metadata = 5;
-}
-
-// A message that contains the metadata of the offloaded data.
-message LiteralOffloadedMetadata {
-    // The location of the offloaded core.Literal.
-    string uri = 1;
-
-    // The size of the offloaded data.
-    uint64 size_bytes = 2;
-
-    // The inferred literal type of the offloaded data.
-    LiteralType inferred_type = 3;
-}
-
-// A collection of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
-message LiteralCollection {
-    repeated Literal literals = 1;
-}
-
-// A map of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
-message LiteralMap {
-    map<string, Literal> literals = 1;
-}
-
-// A collection of BindingData items.
-message BindingDataCollection {
-    repeated BindingData bindings = 1;
-}
-
-// A map of BindingData items.
-message BindingDataMap {
-    map<string, BindingData> bindings = 1;
-}
-
-message UnionInfo {
-  LiteralType targetType = 1;
-}
-
-// Specifies either a simple value or a reference to another output.
-message BindingData {
-    oneof value {
-        // A simple scalar value.
-        Scalar scalar = 1;
-
-        // A collection of binding data. This allows nesting of binding data to any number
-        // of levels.
-        BindingDataCollection collection = 2;
-
-        // References an output promised by another node.
-        OutputReference promise = 3;
-
-        // A map of bindings. The key is always a string.
-        BindingDataMap map = 4;
-
-        // Offloaded literal metadata
-        // When you deserialize the offloaded metadata, it would be of Literal and its type would be defined by LiteralType stored in offloaded_metadata.
-        // Used for nodes that don't have promises from upstream nodes such as ArrayNode subNodes.
-        LiteralOffloadedMetadata offloaded_metadata = 6;
-    }
-
-    UnionInfo union = 5;
-}
-
-// An input/output binding of a variable to either static value or a node output.
-message Binding {
-    // Variable name must match an input/output variable of the node.
-    string var = 1;
-
-    // Data to use to bind this variable.
-    BindingData binding = 2;
-}
-
-// A generic key value pair.
-message KeyValuePair {
-    //required.
-    string key = 1;
-
-    //+optional.
-    string value = 2;
-}
-
-// Retry strategy associated with an executable unit.
-message RetryStrategy {
-    // Number of retries. Retries will be consumed when the job fails with a recoverable error.
-    // The number of retries must be less than or equals to 10.
-    uint32 retries = 5;
-}

package/protos/flyteidl/core/metrics.proto

@@ -1,50 +0,0 @@
-syntax = "proto3";
-
-package flyteidl.core;
-
-option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core";
-
-import "flyteidl/core/identifier.proto";
-import "google/protobuf/timestamp.proto";
-import "google/protobuf/struct.proto";
-
-// Span represents a duration trace of Flyte execution. The id field denotes a Flyte execution entity or an operation
-// which uniquely identifies the Span. The spans attribute allows this Span to be further broken down into more
-// precise definitions.
-message Span {
-    // start_time defines the instance this span began.
-    google.protobuf.Timestamp start_time = 1;
-
-    // end_time defines the instance this span completed.
-    google.protobuf.Timestamp end_time = 2;
-
-    oneof id {
-        // workflow_id is the id of the workflow execution this Span represents.
-        flyteidl.core.WorkflowExecutionIdentifier workflow_id = 3;
-
-        // node_id is the id of the node execution this Span represents.
-        flyteidl.core.NodeExecutionIdentifier node_id = 4;
-
-        // task_id is the id of the task execution this Span represents.
-        flyteidl.core.TaskExecutionIdentifier task_id = 5;
-
-        // operation_id is the id of a unique operation that this Span represents.
-        string operation_id = 6;
-    }
-
-    // spans defines a collection of Spans that breakdown this execution.
-    repeated Span spans = 7;
-}
-
-// ExecutionMetrics is a collection of metrics that are collected during the execution of a Flyte task.
-message ExecutionMetricResult {
-    // The metric this data represents. e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG.
-    string metric = 1;
-
-    // The result data in prometheus range query result format
-    // https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats.
-    // This may include multiple time series, differentiated by their metric labels.
-    // Start time is greater of (execution attempt start, 48h ago)
-    // End time is lesser of (execution attempt end, now)
-    google.protobuf.Struct data = 2;
-}