@flyteorg/flyteidl 0.21.7 → 0.21.11

package/protos/docs/core/core.rst

@@ -57,6 +57,20 @@ Catalog artifact information with specific metadata
 
 
 
+
+
+.. _ref_flyteidl.core.CatalogReservation:
+
+CatalogReservation
+------------------------------------------------------------------
+
+
+
+
+
+
+
+
  
 
 
@@ -79,6 +93,25 @@ Indicates the status of CatalogCaching. The reason why this is not embedded in T
    "CACHE_LOOKUP_FAILURE", "4", "Used to indicate that cache lookup failed because of an error"
    "CACHE_PUT_FAILURE", "5", "Used to indicate that cache lookup failed because of an error"
 
+
+
+.. _ref_flyteidl.core.CatalogReservation.Status:
+
+CatalogReservation.Status
+------------------------------------------------------------------
+
+Indicates the status of a catalog reservation operation.
+
+.. csv-table:: Enum CatalogReservation.Status values
+   :header: "Name", "Number", "Description"
+   :widths: auto
+
+   "RESERVATION_DISABLED", "0", "Used to indicate that reservations are disabled"
+   "RESERVATION_ACQUIRED", "1", "Used to indicate that a reservation was successfully acquired or extended"
+   "RESERVATION_EXISTS", "2", "Used to indicate that an active reservation currently exists"
+   "RESERVATION_RELEASED", "3", "Used to indicate that the reservation has been successfully released"
+   "RESERVATION_FAILURE", "4", "Used to indicate that a reservation operation resulted in failure"
+
  
 
  
@@ -1732,6 +1765,7 @@ Container
    "config", ":ref:`ref_flyteidl.core.KeyValuePair`", "repeated", "**Deprecated.** Allows extra configs to be available for the container. TODO: elaborate on how configs will become available. Deprecated, please use TaskTemplate.config instead."
    "ports", ":ref:`ref_flyteidl.core.ContainerPort`", "repeated", "Ports to open in the container. This feature is not supported by all execution engines. (e.g. supported on K8s but not supported on AWS Batch) Only K8s"
    "data_config", ":ref:`ref_flyteidl.core.DataLoadingConfig`", "", "BETA: Optional configuration for DataLoading. If not specified, then default values are used. This makes it possible to to run a completely portable container, that uses inputs and outputs only from the local file-system and without having any reference to flyteidl. This is supported only on K8s at the moment. If data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories are not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation to understand the default paths. Only K8s"
+   "architecture", ":ref:`ref_flyteidl.core.Container.Architecture`", "", ""
 
 
 
@@ -2007,6 +2041,7 @@ Task Metadata
    "discovery_version", ":ref:`ref_string`", "", "Indicates a logical version to apply to this task for the purpose of discovery."
    "deprecated_error_message", ":ref:`ref_string`", "", "If set, this indicates that this task is deprecated. This will enable owners of tasks to notify consumers of the ending of support for a given task."
    "interruptible", ":ref:`ref_bool`", "", ""
+   "cache_serializable", ":ref:`ref_bool`", "", "Indicates whether the system should attempt to execute discoverable instances in serial to avoid duplicate work"
 
 
 
@@ -2070,6 +2105,25 @@ TaskTemplate.ConfigEntry
 
 
 
+.. _ref_flyteidl.core.Container.Architecture:
+
+Container.Architecture
+------------------------------------------------------------------
+
+Architecture-type the container image supports.
+
+.. csv-table:: Enum Container.Architecture values
+   :header: "Name", "Number", "Description"
+   :widths: auto
+
+   "UNKNOWN", "0", ""
+   "AMD64", "1", ""
+   "ARM64", "2", ""
+   "ARM_V6", "3", ""
+   "ARM_V7", "4", ""
+
+
+
 .. _ref_flyteidl.core.DataLoadingConfig.LiteralMapFormat:
 
 DataLoadingConfig.LiteralMapFormat
@@ -2425,6 +2479,46 @@ Define a set of simple types.
 
 
 
+.. _ref_flyteidl/core/workflow_closure.proto:
+
+flyteidl/core/workflow_closure.proto
+==================================================================
+
+
+
+
+
+.. _ref_flyteidl.core.WorkflowClosure:
+
+WorkflowClosure
+------------------------------------------------------------------
+
+Defines an enclosed package of workflow and tasks it references.
+
+
+
+.. csv-table:: WorkflowClosure type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "workflow", ":ref:`ref_flyteidl.core.WorkflowTemplate`", "", "required. Workflow template."
+   "tasks", ":ref:`ref_flyteidl.core.TaskTemplate`", "repeated", "optional. A collection of tasks referenced by the workflow. Only needed if the workflow references tasks."
+
+
+
+
+
+ 
+
+ 
+
+ 
+
+ 
+
+
+
+
 .. _ref_flyteidl/core/workflow.proto:
 
 flyteidl/core/workflow.proto
@@ -2745,46 +2839,6 @@ Failure Handling Strategy
 
 
 
-.. _ref_flyteidl/core/workflow_closure.proto:
-
-flyteidl/core/workflow_closure.proto
-==================================================================
-
-
-
-
-
-.. _ref_flyteidl.core.WorkflowClosure:
-
-WorkflowClosure
-------------------------------------------------------------------
-
-Defines an enclosed package of workflow and tasks it references.
-
-
-
-.. csv-table:: WorkflowClosure type fields
-   :header: "Field", "Type", "Label", "Description"
-   :widths: auto
-
-   "workflow", ":ref:`ref_flyteidl.core.WorkflowTemplate`", "", "required. Workflow template."
-   "tasks", ":ref:`ref_flyteidl.core.TaskTemplate`", "repeated", "optional. A collection of tasks referenced by the workflow. Only needed if the workflow references tasks."
-
-
-
-
-
- 
-
- 
-
- 
-
- 
-
-
-
-
 .. _ref_google/protobuf/timestamp.proto:
 
 google/protobuf/timestamp.proto
@@ -3101,8 +3155,8 @@ Value
 
 `Value` represents a dynamically typed value which can be either
 null, a number, a string, a boolean, a recursive struct value, or a
-list of values. A producer of value is expected to set one of that
-variants, absence of any variant indicates an error.
+list of values. A producer of value is expected to set one of these
+variants. Absence of any variant indicates an error.
 
 The JSON representation for `Value` is JSON value.
 

package/protos/docs/datacatalog/datacatalog.rst

@@ -261,43 +261,6 @@ Dataset properties we can filter by
 
 
 
-.. _ref_datacatalog.ExtendReservationRequest:
-
-ExtendReservationRequest
-------------------------------------------------------------------
-
-Request to extend reservation
-
-
-
-.. csv-table:: ExtendReservationRequest type fields
-   :header: "Field", "Type", "Label", "Description"
-   :widths: auto
-
-   "dataset_id", ":ref:`ref_datacatalog.DatasetID`", "", ""
-   "tag_name", ":ref:`ref_string`", "", ""
-   "owner_id", ":ref:`ref_string`", "", ""
-
-
-
-
-
-
-
-.. _ref_datacatalog.ExtendReservationResponse:
-
-ExtendReservationResponse
-------------------------------------------------------------------
-
-Response to extend reservation
-
-
-
-
-
-
-
-
 .. _ref_datacatalog.FilterExpression:
 
 FilterExpression
@@ -410,22 +373,22 @@ Dataset.
 
 
 
-.. _ref_datacatalog.GetOrReserveArtifactRequest:
+.. _ref_datacatalog.GetOrExtendReservationRequest:
 
-GetOrReserveArtifactRequest
+GetOrExtendReservationRequest
 ------------------------------------------------------------------
 
-Get the Artifact or try to reserve a spot if the Artifact does not exist.
+Try to acquire or extend an artifact reservation. If an active reservation exists, retreive that instance.
 
 
 
-.. csv-table:: GetOrReserveArtifactRequest type fields
+.. csv-table:: GetOrExtendReservationRequest type fields
    :header: "Field", "Type", "Label", "Description"
    :widths: auto
 
-   "dataset_id", ":ref:`ref_datacatalog.DatasetID`", "", ""
-   "tag_name", ":ref:`ref_string`", "", ""
+   "reservation_id", ":ref:`ref_datacatalog.ReservationID`", "", ""
    "owner_id", ":ref:`ref_string`", "", ""
+   "heartbeat_interval", ":ref:`ref_google.protobuf.Duration`", "", "Requested reservation extension heartbeat interval"
 
 
 
@@ -433,21 +396,20 @@ Get the Artifact or try to reserve a spot if the Artifact does not exist.
 
 
 
-.. _ref_datacatalog.GetOrReserveArtifactResponse:
+.. _ref_datacatalog.GetOrExtendReservationResponse:
 
-GetOrReserveArtifactResponse
+GetOrExtendReservationResponse
 ------------------------------------------------------------------
 
-Response to get artifact or reserve spot.
+Response including either a newly minted reservation or the existing reservation
 
 
 
-.. csv-table:: GetOrReserveArtifactResponse type fields
+.. csv-table:: GetOrExtendReservationResponse type fields
    :header: "Field", "Type", "Label", "Description"
    :widths: auto
 
-   "artifact", ":ref:`ref_datacatalog.Artifact`", "", ""
-   "reservation_status", ":ref:`ref_datacatalog.ReservationStatus`", "", ""
+   "reservation", ":ref:`ref_datacatalog.Reservation`", "", ""
 
 
 
@@ -689,8 +651,7 @@ Request to release reservation
    :header: "Field", "Type", "Label", "Description"
    :widths: auto
 
-   "dataset_id", ":ref:`ref_datacatalog.DatasetID`", "", ""
-   "tag_name", ":ref:`ref_string`", "", ""
+   "reservation_id", ":ref:`ref_datacatalog.ReservationID`", "", ""
    "owner_id", ":ref:`ref_string`", "", ""
 
 
@@ -713,22 +674,46 @@ Response to release reservation
 
 
 
-.. _ref_datacatalog.ReservationStatus:
+.. _ref_datacatalog.Reservation:
 
-ReservationStatus
+Reservation
 ------------------------------------------------------------------
 
-Whether we successfully reserve a spot.
+A reservation including owner, heartbeat interval, expiration timestamp, and various metadata.
 
 
 
-.. csv-table:: ReservationStatus type fields
+.. csv-table:: Reservation type fields
    :header: "Field", "Type", "Label", "Description"
    :widths: auto
 
-   "state", ":ref:`ref_datacatalog.ReservationStatus.State`", "", ""
-   "metadata", ":ref:`ref_datacatalog.Metadata`", "", ""
+   "reservation_id", ":ref:`ref_datacatalog.ReservationID`", "", ""
    "owner_id", ":ref:`ref_string`", "", ""
+   "heartbeat_interval", ":ref:`ref_google.protobuf.Duration`", "", "Recommended heartbeat interval to extend reservation"
+   "expires_at", ":ref:`ref_google.protobuf.Timestamp`", "", "Expiration timestamp of this reservation"
+   "metadata", ":ref:`ref_datacatalog.Metadata`", "", ""
+
+
+
+
+
+
+
+.. _ref_datacatalog.ReservationID:
+
+ReservationID
+------------------------------------------------------------------
+
+ReservationID message that is composed of several string fields.
+
+
+
+.. csv-table:: ReservationID type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "dataset_id", ":ref:`ref_datacatalog.DatasetID`", "", ""
+   "tag_name", ":ref:`ref_string`", "", ""
 
 
 
@@ -839,22 +824,6 @@ PaginationOptions.SortOrder
 
 
 
-.. _ref_datacatalog.ReservationStatus.State:
-
-ReservationStatus.State
-------------------------------------------------------------------
-
-
-
-.. csv-table:: Enum ReservationStatus.State values
-   :header: "Name", "Number", "Description"
-   :widths: auto
-
-   "ACQUIRED", "0", "Acquired the reservation successfully."
-   "ALREADY_IN_PROGRESS", "1", "Indicates an existing active reservation exist for a different owner_id."
-
-
-
 .. _ref_datacatalog.SinglePropertyFilter.ComparisonOperator:
 
 SinglePropertyFilter.ComparisonOperator
@@ -894,8 +863,7 @@ Artifacts are associated with a Dataset, and can be tagged for retrieval.
    "AddTag", ":ref:`ref_datacatalog.AddTagRequest`", ":ref:`ref_datacatalog.AddTagResponse`", "Associate a tag with an artifact. Tags are unique within a Dataset."
    "ListArtifacts", ":ref:`ref_datacatalog.ListArtifactsRequest`", ":ref:`ref_datacatalog.ListArtifactsResponse`", "Return a paginated list of artifacts"
    "ListDatasets", ":ref:`ref_datacatalog.ListDatasetsRequest`", ":ref:`ref_datacatalog.ListDatasetsResponse`", "Return a paginated list of datasets"
-   "GetOrReserveArtifact", ":ref:`ref_datacatalog.GetOrReserveArtifactRequest`", ":ref:`ref_datacatalog.GetOrReserveArtifactResponse`", "Get an artifact and the corresponding data. If the artifact does not exist, try to reserve a spot for populating the artifact. Once you preserve a spot, you should call ExtendReservation API periodically to extend the reservation. Otherwise, the reservation can expire and other tasks may take the spot. If the same owner_id calls this API for the same dataset and it has an active reservation and the artifacts have not been written yet by a different owner, the API will respond with an Acquired Reservation Status (providing idempotency). Note: We may have multiple concurrent tasks with the same signature and the same input that try to populate the same artifact at the same time. Thus with reservation, only one task can run at a time, until the reservation expires. Note: If task A does not extend the reservation in time and the reservation expires, another task B may take over the reservation, resulting in two tasks A and B running in parallel. So a third task C may get the Artifact from A or B, whichever writes last."
-   "ExtendReservation", ":ref:`ref_datacatalog.ExtendReservationRequest`", ":ref:`ref_datacatalog.ExtendReservationResponse`", "Extend the reservation to keep it from expiring. If the reservation expires, other tasks can take over the reservation by calling GetOrReserveArtifact."
+   "GetOrExtendReservation", ":ref:`ref_datacatalog.GetOrExtendReservationRequest`", ":ref:`ref_datacatalog.GetOrExtendReservationResponse`", "Attempts to get or extend a reservation for the corresponding artifact. If one already exists (ie. another entity owns the reservation) then that reservation is retrieved. Once you acquire a reservation, you need to periodically extend the reservation with an identical call. If the reservation is not extended before the defined expiration, it may be acquired by another task. Note: We may have multiple concurrent tasks with the same signature and the same input that try to populate the same artifact at the same time. Thus with reservation, only one task can run at a time, until the reservation expires. Note: If task A does not extend the reservation in time and the reservation expires, another task B may take over the reservation, resulting in two tasks A and B running in parallel. So a third task C may get the Artifact from A or B, whichever writes last."
    "ReleaseReservation", ":ref:`ref_datacatalog.ReleaseReservationRequest`", ":ref:`ref_datacatalog.ReleaseReservationResponse`", "Release the reservation when the task holding the spot fails so that the other tasks can grab the spot."
  <!-- end services -->
 

package/protos/docs/event/event.rst

@@ -236,6 +236,7 @@ TaskNodeMetadata
 
    "cache_status", ":ref:`ref_flyteidl.core.CatalogCacheStatus`", "", "Captures the status of caching for this execution."
    "catalog_key", ":ref:`ref_flyteidl.core.CatalogMetadata`", "", "This structure carries the catalog artifact information"
+   "reservation_status", ":ref:`ref_flyteidl.core.CatalogReservation.Status`", "", "Captures the status of cache reservations for this execution."
    "dynamic_workflow", ":ref:`ref_flyteidl.event.DynamicWorkflowNodeMetadata`", "", "In the case this task launched a dynamic workflow we capture its structure here."
 
 

package/protos/flyteidl/core/catalog.proto

@@ -41,3 +41,19 @@ message CatalogMetadata {
        TaskExecutionIdentifier source_task_execution = 3;
     }
 };
+
+message CatalogReservation {
+    // Indicates the status of a catalog reservation operation.
+    enum Status {
+      // Used to indicate that reservations are disabled
+      RESERVATION_DISABLED = 0;
+      // Used to indicate that a reservation was successfully acquired or extended
+      RESERVATION_ACQUIRED = 1;
+      // Used to indicate that an active reservation currently exists
+      RESERVATION_EXISTS = 2;
+      // Used to indicate that the reservation has been successfully released
+      RESERVATION_RELEASED = 3;
+      // Used to indicate that a reservation operation resulted in failure
+      RESERVATION_FAILURE = 4;
+    }
+}

package/protos/flyteidl/core/tasks.proto

@@ -92,6 +92,9 @@ message TaskMetadata {
     oneof interruptible_value {
         bool interruptible = 8;
     };
+
+    // Indicates whether the system should attempt to execute discoverable instances in serial to avoid duplicate work
+    bool cache_serializable = 9;
 }
 
 // A Task structure that uniquely identifies a task in the system

package/protos/flyteidl/datacatalog/datacatalog.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package datacatalog;
 
 import "flyteidl/core/literals.proto";
+import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 
 /*
@@ -34,25 +35,18 @@ service DataCatalog {
     // Return a paginated list of datasets
     rpc ListDatasets (ListDatasetsRequest) returns (ListDatasetsResponse);
 
-    // Get an artifact and the corresponding data. If the artifact does not exist,
-    // try to reserve a spot for populating the artifact.
-    // Once you preserve a spot, you should call ExtendReservation API periodically
-    // to extend the reservation. Otherwise, the reservation can expire and other
-    // tasks may take the spot.
-    // If the same owner_id calls this API for the same dataset and it has an active reservation and the artifacts have not been written yet by a different owner, the API will respond with an Acquired Reservation Status (providing idempotency).
-    // Note: We may have multiple concurrent tasks with the same signature
-    // and the same input that try to populate the same artifact at the same time.
-    // Thus with reservation, only one task can run at a time, until the reservation
-    // expires.
-    // Note: If task A does not extend the reservation in time and the reservation
-    // expires, another task B may take over the reservation, resulting in two tasks
-    // A and B running in parallel. So a third task C may get the Artifact from A or B,
-    // whichever writes last.
-    rpc GetOrReserveArtifact (GetOrReserveArtifactRequest) returns (GetOrReserveArtifactResponse);
-
-    // Extend the reservation to keep it from expiring. If the reservation expires,
-    // other tasks can take over the reservation by calling GetOrReserveArtifact.
-    rpc ExtendReservation (ExtendReservationRequest) returns (ExtendReservationResponse);
+    // Attempts to get or extend a reservation for the corresponding artifact. If one already exists
+    // (ie. another entity owns the reservation) then that reservation is retrieved.
+    // Once you acquire a reservation, you need to  periodically extend the reservation with an 
+    // identical call. If the reservation is not extended before the defined expiration, it may be
+    // acquired by another task.
+    // Note: We may have multiple concurrent tasks with the same signature and the same input that
+    // try to populate the same artifact at the same time. Thus with reservation, only one task can
+    // run at a time, until the reservation expires.
+    // Note: If task A does not extend the reservation in time and the reservation expires, another
+    // task B may take over the reservation, resulting in two tasks A and B running in parallel. So
+    // a third task C may get the Artifact from A or B, whichever writes last.
+    rpc GetOrExtendReservation (GetOrExtendReservationRequest) returns (GetOrExtendReservationResponse);
 
     // Release the reservation when the task holding the spot fails so that the other tasks
     // can grab the spot.
@@ -174,53 +168,39 @@ message ListDatasetsResponse {
     string next_token = 2;
 }
 
-// Get the Artifact or try to reserve a spot if the Artifact does not exist.
-message GetOrReserveArtifactRequest {
+/*
+ * ReservationID message that is composed of several string fields.
+ */
+message ReservationID {
     DatasetID dataset_id = 1;
     string tag_name = 2;
-    string owner_id = 3;
 }
 
-// Whether we successfully reserve a spot.
-message ReservationStatus {
-    enum State {
-        // Acquired the reservation successfully.
-        ACQUIRED = 0;
-
-        // Indicates an existing active reservation exist for a different owner_id.
-        ALREADY_IN_PROGRESS = 1;
-    };
-
-    State state = 1;
-    Metadata metadata = 2;
-    string owner_id = 3;
+// Try to acquire or extend an artifact reservation. If an active reservation exists, retreive that instance.
+message GetOrExtendReservationRequest {
+    ReservationID reservation_id = 1;
+    string owner_id = 2;
+    google.protobuf.Duration heartbeat_interval = 3; // Requested reservation extension heartbeat interval
 }
 
-// Response to get artifact or reserve spot.
-message GetOrReserveArtifactResponse {
-    oneof value {
-        Artifact artifact = 1;
-        ReservationStatus reservation_status = 2;
-    }
+// A reservation including owner, heartbeat interval, expiration timestamp, and various metadata.
+message Reservation {
+    ReservationID reservation_id = 1;
+    string owner_id = 2;
+    google.protobuf.Duration heartbeat_interval = 3; // Recommended heartbeat interval to extend reservation
+    google.protobuf.Timestamp expires_at = 4; // Expiration timestamp of this reservation
+    Metadata metadata = 6;
 }
 
-// Request to extend reservation
-message ExtendReservationRequest {
-    DatasetID dataset_id = 1;
-    string tag_name = 2;
-    string owner_id = 3;
-}
-
-// Response to extend reservation
-message ExtendReservationResponse {
-
+// Response including either a newly minted reservation or the existing reservation
+message GetOrExtendReservationResponse {
+    Reservation reservation = 1;
 }
 
 // Request to release reservation
 message ReleaseReservationRequest {
-    DatasetID dataset_id = 1;
-    string tag_name = 2;
-    string owner_id = 3;
+    ReservationID reservation_id = 1;
+    string owner_id = 2;
 }
 
 // Response to release reservation

package/protos/flyteidl/event/event.proto

@@ -99,6 +99,8 @@ message TaskNodeMetadata {
     core.CatalogCacheStatus cache_status = 1;
     // This structure carries the catalog artifact information
     core.CatalogMetadata catalog_key = 2;
+    // Captures the status of cache reservations for this execution.
+    core.CatalogReservation.Status reservation_status = 3;
 
     // In the case this task launched a dynamic workflow we capture its structure here.
     DynamicWorkflowNodeMetadata dynamic_workflow = 16;