@flyteorg/flyteidl 1.2.7 → 1.2.9

package/protos/docs/plugins/plugins.rst

@@ -38,6 +38,100 @@ will be executed concurrently.
 
 
 
+..
+   end messages
+
+
+..
+   end enums
+
+
+..
+   end HasExtensions
+
+
+..
+   end services
+
+
+
+
+.. _ref_flyteidl/plugins/dask.proto:
+
+flyteidl/plugins/dask.proto
+==================================================================
+
+
+
+
+
+.. _ref_flyteidl.plugins.DaskCluster:
+
+DaskCluster
+------------------------------------------------------------------
+
+
+
+
+
+.. csv-table:: DaskCluster type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "image", ":ref:`ref_string`", "", "Optional image to use for the scheduler as well as the default worker group. If unset, will use the default image."
+   "nWorkers", ":ref:`ref_int32`", "", "Number of workers in the default worker group"
+   "resources", ":ref:`ref_flyteidl.core.Resources`", "", "Resources assigned to the scheduler as well as all pods of the default worker group. As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices it is advised to only set limits. If requests are not explicitly set, the plugin will make sure to set requests==limits. The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit."
+
+
+
+
+
+
+
+.. _ref_flyteidl.plugins.DaskJob:
+
+DaskJob
+------------------------------------------------------------------
+
+Custom Proto for Dask Plugin
+
+
+
+.. csv-table:: DaskJob type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "namespace", ":ref:`ref_string`", "", "Optional namespace to use for the dask pods. If none is given, the namespace of the Flyte task is used"
+   "jobPodSpec", ":ref:`ref_flyteidl.plugins.JobPodSpec`", "", "Spec for the job pod"
+   "cluster", ":ref:`ref_flyteidl.plugins.DaskCluster`", "", "Cluster"
+
+
+
+
+
+
+
+.. _ref_flyteidl.plugins.JobPodSpec:
+
+JobPodSpec
+------------------------------------------------------------------
+
+Specification for the job pod
+
+
+
+.. csv-table:: JobPodSpec type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "image", ":ref:`ref_string`", "", "Optional image to use. If unset, will use the default image."
+   "resources", ":ref:`ref_flyteidl.core.Resources`", "", "Resources assigned to the job pod."
+
+
+
+
+
+
 ..
    end messages
 
@@ -501,6 +595,7 @@ Custom Proto for Spark Plugin.
    "sparkConf", ":ref:`ref_flyteidl.plugins.SparkJob.SparkConfEntry`", "repeated", ""
    "hadoopConf", ":ref:`ref_flyteidl.plugins.SparkJob.HadoopConfEntry`", "repeated", ""
    "executorPath", ":ref:`ref_string`", "", "Executor path for Python jobs."
+   "databricksConf", ":ref:`ref_string`", "", "databricksConf is base64 encoded string which stores databricks job configuration. Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure The config is automatically encoded by flytekit, and decoded in the propeller."
 
 
 

package/protos/flyteidl/core/tasks.proto

@@ -98,6 +98,14 @@ message TaskMetadata {
 
     // Indicates whether the task will generate a Deck URI when it finishes executing.
     bool generates_deck = 10;
+
+    // Arbitrary tags that allow users and the platform to store small but arbitrary labels
+    map<string, string> tags = 11;
+
+    // pod_template_name is the unique name of a PodTemplate k8s resource to be used as the base configuration if this
+    // task creates a k8s Pod. If this value is set, the specified PodTemplate will be used instead of, but applied
+    // identically as, the default PodTemplate configured in FlytePropeller.
+    string pod_template_name = 12;
 }
 
 // A Task structure that uniquely identifies a task in the system

package/protos/flyteidl/core/workflow.proto

@@ -196,6 +196,9 @@ message WorkflowMetadata {
 
     // Defines how the system should behave when a failure is detected in the workflow execution.
     OnFailurePolicy on_failure = 2;
+
+    // Arbitrary tags that allow users and the platform to store small but arbitrary labels
+    map<string, string> tags = 3; 
 }
 
 // The difference between these settings and the WorkflowMetadata ones is that these are meant to be passed down to

package/protos/flyteidl/datacatalog/datacatalog.proto

@@ -203,34 +203,55 @@ message UpdateArtifactResponse {
  * ReservationID message that is composed of several string fields.
  */
 message ReservationID {
+    // The unique ID for the reserved dataset
     DatasetID dataset_id = 1;
+
+    // The specific artifact tag for the reservation
     string tag_name = 2;
 }
 
 // Try to acquire or extend an artifact reservation. If an active reservation exists, retreive that instance.
 message GetOrExtendReservationRequest {
+    // The unique ID for the reservation
     ReservationID reservation_id = 1;
+
+    // The unique ID of the owner for the reservation
     string owner_id = 2;
-    google.protobuf.Duration heartbeat_interval = 3; // Requested reservation extension heartbeat interval
+
+    // Requested reservation extension heartbeat interval
+    google.protobuf.Duration heartbeat_interval = 3;
 }
 
 // A reservation including owner, heartbeat interval, expiration timestamp, and various metadata.
 message Reservation {
+    // The unique ID for the reservation
     ReservationID reservation_id = 1;
+
+    // The unique ID of the owner for the reservation
     string owner_id = 2;
-    google.protobuf.Duration heartbeat_interval = 3; // Recommended heartbeat interval to extend reservation
-    google.protobuf.Timestamp expires_at = 4; // Expiration timestamp of this reservation
+
+    // Recommended heartbeat interval to extend reservation
+    google.protobuf.Duration heartbeat_interval = 3;
+
+    // Expiration timestamp of this reservation
+    google.protobuf.Timestamp expires_at = 4;
+
+    // Free-form metadata associated with the artifact
     Metadata metadata = 6;
 }
 
 // Response including either a newly minted reservation or the existing reservation
 message GetOrExtendReservationResponse {
+    // The reservation to be acquired or extended
     Reservation reservation = 1;
 }
 
 // Request to release reservation
 message ReleaseReservationRequest {
+    // The unique ID for the reservation
     ReservationID reservation_id = 1;
+
+    // The unique ID of the owner for the reservation
     string owner_id = 2;
 }
 

package/protos/flyteidl/event/event.proto

@@ -51,7 +51,12 @@ message NodeExecutionEvent {
     // by the executor of the node.
     google.protobuf.Timestamp occurred_at = 4;
 
-    string input_uri = 5;
+    oneof input_value {
+        string input_uri = 5;
+
+        // Raw input data consumed by this node execution.
+        core.LiteralMap input_data = 20;
+    }
 
     oneof output_result {
         // URL to the output of the execution, it encodes all the information
@@ -165,9 +170,15 @@ message TaskExecutionEvent {
     // by the executor of the task.
     google.protobuf.Timestamp occurred_at = 7;
 
-    // URI of the input file, it encodes all the information
-    // including Cloud source provider. ie., s3://...
-    string input_uri = 8;
+
+    oneof input_value {
+      // URI of the input file, it encodes all the information
+      // including Cloud source provider. ie., s3://...
+      string input_uri = 8;
+
+      // Raw input data consumed by this task execution.
+      core.LiteralMap input_data = 19;
+    }
 
     oneof output_result {
         // URI to the output of the execution, it will be in a format that encodes all the information

package/protos/flyteidl/plugins/dask.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+import "flyteidl/core/tasks.proto";
+
+package flyteidl.plugins;
+
+option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
+
+
+// Custom Proto for Dask Plugin.
+message DaskJob {
+    // Spec for the scheduler pod.
+    DaskScheduler scheduler = 1; 
+
+    // Spec of the default worker group.
+    DaskWorkerGroup workers = 2;
+}
+
+// Specification for the scheduler pod.
+message DaskScheduler {
+    // Optional image to use. If unset, will use the default image.
+    string image = 1;
+
+    // Resources assigned to the scheduler pod.
+    core.Resources resources = 2;
+}
+
+message DaskWorkerGroup {
+    // Number of workers in the group.
+    uint32 number_of_workers = 1;
+
+    // Optional image to use for the pods of the worker group. If unset, will use the default image.
+    string image = 2;
+    
+    // Resources assigned to the all pods of the worker group.
+    // As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices 
+    // it is advised to only set limits. If requests are not explicitly set, the plugin will make
+    // sure to set requests==limits.
+    // The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit.
+    core.Resources resources = 3;
+}

package/protos/flyteidl/plugins/spark.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 
 package flyteidl.plugins;
+import "google/protobuf/struct.proto";
 
 option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
 
@@ -21,4 +22,13 @@ message SparkJob {
     map<string, string> sparkConf = 4;
     map<string, string> hadoopConf = 5;
     string executorPath = 6; // Executor path for Python jobs.
+    // Databricks job configuration.
+    // Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure.
+    google.protobuf.Struct databricksConf = 7;
+    // Databricks access token. https://docs.databricks.com/dev-tools/api/latest/authentication.html
+    // This token can be set in either flytepropeller or flytekit.
+    string databricksToken = 8;
+    // Domain name of your deployment. Use the form <account>.cloud.databricks.com.
+    // This instance name can be set in either flytepropeller or flytekit.
+    string databricksInstance = 9;
 }