@flyteorg/flyteidl 1.3.0 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flyteorg/flyteidl",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Compiled protocol buffers and gRPC service clients/servers for Flyte IDLs",
   "repository": {
     "type": "git",

package/protos/docs/plugins/plugins.rst

@@ -38,6 +38,100 @@ will be executed concurrently.
 
 
 
+..
+   end messages
+
+
+..
+   end enums
+
+
+..
+   end HasExtensions
+
+
+..
+   end services
+
+
+
+
+.. _ref_flyteidl/plugins/dask.proto:
+
+flyteidl/plugins/dask.proto
+==================================================================
+
+
+
+
+
+.. _ref_flyteidl.plugins.DaskCluster:
+
+DaskCluster
+------------------------------------------------------------------
+
+
+
+
+
+.. csv-table:: DaskCluster type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "image", ":ref:`ref_string`", "", "Optional image to use for the scheduler as well as the default worker group. If unset, will use the default image."
+   "nWorkers", ":ref:`ref_int32`", "", "Number of workers in the default worker group"
+   "resources", ":ref:`ref_flyteidl.core.Resources`", "", "Resources assigned to the scheduler as well as all pods of the default worker group. As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices it is advised to only set limits. If requests are not explicitly set, the plugin will make sure to set requests==limits. The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit."
+
+
+
+
+
+
+
+.. _ref_flyteidl.plugins.DaskJob:
+
+DaskJob
+------------------------------------------------------------------
+
+Custom Proto for Dask Plugin
+
+
+
+.. csv-table:: DaskJob type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "namespace", ":ref:`ref_string`", "", "Optional namespace to use for the dask pods. If none is given, the namespace of the Flyte task is used"
+   "jobPodSpec", ":ref:`ref_flyteidl.plugins.JobPodSpec`", "", "Spec for the job pod"
+   "cluster", ":ref:`ref_flyteidl.plugins.DaskCluster`", "", "Cluster"
+
+
+
+
+
+
+
+.. _ref_flyteidl.plugins.JobPodSpec:
+
+JobPodSpec
+------------------------------------------------------------------
+
+Specification for the job pod
+
+
+
+.. csv-table:: JobPodSpec type fields
+   :header: "Field", "Type", "Label", "Description"
+   :widths: auto
+
+   "image", ":ref:`ref_string`", "", "Optional image to use. If unset, will use the default image."
+   "resources", ":ref:`ref_flyteidl.core.Resources`", "", "Resources assigned to the job pod."
+
+
+
+
+
+
 ..
    end messages
 
@@ -501,6 +595,7 @@ Custom Proto for Spark Plugin.
    "sparkConf", ":ref:`ref_flyteidl.plugins.SparkJob.SparkConfEntry`", "repeated", ""
    "hadoopConf", ":ref:`ref_flyteidl.plugins.SparkJob.HadoopConfEntry`", "repeated", ""
    "executorPath", ":ref:`ref_string`", "", "Executor path for Python jobs."
+   "databricksConf", ":ref:`ref_string`", "", "databricksConf is base64 encoded string which stores databricks job configuration. Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure The config is automatically encoded by flytekit, and decoded in the propeller."
 
 
 

package/protos/flyteidl/plugins/dask.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+import "flyteidl/core/tasks.proto";
+
+package flyteidl.plugins;
+
+option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
+
+
+// Custom Proto for Dask Plugin.
+message DaskJob {
+    // Spec for the scheduler pod.
+    DaskScheduler scheduler = 1; 
+
+    // Spec of the default worker group.
+    DaskWorkerGroup workers = 2;
+}
+
+// Specification for the scheduler pod.
+message DaskScheduler {
+    // Optional image to use. If unset, will use the default image.
+    string image = 1;
+
+    // Resources assigned to the scheduler pod.
+    core.Resources resources = 2;
+}
+
+message DaskWorkerGroup {
+    // Number of workers in the group.
+    uint32 number_of_workers = 1;
+
+    // Optional image to use for the pods of the worker group. If unset, will use the default image.
+    string image = 2;
+    
+    // Resources assigned to the all pods of the worker group.
+    // As per https://kubernetes.dask.org/en/latest/kubecluster.html?highlight=limit#best-practices 
+    // it is advised to only set limits. If requests are not explicitly set, the plugin will make
+    // sure to set requests==limits.
+    // The plugin sets ` --memory-limit` as well as `--nthreads` for the workers according to the limit.
+    core.Resources resources = 3;
+}

package/protos/flyteidl/plugins/spark.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 
 package flyteidl.plugins;
+import "google/protobuf/struct.proto";
 
 option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
 
@@ -21,4 +22,13 @@ message SparkJob {
     map<string, string> sparkConf = 4;
     map<string, string> hadoopConf = 5;
     string executorPath = 6; // Executor path for Python jobs.
+    // Databricks job configuration.
+    // Config structure can be found here. https://docs.databricks.com/dev-tools/api/2.0/jobs.html#request-structure.
+    google.protobuf.Struct databricksConf = 7;
+    // Databricks access token. https://docs.databricks.com/dev-tools/api/latest/authentication.html
+    // This token can be set in either flytepropeller or flytekit.
+    string databricksToken = 8;
+    // Domain name of your deployment. Use the form <account>.cloud.databricks.com.
+    // This instance name can be set in either flytepropeller or flytekit.
+    string databricksInstance = 9;
 }