@flyteorg/flyteidl 1.14.2 → 1.15.0

package/gen/pb-js/flyteidl.d.ts

@@ -5181,6 +5181,9 @@ export namespace flyteidl {
 
             /** TaskNodeOverrides containerImage */
             containerImage?: (string|null);
+
+            /** TaskNodeOverrides podTemplate */
+            podTemplate?: (flyteidl.core.IK8sPod|null);
         }
 
         /** Represents a TaskNodeOverrides. */
@@ -5201,6 +5204,9 @@ export namespace flyteidl {
             /** TaskNodeOverrides containerImage. */
             public containerImage: string;
 
+            /** TaskNodeOverrides podTemplate. */
+            public podTemplate?: (flyteidl.core.IK8sPod|null);
+
             /**
              * Creates a new TaskNodeOverrides instance using the specified properties.
              * @param [properties] Properties to set
@@ -6268,11 +6274,78 @@ export namespace flyteidl {
             public static verify(message: { [k: string]: any }): (string|null);
         }
 
+        /** Properties of a SharedMemory. */
+        interface ISharedMemory {
+
+            /** SharedMemory mountPath */
+            mountPath?: (string|null);
+
+            /** SharedMemory mountName */
+            mountName?: (string|null);
+
+            /** SharedMemory sizeLimit */
+            sizeLimit?: (string|null);
+        }
+
+        /** Represents a SharedMemory. */
+        class SharedMemory implements ISharedMemory {
+
+            /**
+             * Constructs a new SharedMemory.
+             * @param [properties] Properties to set
+             */
+            constructor(properties?: flyteidl.core.ISharedMemory);
+
+            /** SharedMemory mountPath. */
+            public mountPath: string;
+
+            /** SharedMemory mountName. */
+            public mountName: string;
+
+            /** SharedMemory sizeLimit. */
+            public sizeLimit: string;
+
+            /**
+             * Creates a new SharedMemory instance using the specified properties.
+             * @param [properties] Properties to set
+             * @returns SharedMemory instance
+             */
+            public static create(properties?: flyteidl.core.ISharedMemory): flyteidl.core.SharedMemory;
+
+            /**
+             * Encodes the specified SharedMemory message. Does not implicitly {@link flyteidl.core.SharedMemory.verify|verify} messages.
+             * @param message SharedMemory message or plain object to encode
+             * @param [writer] Writer to encode to
+             * @returns Writer
+             */
+            public static encode(message: flyteidl.core.ISharedMemory, writer?: $protobuf.Writer): $protobuf.Writer;
+
+            /**
+             * Decodes a SharedMemory message from the specified reader or buffer.
+             * @param reader Reader or buffer to decode from
+             * @param [length] Message length if known beforehand
+             * @returns SharedMemory
+             * @throws {Error} If the payload is not a reader or valid buffer
+             * @throws {$protobuf.util.ProtocolError} If required fields are missing
+             */
+            public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flyteidl.core.SharedMemory;
+
+            /**
+             * Verifies a SharedMemory message.
+             * @param message Plain object to verify
+             * @returns `null` if valid, otherwise the reason why it is not
+             */
+            public static verify(message: { [k: string]: any }): (string|null);
+        }
+
         /** Properties of an ExtendedResources. */
         interface IExtendedResources {
 
             /** ExtendedResources gpuAccelerator */
             gpuAccelerator?: (flyteidl.core.IGPUAccelerator|null);
+
+            /** ExtendedResources sharedMemory */
+            sharedMemory?: (flyteidl.core.ISharedMemory|null);
         }
 
         /** Represents an ExtendedResources. */
@@ -6287,6 +6360,9 @@ export namespace flyteidl {
             /** ExtendedResources gpuAccelerator. */
             public gpuAccelerator?: (flyteidl.core.IGPUAccelerator|null);
 
+            /** ExtendedResources sharedMemory. */
+            public sharedMemory?: (flyteidl.core.ISharedMemory|null);
+
             /**
              * Creates a new ExtendedResources instance using the specified properties.
              * @param [properties] Properties to set
@@ -6983,6 +7059,9 @@ export namespace flyteidl {
 
             /** K8sPod dataConfig */
             dataConfig?: (flyteidl.core.IDataLoadingConfig|null);
+
+            /** K8sPod primaryContainerName */
+            primaryContainerName?: (string|null);
         }
 
         /** Represents a K8sPod. */
@@ -7003,6 +7082,9 @@ export namespace flyteidl {
             /** K8sPod dataConfig. */
             public dataConfig?: (flyteidl.core.IDataLoadingConfig|null);
 
+            /** K8sPod primaryContainerName. */
+            public primaryContainerName: string;
+
             /**
              * Creates a new K8sPod instance using the specified properties.
              * @param [properties] Properties to set

package/gen/pb-js/flyteidl.js

@@ -12590,6 +12590,7 @@
                  * @property {flyteidl.core.IResources|null} [resources] TaskNodeOverrides resources
                  * @property {flyteidl.core.IExtendedResources|null} [extendedResources] TaskNodeOverrides extendedResources
                  * @property {string|null} [containerImage] TaskNodeOverrides containerImage
+                 * @property {flyteidl.core.IK8sPod|null} [podTemplate] TaskNodeOverrides podTemplate
                  */
     
                 /**
@@ -12631,6 +12632,14 @@
                  */
                 TaskNodeOverrides.prototype.containerImage = "";
     
+                /**
+                 * TaskNodeOverrides podTemplate.
+                 * @member {flyteidl.core.IK8sPod|null|undefined} podTemplate
+                 * @memberof flyteidl.core.TaskNodeOverrides
+                 * @instance
+                 */
+                TaskNodeOverrides.prototype.podTemplate = null;
+    
                 /**
                  * Creates a new TaskNodeOverrides instance using the specified properties.
                  * @function create
@@ -12661,6 +12670,8 @@
                         $root.flyteidl.core.ExtendedResources.encode(message.extendedResources, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
                     if (message.containerImage != null && message.hasOwnProperty("containerImage"))
                         writer.uint32(/* id 3, wireType 2 =*/26).string(message.containerImage);
+                    if (message.podTemplate != null && message.hasOwnProperty("podTemplate"))
+                        $root.flyteidl.core.K8sPod.encode(message.podTemplate, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim();
                     return writer;
                 };
     
@@ -12691,6 +12702,9 @@
                         case 3:
                             message.containerImage = reader.string();
                             break;
+                        case 4:
+                            message.podTemplate = $root.flyteidl.core.K8sPod.decode(reader, reader.uint32());
+                            break;
                         default:
                             reader.skipType(tag & 7);
                             break;
@@ -12723,6 +12737,11 @@
                     if (message.containerImage != null && message.hasOwnProperty("containerImage"))
                         if (!$util.isString(message.containerImage))
                             return "containerImage: string expected";
+                    if (message.podTemplate != null && message.hasOwnProperty("podTemplate")) {
+                        var error = $root.flyteidl.core.K8sPod.verify(message.podTemplate);
+                        if (error)
+                            return "podTemplate." + error;
+                    }
                     return null;
                 };
     
@@ -15117,6 +15136,150 @@
                 return GPUAccelerator;
             })();
     
+            core.SharedMemory = (function() {
+    
+                /**
+                 * Properties of a SharedMemory.
+                 * @memberof flyteidl.core
+                 * @interface ISharedMemory
+                 * @property {string|null} [mountPath] SharedMemory mountPath
+                 * @property {string|null} [mountName] SharedMemory mountName
+                 * @property {string|null} [sizeLimit] SharedMemory sizeLimit
+                 */
+    
+                /**
+                 * Constructs a new SharedMemory.
+                 * @memberof flyteidl.core
+                 * @classdesc Represents a SharedMemory.
+                 * @implements ISharedMemory
+                 * @constructor
+                 * @param {flyteidl.core.ISharedMemory=} [properties] Properties to set
+                 */
+                function SharedMemory(properties) {
+                    if (properties)
+                        for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i)
+                            if (properties[keys[i]] != null)
+                                this[keys[i]] = properties[keys[i]];
+                }
+    
+                /**
+                 * SharedMemory mountPath.
+                 * @member {string} mountPath
+                 * @memberof flyteidl.core.SharedMemory
+                 * @instance
+                 */
+                SharedMemory.prototype.mountPath = "";
+    
+                /**
+                 * SharedMemory mountName.
+                 * @member {string} mountName
+                 * @memberof flyteidl.core.SharedMemory
+                 * @instance
+                 */
+                SharedMemory.prototype.mountName = "";
+    
+                /**
+                 * SharedMemory sizeLimit.
+                 * @member {string} sizeLimit
+                 * @memberof flyteidl.core.SharedMemory
+                 * @instance
+                 */
+                SharedMemory.prototype.sizeLimit = "";
+    
+                /**
+                 * Creates a new SharedMemory instance using the specified properties.
+                 * @function create
+                 * @memberof flyteidl.core.SharedMemory
+                 * @static
+                 * @param {flyteidl.core.ISharedMemory=} [properties] Properties to set
+                 * @returns {flyteidl.core.SharedMemory} SharedMemory instance
+                 */
+                SharedMemory.create = function create(properties) {
+                    return new SharedMemory(properties);
+                };
+    
+                /**
+                 * Encodes the specified SharedMemory message. Does not implicitly {@link flyteidl.core.SharedMemory.verify|verify} messages.
+                 * @function encode
+                 * @memberof flyteidl.core.SharedMemory
+                 * @static
+                 * @param {flyteidl.core.ISharedMemory} message SharedMemory message or plain object to encode
+                 * @param {$protobuf.Writer} [writer] Writer to encode to
+                 * @returns {$protobuf.Writer} Writer
+                 */
+                SharedMemory.encode = function encode(message, writer) {
+                    if (!writer)
+                        writer = $Writer.create();
+                    if (message.mountPath != null && message.hasOwnProperty("mountPath"))
+                        writer.uint32(/* id 1, wireType 2 =*/10).string(message.mountPath);
+                    if (message.mountName != null && message.hasOwnProperty("mountName"))
+                        writer.uint32(/* id 2, wireType 2 =*/18).string(message.mountName);
+                    if (message.sizeLimit != null && message.hasOwnProperty("sizeLimit"))
+                        writer.uint32(/* id 3, wireType 2 =*/26).string(message.sizeLimit);
+                    return writer;
+                };
+    
+                /**
+                 * Decodes a SharedMemory message from the specified reader or buffer.
+                 * @function decode
+                 * @memberof flyteidl.core.SharedMemory
+                 * @static
+                 * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from
+                 * @param {number} [length] Message length if known beforehand
+                 * @returns {flyteidl.core.SharedMemory} SharedMemory
+                 * @throws {Error} If the payload is not a reader or valid buffer
+                 * @throws {$protobuf.util.ProtocolError} If required fields are missing
+                 */
+                SharedMemory.decode = function decode(reader, length) {
+                    if (!(reader instanceof $Reader))
+                        reader = $Reader.create(reader);
+                    var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.core.SharedMemory();
+                    while (reader.pos < end) {
+                        var tag = reader.uint32();
+                        switch (tag >>> 3) {
+                        case 1:
+                            message.mountPath = reader.string();
+                            break;
+                        case 2:
+                            message.mountName = reader.string();
+                            break;
+                        case 3:
+                            message.sizeLimit = reader.string();
+                            break;
+                        default:
+                            reader.skipType(tag & 7);
+                            break;
+                        }
+                    }
+                    return message;
+                };
+    
+                /**
+                 * Verifies a SharedMemory message.
+                 * @function verify
+                 * @memberof flyteidl.core.SharedMemory
+                 * @static
+                 * @param {Object.<string,*>} message Plain object to verify
+                 * @returns {string|null} `null` if valid, otherwise the reason why it is not
+                 */
+                SharedMemory.verify = function verify(message) {
+                    if (typeof message !== "object" || message === null)
+                        return "object expected";
+                    if (message.mountPath != null && message.hasOwnProperty("mountPath"))
+                        if (!$util.isString(message.mountPath))
+                            return "mountPath: string expected";
+                    if (message.mountName != null && message.hasOwnProperty("mountName"))
+                        if (!$util.isString(message.mountName))
+                            return "mountName: string expected";
+                    if (message.sizeLimit != null && message.hasOwnProperty("sizeLimit"))
+                        if (!$util.isString(message.sizeLimit))
+                            return "sizeLimit: string expected";
+                    return null;
+                };
+    
+                return SharedMemory;
+            })();
+    
             core.ExtendedResources = (function() {
     
                 /**
@@ -15124,6 +15287,7 @@
                  * @memberof flyteidl.core
                  * @interface IExtendedResources
                  * @property {flyteidl.core.IGPUAccelerator|null} [gpuAccelerator] ExtendedResources gpuAccelerator
+                 * @property {flyteidl.core.ISharedMemory|null} [sharedMemory] ExtendedResources sharedMemory
                  */
     
                 /**
@@ -15149,6 +15313,14 @@
                  */
                 ExtendedResources.prototype.gpuAccelerator = null;
     
+                /**
+                 * ExtendedResources sharedMemory.
+                 * @member {flyteidl.core.ISharedMemory|null|undefined} sharedMemory
+                 * @memberof flyteidl.core.ExtendedResources
+                 * @instance
+                 */
+                ExtendedResources.prototype.sharedMemory = null;
+    
                 /**
                  * Creates a new ExtendedResources instance using the specified properties.
                  * @function create
@@ -15175,6 +15347,8 @@
                         writer = $Writer.create();
                     if (message.gpuAccelerator != null && message.hasOwnProperty("gpuAccelerator"))
                         $root.flyteidl.core.GPUAccelerator.encode(message.gpuAccelerator, writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim();
+                    if (message.sharedMemory != null && message.hasOwnProperty("sharedMemory"))
+                        $root.flyteidl.core.SharedMemory.encode(message.sharedMemory, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
                     return writer;
                 };
     
@@ -15199,6 +15373,9 @@
                         case 1:
                             message.gpuAccelerator = $root.flyteidl.core.GPUAccelerator.decode(reader, reader.uint32());
                             break;
+                        case 2:
+                            message.sharedMemory = $root.flyteidl.core.SharedMemory.decode(reader, reader.uint32());
+                            break;
                         default:
                             reader.skipType(tag & 7);
                             break;
@@ -15223,6 +15400,11 @@
                         if (error)
                             return "gpuAccelerator." + error;
                     }
+                    if (message.sharedMemory != null && message.hasOwnProperty("sharedMemory")) {
+                        var error = $root.flyteidl.core.SharedMemory.verify(message.sharedMemory);
+                        if (error)
+                            return "sharedMemory." + error;
+                    }
                     return null;
                 };
     
@@ -16939,6 +17121,7 @@
                  * @property {flyteidl.core.IK8sObjectMetadata|null} [metadata] K8sPod metadata
                  * @property {google.protobuf.IStruct|null} [podSpec] K8sPod podSpec
                  * @property {flyteidl.core.IDataLoadingConfig|null} [dataConfig] K8sPod dataConfig
+                 * @property {string|null} [primaryContainerName] K8sPod primaryContainerName
                  */
     
                 /**
@@ -16980,6 +17163,14 @@
                  */
                 K8sPod.prototype.dataConfig = null;
     
+                /**
+                 * K8sPod primaryContainerName.
+                 * @member {string} primaryContainerName
+                 * @memberof flyteidl.core.K8sPod
+                 * @instance
+                 */
+                K8sPod.prototype.primaryContainerName = "";
+    
                 /**
                  * Creates a new K8sPod instance using the specified properties.
                  * @function create
@@ -17010,6 +17201,8 @@
                         $root.google.protobuf.Struct.encode(message.podSpec, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim();
                     if (message.dataConfig != null && message.hasOwnProperty("dataConfig"))
                         $root.flyteidl.core.DataLoadingConfig.encode(message.dataConfig, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim();
+                    if (message.primaryContainerName != null && message.hasOwnProperty("primaryContainerName"))
+                        writer.uint32(/* id 4, wireType 2 =*/34).string(message.primaryContainerName);
                     return writer;
                 };
     
@@ -17040,6 +17233,9 @@
                         case 3:
                             message.dataConfig = $root.flyteidl.core.DataLoadingConfig.decode(reader, reader.uint32());
                             break;
+                        case 4:
+                            message.primaryContainerName = reader.string();
+                            break;
                         default:
                             reader.skipType(tag & 7);
                             break;
@@ -17074,6 +17270,9 @@
                         if (error)
                             return "dataConfig." + error;
                     }
+                    if (message.primaryContainerName != null && message.hasOwnProperty("primaryContainerName"))
+                        if (!$util.isString(message.primaryContainerName))
+                            return "primaryContainerName: string expected";
                     return null;
                 };
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flyteorg/flyteidl",
-  "version": "1.14.2",
+  "version": "1.15.0",
   "description": "Compiled protocol buffers and gRPC service clients/servers for Flyte IDLs",
   "repository": {
     "type": "git",

package/protos/flyteidl/core/tasks.proto

@@ -61,12 +61,25 @@ message GPUAccelerator {
     }
 }
 
+// Metadata associated with configuring a shared memory volume for a task.
+message SharedMemory {
+    // Mount path to place in container
+    string mount_path = 1;
+    // Name for volume
+    string mount_name = 2;
+    // Size limit for shared memory. If not set, then the shared memory is equal
+    // to the allocated memory.
+    // +optional
+    string size_limit = 3;
+}
+
 // Encapsulates all non-standard resources, not captured by v1.ResourceRequirements, to
 // allocate to a task.
 message ExtendedResources {
     // GPU accelerator to select for task. Contains information about device type, and
     // for multi-instance GPUs, the partition size to use.
     GPUAccelerator gpu_accelerator = 1;
+    SharedMemory shared_memory = 2;
 }
 
 // Runtime information. This is loosely defined to allow for extensibility.
@@ -331,6 +344,10 @@ message K8sPod {
     // to understand the default paths.
     // Only K8s
     DataLoadingConfig data_config = 3;
+
+
+    // Defines the primary container name when pod template override is executed.
+    string primary_container_name = 4;
 }
 
 // Metadata for building a kubernetes object when a task is executed.

package/protos/flyteidl/core/workflow.proto

@@ -339,6 +339,10 @@ message TaskNodeOverrides {
 
     // Override for the image used by task pods.
     string container_image = 3;
+
+    // Override for the pod template used by task pods
+    //+optional
+    K8sPod pod_template = 4;
 }
 
 // A structure that uniquely identifies a launch plan in the system.

package/protos/flyteidl/plugins/spark.proto

@@ -1,10 +1,12 @@
 syntax = "proto3";
 
 package flyteidl.plugins;
-import "google/protobuf/struct.proto";
 
 option go_package = "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/plugins";
 
+import "google/protobuf/struct.proto";
+import "flyteidl/core/tasks.proto";
+
 message SparkApplication {
     enum Type {
         PYTHON = 0;
@@ -31,4 +33,10 @@ message SparkJob {
     // Domain name of your deployment. Use the form <account>.cloud.databricks.com.
     // This instance name can be set in either flytepropeller or flytekit.
     string databricksInstance = 9;
+
+    // Pod Spec for the Spark driver pod
+    core.K8sPod driverPod = 10;
+
+    // Pod Spec for the Spark executor pod
+    core.K8sPod executorPod = 11;
 }